Question

我编写了一个kafka-producer代码来生成avro数据，但是在序列化数据时显示以下错误：

线程“主”中的异常 org.apache.kafka.common.errors.SerializationException：错误序列化Avro消息引起原因：java.net.UnknownHostException：sandbox-hdf.hortonworks.com atjava.net.AbstractPlainSocketImpl.connect（AbstractPlainSocketImpl.java:184）在java.net.PlainSocketImpl.connect（PlainSocketImpl.java:172）在java.net.SocksSocketImpl.connect（SocksSocketImpl.java:392）在java.net.Socket.connect（Socket.java:589）在java.net.Socket.connect（Socket.java:538）在sun.net.NetworkClient.doConnect（NetworkClient.java:180）在sun.net.www.http.HttpClient.openServer（HttpClient.java:463）在sun.net.www.http.HttpClient.openServer（HttpClient.java:558）在sun.net.www.http.HttpClient。（HttpClient.java:242）在sun.net.www.http.HttpClient.New（HttpClient.java:339）在sun.net.www.http.HttpClient.New（HttpClient.java:357）在sun.net.www.protocol.http.HttpURLConnection.getNewHttpClient（HttpURLConnection.java:1220）在sun.net.www.protocol.http.HttpURLConnection.plainConnect0（HttpURLConnection.java:1156）在sun.net.www.protocol.http.HttpURLConnection.plainConnect（HttpURLConnection.java:1050）在sun.net.www.protocol.http.HttpURLConnection.connect（HttpURLConnection.java:984）......

以下是我的生产者代码：

package com.perfaware.kafka01;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.example.Customer;
import io.confluent.kafka.serializers.KafkaAvroSerializer;
import org.apache.kafka.clients.producer.*;
import org.apache.kafka.common.serialization.StringSerializer;

import java.util.Properties;
import java.util.concurrent.ExecutionException;

public class producerAvro {

    public static void main(String[] args) throws InterruptedException, ExecutionException {
        Properties properties = new Properties();
        // setting producer properties

        properties.setProperty("bootstrap.servers", "localhost:9092");
        properties.setProperty("acks", "1");
        properties.setProperty("retries", "10");

        // Serialization(avro part)
        properties.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
        properties.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName());
        properties.setProperty("schema.registry.url", "http://sandbox-hdf.hortonworks.com:7788/api/v1");

        Producer<String, Customer> producer = new KafkaProducer<String, Customer>(properties);

        String topic = "topic1";

        Customer customer = Customer.newBuilder()
                .setAge(21)
                .setAutomatedEmail(false)
                .setFirstName("Manish")
                .setLastName("B.")
                .setHeight(176f)
                .setWeight(62f)
                .build();

        ProducerRecord<String, Customer> producerRecord = new ProducerRecord<String, Customer>("topic1", customer);

        System.out.println(customer);
        producer.send(producerRecord, new Callback() {
            public void onCompletion(RecordMetadata metadata, Exception exception) {
                if (exception == null) {
                    System.out.println(metadata.toString());
                } else {
                    exception.printStackTrace();
                }
            }
        }).get();

        producer.flush();
        producer.close();
    }
}

如果有帮助，我还将附加我的pom.xml文件：

 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>Kafka_Avro</groupId>
  <artifactId>Kafka_Avro_Practise</artifactId>
  <version>0.0.1-SNAPSHOT</version>

  <properties>
    <avro.verion>1.7.4</avro.verion>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <maven.compiler.source>1.8</maven.compiler.source>
        <maven.compiler.target>1.8</maven.compiler.target>
        <confluent.version>3.1.1</confluent.version>
  </properties>

 <repositories>
        <repository>
            <id>confluent</id>
            <url>http://packages.confluent.io/maven/</url>
        </repository>
 </repositories>

  <dependencies>


  <!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka-tools -->
<dependency>
    <groupId>org.apache.kafka</groupId>
    <artifactId>kafka-tools</artifactId>
    <version>2.0.0</version>
</dependency>



  <dependency>
            <groupId>io.confluent</groupId>
            <artifactId>kafka-avro-serializer</artifactId>
            <version>3.1.1</version>
        </dependency>

  <!-- https://mvnrepository.com/artifact/org.apache.avro/avro -->
<dependency>
    <groupId>org.apache.avro</groupId>
    <artifactId>avro</artifactId>
    <version>1.8.2</version>
</dependency>

<!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka-clients -->
<dependency>
    <groupId>org.apache.kafka</groupId>
    <artifactId>kafka-clients</artifactId>
    <version>1.1.1</version>
</dependency>

<!-- https://mvnrepository.com/artifact/org.apache.avro/avro-compiler -->
<dependency>
    <groupId>org.apache.avro</groupId>
    <artifactId>avro-compiler</artifactId>
    <version>1.8.2</version>
</dependency>

<dependency>
            <groupId>com.google.code.gson</groupId>
            <artifactId>gson</artifactId>
            <version>2.3.1</version>
            <scope>provided</scope>
        </dependency>

<!-- https://mvnrepository.com/artifact/org.apache.maven.plugins/maven-compiler-plugin -->
<dependency>
    <groupId>org.apache.maven.plugins</groupId>
    <artifactId>maven-compiler-plugin</artifactId>
    <version>3.8.0</version>
</dependency>

<!-- https://mvnrepository.com/artifact/org.apache.avro/avro-mapred -->
<dependency>
    <groupId>org.apache.avro</groupId>
    <artifactId>avro-mapred</artifactId>
    <version>1.8.1</version>
</dependency>

<!-- https://mvnrepository.com/artifact/org.apache.avro/avro-ipc -->
<dependency>
    <groupId>org.apache.avro</groupId>
    <artifactId>avro-ipc</artifactId>
    <version>1.8.2</version>
</dependency>

<!-- https://mvnrepository.com/artifact/commons-codec/commons-codec -->
<dependency>
    <groupId>commons-codec</groupId>
    <artifactId>commons-codec</artifactId>
    <version>1.11</version>
</dependency>

  </dependencies>

   <build>
        <plugins>
            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <version>3.0.0</version>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>

我还尝试过更改值序列化器：

com.hortonworks.registries.schemaregistry.serdes.avro.kafka.KafkaAvroSerializer

但这并不能解决问题。

Answer 1

UnknownHostException：沙盒-hdf.hortonworks.com

如果您使用的是沙箱，则应编辑/etc/hosts文件以使其成为已知主机

但是，如果使用注册表，则肯定要使用Hortonworks序列化程序。目前尚不清楚您在使用它时遇到什么错误，但是如果相同，则是网络问题，与Avro无关。

"value.serializer","com.hortonworks.registries.schemaregistry.serdes.avro.kafka.KafkaAvroSerializer"

另外，bootstrap.servers可能还需要解析到沙箱的Kafka实例，而不仅仅是本地主机

如果您确实想使用Confluent，但不确定是否可以使用，则需要使用一致的Kafka版本号：例如您已经放置了基于Kafka 1.1.1的Kafka 2.0，3.1.1和Confluent 0.10.x。
与Avro类似，例如，尽管不需要IPC或Mapred Avro库来使代码正常工作，但所有内容都应仅设置为1.8.1。也许也不是编译器。

Kafka Producer-Consumer无法产生/使用Avro数据

1 个答案: