我在使用来自kafka的事件时遇到了BufferUnderflowException错误。我能够成功读取大约100k消息，之后我发现了这个错误。
我遇到过几个stackoverflow帖子，当服务器版本和消费者版本不符合此link时，会发生这种情况。但是，我的消费者和服务器版本匹配。
我很感激您对如何调试此问题的想法。
org.apache.kafka.common.protocol.types.SchemaException: Error reading field 'responses': Error reading field 'topic': java.nio.BufferUnderflowException
at org.apache.kafka.common.protocol.types.Schema.read(Schema.java:71)
at org.apache.kafka.clients.NetworkClient.handleCompletedReceives(NetworkClient.java:464)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:279)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.clientPoll(ConsumerNetworkClient.java:303)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:197)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:187)
at org.apache.kafka.clients.consumer.KafkaConsumer.pollOnce(KafkaConsumer.java:877)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:829)
at com.howtoprogram.kafka.ConsumerTest.main(ConsumerTest.java:58)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
Kafka服务器版
kafka_2.10-0.9.0.0
Kafka消费者版
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.10</artifactId>
<version>0.9.0.0</version>
</dependency>
消费者罐
kafka_2.10-0.9.0.0.jar
代码
import java.io.PrintStream;
import java.util.Arrays;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
public class ConsumerTest {
public static AtomicLong totalMessages = new AtomicLong(0);
static String bootstrapServers = "10.0.1.1:9092";
static String consumerGroup = ConsumerTest.class.getSimpleName();
static String topic = "events.topic";
public static void main(String[] args) {
Properties props = new Properties();
props.put("bootstrap.servers", bootstrapServers);
props.put("group.id", consumerGroup);
props.put("enable.auto.commit", "true");
props.put("auto.commit.interval.ms", "1000");
props.put("auto.offset.reset", "earliest");
props.put("session.timeout.ms", "30000");
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
System.out.println("init kafka consumer kafka - 0.9.0.0" );
try {
KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<>(props);
kafkaConsumer.subscribe(Arrays.asList(topic));
System.out.println("subscribed to kafka topic " + topic );
while (true) {
ConsumerRecords<String, String> records = kafkaConsumer.poll(1000);
totalMessages.addAndGet(records.count());
System.out.println("totalMessages = " + totalMessages.get());
for (ConsumerRecord<String, String> record : records) {
}
}}
catch (Exception e){
System.out.println("something went wrong");
e.printStackTrace(new PrintStream(System.out));
}
}
}
不确定是否重要但是，制作人是一个火花作业，版本0.8.2 ..所以，有点旧。
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-kafka-0-8_2.11</artifactId>
<version>2.0.1</version>
</dependency>
----- ------更新
我能够与我写的另一位制片人重现这一点。生产者基于使用kafka_2.10-0.9.0.0.jar的代码（与消费者相同）
import java.io.PrintStream;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
public class ProducerTest {
static String topic = "events.topic";
static String bootstrapServers = "10.0.1.1:9092";
static AtomicLong counter = new AtomicLong();
public static void main(String[] args) {
Properties props = new Properties();
props.put("bootstrap.servers", bootstrapServers);
props.put("acks", "all");
props.put("retries", 0);
props.put("batch.size", 16384);
props.put("linger.ms", 1);
props.put("buffer.memory", 33554432);
props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
Producer<String, String> producer = null;
DummyObject event = new DummyObject();
event.id = "somerandompsid";
event.country_cd = "US";
event.cont_id = "programId";
event.language = "EN";
event.category = "category";
event.subcategory = "sub category";
event.source_id = "12234";
Gson gson = new GsonBuilder().create();
String json = gson.toJson(event);
try {
producer = new KafkaProducer<>(props);
System.out.println("kafka producer " + bootstrapServers + ",topic= " + topic);
for (int i = 0; i < 1000000; i++) {
producer.send(new ProducerRecord<String, String>(topic, json));
counter.addAndGet(1);
// System.out.println("Sent:" + json);
}
System.out.println("sent messages " + counter.get());
} catch (Exception e) {
e.printStackTrace();
e.printStackTrace(new PrintStream(System.out));
} finally {
producer.close();
}
}
}
-------另一次更新-------
将消费者降级为使用 - kafka_2.11-0.8.2.1.jar并使用kafka streams更改它。虽然我没有遇到上述缓冲区流量异常，但消费者有时会卡住，并且不会读取kafka上的所有消息。我也没有看到任何例外。以下是代码，
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicLong;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.message.MessageAndMetadata;
public class KafkaConsumerThread extends Thread {
private final ConsumerConnector consumer;
private final String topic;
AtomicLong counter = new AtomicLong(0);
public KafkaConsumerThread(String topic) {
consumer = kafka.consumer.Consumer.createJavaConsumerConnector(createConsumerConfig());
this.topic = topic;
}
private static ConsumerConfig createConsumerConfig() {
Properties props = new Properties();
props.put("zookeeper.connect", KafkaProperties.zkConnect);
props.put("group.id", KafkaProperties.groupId);
props.put("zookeeper.session.timeout.ms", "4000");
props.put("zookeeper.sync.time.ms", "200");
props.put("auto.commit.interval.ms", "1000");
System.out.println("listening to zookeper " + KafkaProperties.zkConnect);
return new ConsumerConfig(props);
}
public void run() {
System.out.println("listening to topic " + topic);
try {
Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
topicCountMap.put(topic, new Integer(1));
Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap);
KafkaStream<byte[], byte[]> stream = consumerMap.get(topic).get(0);
System.out.println("got the stream");
ConsumerIterator<byte[], byte[]> it = stream.iterator();
System.out.println("iterating the stream");
while (it.hasNext()) {
MessageAndMetadata<byte[], byte[]> next = it.next();
String msg = new String(next.message());
// System.out.println(msg);
counter.incrementAndGet();
System.out.println("Consumed = " + counter.get());
}
} catch (Exception e) {
System.out.println("something went wrong ");
e.printStackTrace(System.out);
}
}
public static void main(String args[]) {
try {
KafkaConsumerThread consumerThread = new KafkaConsumerThread(KafkaProperties.topic);
consumerThread.start();
} catch (Exception e) {
System.out.println("something went wrong main loop ");
e.printStackTrace(System.out);
}
for (;;) {
}
}
}