Kafka生产者尝试一次发送批量消息(10k),但是在传输3k之后,它开始抛出错误。我在下面的代码中使用了collect函数,这可能是一个问题,所以我想用map替换它。
//Memberjson is JavaRDD<String>
for (String outputMbrJson : memberjson.collect()) {
try {
Producer kafkaProducer = new Producer(configFile);
kafkaProducer.runProducer(Arrays.asList(outputMbrJson).iterator());
}
catch (Throwable e) {
e.printStackTrace();
}
}
---------------------------------------
//runProducer method
public Iterator<String> runProducer(Iterator<String> jsons) throws Exception {
final Producer<Long, String> producer = createProducer();
ArrayList<String> ret = new ArrayList<String>();
try {
while (jsons.hasNext()) {
String jsonobj = jsons.next();
ret.add(EMPTY_STRING);
try {
ProducerRecord<Long, String> record = new ProducerRecord<Long, String>(conf.getProperty("producer.topic"), jsonobj.toString());
producer.send(record);
} catch (Exception e) {
System.out.println(e.getMessage());
}
}
}
catch (Exception e) {
e.printStackTrace();
}
finally {
producer.flush();
producer.close();
}
return ret.iterator();
}