考虑Express服务器+ Kafka Producer:
var express = require('express');
var app = express();
var Kafka = require('node-rdkafka');
// Kafka
var producer = new Kafka.Producer({
//'debug' : 'all',
'metadata.broker.list': 'localhost:9092',
dr_cb: true, //delivery report callback
});
var topicName = 'test';
//logging debug messages, if debug is enabled
producer.on('event.log', function (log) {
console.log(log);
});
//logging all errors
producer.on('event.error', function (err) {
console.error('Error from producer');
console.error(err);
});
//counter to stop this sample after maxMessages are sent
var counter = 0;
var maxMessages = 10;
producer.on('delivery-report', function (err, report) {
console.log('delivery-report: ' + JSON.stringify(report));
counter++;
});
//Wait for the ready event before producing
producer.on('ready', function (arg) {
console.log('producer ready.' + JSON.stringify(arg));
for (var i = 0; i < maxMessages; i++) {
var value = Buffer.from('value-' + i);
var key = 'key-' + i;
// if partition is set to -1, librdkafka will use the default partitioner
var partition = -1;
var headers = [{ header: 'header value' }];
producer.produce(topicName, partition, value, key, Date.now(), '', headers);
}
//need to keep polling for a while to ensure the delivery reports are received
var pollLoop = setInterval(function () {
producer.poll();
if (counter === maxMessages) {
clearInterval(pollLoop);
producer.disconnect();
}
}, 1000);
});
producer.on('disconnected', function (arg) {
console.log('producer disconnected. ' + JSON.stringify(arg));
});
//starting the producer
producer.connect();
// End Kafka
app.get('/', (req, res) => res.send('Ready to send messages!'));
app.post('/:maxMessages', function (req, res) {
if (req.params.maxMessages) {
var maxMessages = parseInt(req.params.maxMessages);
console.log(`Got it ... ${maxMessages}`);
for (var i = 0; i < maxMessages; i++) {
var value = new Buffer.from('MyProducerTest - value-' + i);
var key = 'key-' + i;
// if partition is set to -1, librdkafka will use the default partitioner
var partition = -1;
producer.produce(topicName, partition, value, key);
} // end for
res.status(200).json({
success: true,
});
} // end if
console.log('Out of IF ...');
}); // end app.post()
app.listen(3000, () => console.log('Example app listening on port 3000!'));
和消费者:
var Transform = require('stream').Transform;
var Kafka = require('node-rdkafka');
var stream = Kafka.KafkaConsumer.createReadStream(
{
'metadata.broker.list': 'localhost:9092',
'group.id': 'librd-test',
'socket.keepalive.enable': true,
'enable.auto.commit': false,
},
{},
{
topics: 'test',
waitInterval: 0,
objectMode: false,
}
);
stream.on('error', function (err) {
if (err) console.log(err);
process.exit(1);
});
stream.pipe(process.stdout);
stream.on('error', function (err) {
console.log(err);
process.exit(1);
});
stream.consumer.on('event.error', function (err) {
console.log(err);
});
console.log('Waiting to consume...');
当我向http://localhost:3000/20
发出邮递员POST请求时,我进入生产者屏幕:
Error: Producer not connected
at Producer.produce (C:\Development-T410\Kafka\node_modules\node-rdkafka\lib\producer.js:125:11)
at C:\Development-T410\Kafka\app.js:79:16
at Layer.handle [as handle_request] (C:\Development-T410\Kafka\node_modules\express\lib\router\layer.js:95:5)
at next (C:\Development-T410\Kafka\node_modules\express\lib\router\route.js:137:13)
at Route.dispatch (C:\Development-T410\Kafka\node_modules\express\lib\router\route.js:112:3)
at Layer.handle [as handle_request] (C:\Development-T410\Kafka\node_modules\express\lib\router\layer.js:95:5)
at C:\Development-T410\Kafka\node_modules\express\lib\router\index.js:281:22
at param (C:\Development-T410\Kafka\node_modules\express\lib\router\index.js:354:14)
at param (C:\Development-T410\Kafka\node_modules\express\lib\router\index.js:365:14)
at Function.process_params (C:\Development-T410\Kafka\node_modules\express\lib\router\index.js:410:3)
关于消费者:
LibrdKafkaError: Local: Broker transport failure
at Function.createLibrdkafkaError [as create] (C:\Development-T410\Kafka\node_modules\node-rdkafka\lib\error.js:411:10)
at C:\Development-T410\Kafka\node_modules\node-rdkafka\lib\client.js:350:28 {
origin: 'local',
message: 'broker transport failure',
code: -195,
errno: -195
}
为什么?