Question

我将消息从kafka消费者推送到mongodb。

如果我将MongoClient.connect(url, function(err, client) {})放在consumer.on('message', callback)的回调中，则使用者可以从头开始获取消息。

示例代码：

consumer.on('message', (message) => {
    MongoDB.connectDB((err) => {
        if (err) throw err
        const db = MongoDB.getDB();
        const collectionKafka = db.collection('transaction');
        try {
            insertMessage(message, collectionKafka);  
        } catch (e) {
            throw e
        }
    })
});

但是，如果我将consumer.on('message', callback)放在MongoClient.connect(url, function(err, client) {})内，则消费者不会从头开始获得消息。

使用此代码，使用者仅使用最新消息，而不能从头开始使用。如何解决这个问题

const kafka = require('kafka-node');
const mongo = require('mongodb');
const assert = require('assert');

const { Consumer, Offset, KafkaClient } = kafka;
const { MongoClient } = mongo;

const topic = 'testprocessing';

const url = 'mongodb://localhost:27017';
const dbName = 'test_kafka_processing';
let mongodb;

const client = new KafkaClient({kafkaHost: 'localhost:9092'});

const topics = [{
    topic: 'testprocessing',
    offset: 0,
    partition: 0
}];

const options = {
    autoCommit: false,
    fetchMaxWaitMs: 1000,
    fetchMaxBytes: 1024 * 1024,
    fromOffset: true
};

MongoClient.connect(url, function(err, client) {
    assert.equal(null, err);
    mongodb = client.db(dbName);

    consumer.on('message', (message) => {
        const collection = mongodb.collection('transaction');
        // Insert some documents
        let msg = JSON.parse(message.value);
        collection.insertOne(msg,
            function(err, result) {
                assert.equal(err, null);
                console.log("Inserted message into the collection");
                //callback(result);
        });
    });

    consumer.on('error', (err) => {
        console.log('error', err);
    });
});

const consumer = new Consumer(client, topics, options);

即使我将初始consumer放在MongoClient.connect的回调中。消费者仍然无法提取消息

const kafka = require('kafka-node');
const mongo = require('mongodb');
const assert = require('assert');

const { Consumer, Offset, KafkaClient } = kafka;
const { MongoClient } = mongo;

const topic = 'testprocessing';

const url = 'mongodb://localhost:27017';
const dbName = 'test_kafka_processing';
let mongodb;

const client = new KafkaClient({kafkaHost: 'localhost:9092'});

const topics = [{
    topic: 'testprocessing',
    offset: 0,
    partition: 0
}];

const options = {
    autoCommit: false,
    fetchMaxWaitMs: 1000,
    fetchMaxBytes: 1024 * 1024,
    fromOffset: true
};

MongoClient.connect(url, function(err, client) {
    assert.equal(null, err);
    mongodb = client.db(dbName);
    const consumer = new Consumer(client, topics, options); // <== put the initial of consumer at this time
    consumer.on('message', (message) => {
        const collection = mongodb.collection('transaction');
        // Insert some documents
        let msg = JSON.parse(message.value);
        collection.insertOne(msg,
            function(err, result) {
                assert.equal(err, null);
                console.log("Inserted message into the collection");
                //callback(result);
        });
    });

    consumer.on('error', (err) => {
        console.log('error', err);
    });
});

Answer 1

使用者未在“暂停”模式下初始化。因此，使用者将在建立MongoDB连接之前，甚至在建立.on('message')处理程序之前，开始获取Kafka消息。

解决方法是在使用者的选项对象中设置paused选项标志。

Link to the lib code line

const options = {
    autoCommit: false,
    fetchMaxWaitMs: 1000,
    fetchMaxBytes: 1024 * 1024,
    fromOffset: true,
    paused: true // <-- "start the consumer paused"
};

const consumer = new Consumer(client, topics, options);

MongoClient.connect(url, function(err, client) {
    assert.equal(null, err);
    mongodb = client.db(dbName);

    consumer.on('message', (message) => {
        const collection = mongodb.collection('transaction');
        // Insert some documents
        let msg = JSON.parse(message.value);
        collection.insertOne(msg,
            function(err, result) {
                assert.equal(err, null);
                console.log("Inserted message into the collection");
                //callback(result);
        });
    });

    consumer.on('error', (err) => {
        console.log('error', err);
    });

    consumer.resume() // <-- the consumer starts processing kafka messages after a MongoDB connection has been established
});

将消费.on（'message，callback）放入另一个回调

1 个答案: