Question

我想结合kafka-node和node tail来观察一些日志文件并将任何新条目发送到kafka集群。我有以下代码可以使用。

var info={
         "zookeepers":"127.0.0.1:9500",
         "topics":{
             "server":[
                "/tmp/server-1.log",
                "/tmp/server-2.log"
             ],
             "audit":[
                 "/tmp/audit-1.log",
                 "/tmp/audit-2.log"
             ]
         }
     }

function dldmProducer (logFiletoTail, fileTopic, zookeepers){
    var kafka = require('kafka-node');
    var Producer = kafka.Producer;
    var client = new kafka.Client(zookeepers);
    var producer = new Producer(client);
    var partitionNo = 0

    producer.on('ready', function () {
        var Tail = require('tail').Tail;
        var tail = new Tail(logFiletoTail);
        tail.on("line", function(data) {
            dataArray=data.split("\n")
            var payloads = [{ topic: fileTopic, messages: dataArray, partition: partitionNo },];
            partitionNo = (partitionNo +1)%3;
            producer.send(payloads, function(err, data){
                //console.log(payloads);
                //console.log(data);
            });
        });
        tail.on("error", function(error) {
            //console.log('TAIL ERROR: ', error);
        });

    });
    producer.on('error', function(error){
        //console.log('PRODUCER ERROR: ', error)
    });
}

for (var topicItem in info['topics']){
    for (var i = 0; i<info['topics'][topicItem].length; i++){
        dldmProducer(info['topics'][topicItem][i],topicItem,info['zookeepers'])
    }
}

但问题是，错误处理是不存在的，所以我担心如果在producer.send(payloads,function(err,data){})期间出现问题，payloads将永远丢失。我将函数dldmProducer重写为以下函数，试图捕获任何未成功发送的有效负载并重新发送它们。（如果有效负载成功发送，则响应将在data。data为undefined，否则，或者至少是我的理解）

function dldmProducer (logFiletoTail, fileTopic, zookeepers){
    var kafka = require('kafka-node');
    var Producer = kafka.Producer;
    var client = new kafka.Client(zookeepers);
    var producer = new Producer(client);
    var partitionNo = 0
    var buffer = []
    var Tail = require('tail').Tail;
    var tail = new Tail(logFiletoTail);
    tail.on("line", function(data) {
        var dataArray=data.split("\n")
        console.log(dataArray)
        buffer = buffer.concat(dataArray)
    });
    tail.on("error", function(error) {
        console.log('TAIL ERROR: ', error);
    });
    producer.on('ready', function () {
        while (true){
            var logItem = buffer.shift()
            if (logItem){
                console.log(logItem)
                var payloads = [{ topic: fileTopic, messages: [logItem], partition: partitionNo },];
                partitionNo = (partitionNo +1)%3;
                producer.send(payloads, function(error, data){
                    if (!data){
                        console.log(data)
                        buffer.push(logItem)
                    }
                    if (error){
                        console.log('KAFKA RESPONSE ERROR: ', error)
                    }
                });
            }   
        }

    });

    producer.on('error', function(error){
        console.log('PRODUCER ERROR: ', error)
    });
}

该功能似乎不起作用。它甚至没有正确地拖尾文件，即使我不断地将虚拟内容写入文件，也几乎没有任何来自文件的信息记录在屏幕上。我还不习惯javascript那种想法所以任何指针都会很棒。

如何组合kafka节点生成器和节点尾部？

0 个答案: