我正在使用StreamSets作为提取工具,将记录从Oracle数据库提取到Kafka主题。现在,我想通过StreamSets本身使用它,还想计算Kafka主题中的记录数。
我该怎么做。请帮助
答案 0 :(得分:1)
您可以使用StreamSets Data Collector的history
REST API来检索每个阶段具有记录计数的数据。例如,这是给定管道的最后一次运行的计数器。我正在使用出色的jq工具在命令行中解析JSON。
$ curl -s -u admin:admin -H 'X-Requested-By:sdc' http://localhost:18630/rest/v1/pipeline/RedshiftStreamingwithKinesisFirehose537add73-bb16-4358-a26a-a51576dea32b/history | jq -r .[0].metrics | jq .counters
{
"pipeline.batchCount.counter": {
"count": 1029
},
"pipeline.batchErrorMessages.counter": {
"count": 0
},
"pipeline.batchErrorRecords.counter": {
"count": 0
},
"pipeline.batchInputRecords.counter": {
"count": 648226
},
"pipeline.batchOutputRecords.counter": {
"count": 648226
},
"stage.ExpressionEvaluator_01.errorRecords.counter": {
"count": 0
},
"stage.ExpressionEvaluator_01.inputRecords.counter": {
"count": 648226
},
"stage.ExpressionEvaluator_01.outputRecords.counter": {
"count": 648226
},
"stage.ExpressionEvaluator_01.stageErrors.counter": {
"count": 0
},
"stage.ExpressionEvaluator_01:ExpressionEvaluator_01OutputLane15561338960790.outputRecords.counter": {
"count": 648226
},
"stage.FieldOrder_01.errorRecords.counter": {
"count": 0
},
"stage.FieldOrder_01.inputRecords.counter": {
"count": 648226
},
"stage.FieldOrder_01.outputRecords.counter": {
"count": 648226
},
"stage.FieldOrder_01.stageErrors.counter": {
"count": 0
},
"stage.FieldOrder_01:FieldOrder_01OutputLane15561351879260.outputRecords.counter": {
"count": 648226
},
"stage.FieldTypeConverter_01.errorRecords.counter": {
"count": 0
},
"stage.FieldTypeConverter_01.inputRecords.counter": {
"count": 648226
},
"stage.FieldTypeConverter_01.outputRecords.counter": {
"count": 648226
},
"stage.FieldTypeConverter_01.stageErrors.counter": {
"count": 0
},
"stage.FieldTypeConverter_01:FieldTypeConverter_01OutputLane15560499048280.outputRecords.counter": {
"count": 648226
},
"stage.KinesisFirehose_01.errorRecords.counter": {
"count": 0
},
"stage.KinesisFirehose_01.inputRecords.counter": {
"count": 648226
},
"stage.KinesisFirehose_01.outputRecords.counter": {
"count": 648226
},
"stage.KinesisFirehose_01.stageErrors.counter": {
"count": 0
},
"stage.MySQLBinaryLog_01.errorRecords.counter": {
"count": 0
},
"stage.MySQLBinaryLog_01.inputRecords.counter": {
"count": 0
},
"stage.MySQLBinaryLog_01.outputRecords.counter": {
"count": 648226
},
"stage.MySQLBinaryLog_01.stageErrors.counter": {
"count": 0
},
"stage.MySQLBinaryLog_01:MySQLBinaryLog_01OutputLane15561313696850.outputRecords.counter": {
"count": 648226
},
"stage.StreamSelector_01.errorRecords.counter": {
"count": 0
},
"stage.StreamSelector_01.inputRecords.counter": {
"count": 648226
},
"stage.StreamSelector_01.outputRecords.counter": {
"count": 648226
},
"stage.StreamSelector_01.stageErrors.counter": {
"count": 0
},
"stage.StreamSelector_01:StreamSelector_01OutputLane1556133811620.outputRecords.counter": {
"count": 0
},
"stage.StreamSelector_01:StreamSelector_01OutputLane1556133816638.outputRecords.counter": {
"count": 648226
},
"stage.Trash_01.errorRecords.counter": {
"count": 0
},
"stage.Trash_01.inputRecords.counter": {
"count": 0
},
"stage.Trash_01.outputRecords.counter": {
"count": 0
},
"stage.Trash_01.stageErrors.counter": {
"count": 0
}
}