Question

在使用putinfluxDB之前，我有一组从kafka主题，mergeContent和ExecuteScript读取的进程，以读取json字段并映射到influxDB。我有特定的脚本，因为json消息的格式不同，存在不同的字段和标记，因此需要这样做。我想创建一个通用脚本，该脚本可以读取json中的任何字段并将其相应地映射到influxDB进程。

例如：

flowFile = session.get();
if (flowFile != null) {

var StreamCallback = 
   Java.type("org.apache.nifi.processor.io.StreamCallback");
var IOUtils = Java.type("org.apache.commons.io.IOUtils");
var StandardCharsets = Java.type("java.nio.charset.StandardCharsets");
var transformed_message = {};
var error = false;
var line = "test_measurement";
var sep = "\n"

// Get attributes
flowFile = session.write(flowFile, new StreamCallback(function (inputStream, 
 outputStream) {
    // Read input FlowFile content
    var type = flowFile.getAttribute('type')
    var time = Number(flowFile.getAttribute('filename'))
    var meta_name = flowFile.getAttribute('meta_name')
    var remote_source_name = flowFile.getAttribute('gethttp.remote.source')

    var content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); // 
     message or content
    var message_content = {};
    try {
        message_content = JSON.parse(content);
        transformed_message.state = (((message_content || {}).state || "null"));
        transformed_message.new_records = (((message_content || {}).new_records || 0));
        transformed_message.updated_records = (((message_content || {}).updated_records || 0));
        transformed_message.changed_records = (((message_content || {}).changed_records || 0));
        transformed_message.deleted_records = (((message_content || {}).deleted_records || 0));
        transformed_message.missing_records = (((message_content || {}).missing_records || 0));

        line = line + ",remote_source_name="+ remote_source_name + ",meta_name="+meta_name+",message_type=" + type + " " 
                 + "new_records=" + transformed_message.new_records + ","
                 + "updated_records=" + transformed_message.updated_records + ","
                 + "changed_records=" + transformed_message.changed_records + ","
                 + "deleted_records=" + transformed_message.deleted_records + ","
                 + "missing_records=" + transformed_message.missing_records + " "
                 + time * 1000000 + sep;


        // Write output content
        if (transformed_message) {
            outputStream.write(line.getBytes(StandardCharsets.UTF_8));
        }
    } catch (e) {
        error = true;
        log.error('Something went wrong', e)
        outputStream.write(content.getBytes(StandardCharsets.UTF_8));
    }
}));

if (error) {
    session.transfer(flowFile, REL_FAILURE)
} else {
    session.transfer(flowFile, REL_SUCCESS)
}

}

因此在上面的脚本中，JSON.parse（content）行读取了在那里专门声明的标签。在字段和标签未知的地方可以使用相同类型的脚本吗？

编辑：

来源Json

{"new_records":"1022",
 "updated_records":"546565",
 "changed_records":"546566",
 "deleted_records":"122345",
 "missing_records":"78"}

预期的线路协议大量涌入

"test_measurement",remote_source_name="jmx_k_server",meta_name="fetchRecord",message_type="text"  new_records="1022",updated_records="546565",changed_records="546566","deleted_records"="122345","missing_records"="78" 1265522132

json格式

data = [
{
  “measurement”: MEASUREMENT,
  “tags”: {
       TAGS
  },
  “fields”: {
     FIELDS
   }
 }
 ]

Answer 1

在研究类似问题时偶然发现：

只需循环json中的键...（假设您使用的是javascript / ECMAScript）。

关键如下：

message_content = JSON.parse(content);
for (key in message_content){
    line += key + "=" + message_content[key] + ","
}

我用过的完整片段：

flowFile = session.get();
if (flowFile != null) {

var StreamCallback = 
   Java.type("org.apache.nifi.processor.io.StreamCallback");
var IOUtils = Java.type("org.apache.commons.io.IOUtils");
var StandardCharsets = Java.type("java.nio.charset.StandardCharsets");
var transformed_message = {};
var error = false;
var line = "test_measurement";
var sep = "\n"

// Get attributes
flowFile = session.write(flowFile, new StreamCallback(function (inputStream, 
 outputStream) {
    // Read input FlowFile content
    var type = flowFile.getAttribute('type')
    var time = Number(flowFile.getAttribute('filename'))
    var meta_name = flowFile.getAttribute('meta_name')
    var remote_source_name = flowFile.getAttribute('gethttp.remote.source')

    var content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); // message or content
    var message_content = {};
    try {
        message_content = JSON.parse(content);
        for (key in message_content){
            if (key == 't') {
                line +=  " " + message_content[key]
            }
            else {          
                line += "," + key + "=" + message_content[key]
            }
        }
        line += sep

        // Write output content
        if (transformed_message) {
            outputStream.write(line.getBytes(StandardCharsets.UTF_8));
        }
    } catch (e) {
        error = true;
        log.error('Something went wrong', e)
        outputStream.write(content.getBytes(StandardCharsets.UTF_8));
    }
}));

if (error) {
    session.transfer(flowFile, REL_FAILURE)
} else {
    session.transfer(flowFile, REL_SUCCESS)
}
}

nifi executeScript用于向influxDB的线路协议声明

1 个答案: