Question

我正在尝试摄取json fileformat后生成的库存数据。

{  
   "_meta":{  
      "hostvars":{  
         "host1":{  
            "foreman":{  
               "architecture_id":1,
               "architecture_name":"x86_64",
               "capabilities":[  
                  "build"
               ],
               "certname":"host1",
               "comment":"this is hostname1",
               "created_at":"2017-03-08T15:27:11Z",
               "disk":"10gb",
               "domain_id":5,

            },
            "foreman_facts":{  
               "boardmanufacturer":"Intel Corporation",
               "boardproductname":"440BX Desktop Reference Platform",
               "ipaddress":"1.1.1.1",
               "ipaddress_eth0":"1.1.1.2",
               "ipaddress_lo":"127.0.0.1",

            },
            "foreman_params":{  

            }
         },
         "host2":{  
            "foreman":{  
               "architecture_id":1,
               "architecture_name":"x86_64",
               "capabilities":[  
                  "build"
               ],
               "certname":"host2",
               "comment":"this hostname2",
               "created_at":"2017-03-08T15:27:11Z",
               "disk":"20gb",
               "domain_id":5,

            },
            "foreman_facts":{  
               "boardmanufacturer":"Intel Corporation",
               "boardproductname":"440BX Desktop Reference Platform",
               "ipaddress":"2.1.1.1",
               "ipaddress_eth0":"2.2.2.2",
               "ipaddress_lo":"127.0.0.1",

            },
            "foreman_params":{  

            }
         },
         "foreman_all":[  
            "host3",
            "host4",

         ],
         "foreman_environment: [ 
         "computer1",
         "computer2"
      ],

使用以下代码管理以在ElasticSeach中获取数据。

文件节拍配置：

multiline.pattern: '^{'

multiline.negate: true

multiline.match: after

output.logstash:
  # The Logstash hosts
 hosts: ["localhost:5044"]

Logstash：

 input {
 beats {
        port => "5044"
       }

}

output {

elasticsearch {
        hosts => [ "10.1.7.5:9200" ]
index => "inventory-%{+YYYY-MM-dd}"
}
stdout {}

}

但是我注意到filebeat将整个json文件视为一条消息。想知道如果我可以打破消息并仅发送 hostvars 部分并根据每个主机名索引文档，并忽略 foreman_all 和 foreman_environment 来自上述json数据的字段。以上是样本数据，我必须摄取大约10万条记录，所以要确保我在网络上尽可能少地发送数据。

我想在Elasticsearch中以下列格式摄取数据。想知道是否有人可以建议使用最好的配置。

弹性文档ID 1

computer name : "host1"
"architecture_id": 1,
"architecture_name": "x86_64",
"capabilities": ["build"],
"Company hardware name": "host1",
"comment": "this is hostname1",
"created_at": "2017-03-08T15:27:11Z",
"disk": "10gb",
"domain_id": 5,
"foreman_facts": {
"boardmanufacturer": "Intel Corporation",
"boardproductname": "440BX Desktop Reference Platform",
"ipaddress": "1.1.1.1",
"ipaddress_eth0": "1.1.1.2",
"ipaddress_lo": "127.0.0.1",

弹性文档ID 2

"computer name"" : "host2"
"architecture_id": 1,
"architecture_name": "x86_64",
"capabilities": ["build"],
"certname": "host2",
"comment": "this hostname2",
"created_at": "2017-03-08T15:27:11Z",
"disk": "20gb",
"domain_id": 5,
"boardmanufacturer": "Intel Corporation",
"boardproductname": "440BX Desktop Reference Platform",
"ipaddress": "2.1.1.1",
"ipaddress_eth0": "2.2.2.2",
"ipaddress_lo": "127.0.0.1",

Answer 1

首先你应该在filebeat.yml中设置document_type，如下所示：

filebeat:
    prospectors:
        - input_type: log
    paths:
        - "/home/ubuntu/data/test.json"
    document_type: json
    json.message_key: log
    json.keys_under_root: true
    json.overwrite_keys: true

看看这可能有所帮助：https://www.elastic.co/blog/structured-logging-filebeat

然后你可以在logstash中获取json值并将它们设置为新字段（在logstash.conf中配置）：
```
json  {
      source => "parameter"
      target => "parameterData"
      remove_field => "parameter"
}
```

文献：https://www.elastic.co/guide/en/logstash/current/plugins-filters-json.html

您可以在logstash中使用std_in和std_out进行测试。

Answer 2

我根据您的建议使用了下面的配置，并看到下面提到的enter image description here的Json错误消息。看起来像filbeat单独发送每一行，如果我使用如下所述的多行选项，那么我看到Filebeat和logstash将整个json文件作为一条消息发送。这就是我想要根据上面提到的主机名来打破消息。

filebeat.prospectors:


- type: log

  # Change to true to enable this prospector configuration.
  enabled: true

  # Paths that should be crawled and fetched. Glob based paths.
  paths:
    - /opt/uploaddata/*.json
    #- c:\programdata\elasticsearch\logs\*


  ### JSON configuration

  document_type: json

  json.message_key: log


  json.keys_under_root: true

  json.overwrite_keys: true

  #json.add_error_key: false


output.logstash:
  # The Logstash hosts
  hosts: ["localhost:5044"]


#=========================== Logstash  =============================
input {
 beats {
        port => "5044"
       }
}

filter {
json

{
      source => "parameter"
      target => "parameterData"
      remove_field => "parameter"
}

}
output {

elasticsearch {
        hosts => [ "10.138.7.51:9200" ]
index => "inventory-%{+YYYY-MM-dd}"
}
stdout {
codec => rubydebug
}
}

#=========================== Filbear Errors =============================

2017/11/24 16:45:14.226665 json.go:32: ERR Error decoding JSON: json: cannot unmarshal string into Go value of type map[string]interface {}
2017/11/24 16:45:14.226757 processor.go:262: DBG Publish event: {
  "@timestamp": "2017-11-24T16:45:14.226Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "doc",
    "version": "6.0.0"
  },
  "json": {},
  "message": "            \"host4\",",
  "prospector": {
    "type": "log"
  },
  "beat": {
    "name": "filebeat",
    "hostname": "filebeat",
    "version": "6.0.0"
  },
  "source": "/opt/uploaddata/data.json",
  "offset": 1710
}
2017/11/24 16:45:14.226800 json.go:32: ERR Error decoding JSON: EOF
2017/11/24 16:45:14.226889 processor.go:262: DBG Publish event: {
  "@timestamp": "2017-11-24T16:45:14.226Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "doc",
    "version": "6.0.0"
  },
  "json": {},
  "message": "",
  "source": "/opt/uploaddata/data.json",
  "offset": 1712,
  "prospector": {
    "type": "log"
  },
  "beat": {
    "name": "filebeat",
    "hostname": "filebeat",
    "version": "6.0.0"
  }


#=========================== Logstash Logs  =============================

{
    "@timestamp" => 2017-11-24T16:45:14.226Z,
        "offset" => 1638,
      "@version" => "1",
          "beat" => {
            "name" => "filebeat",
        "hostname" => "filebeat",
         "version" => "6.0.0"
    },
          "host" => "filebeat",
    "prospector" => {
        "type" => "log"
    },
          "json" => {},
        "source" => "/opt/uploaddata/data.json",
       "message" => "         },",
          "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]
}
{
    "@timestamp" => 2017-11-24T16:45:14.226Z,
        "offset" => 1666,
      "@version" => "1",
          "beat" => {
            "name" => "filebeat",
        "hostname" => "filebeat",
         "version" => "6.0.0"
    },
          "host" => "filebeat",
          "json" => {},
    "prospector" => {
        "type" => "log"
    },
        "source" => "/opt/uploaddata/data.json",
       "message" => "         \"foreman_all\":[  ",
          "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]
}

＃=========================== Filebeat配置================= ============

library(DiagrammeR)
library(data.table)
niv <- c("A","B","C","D","E","X","Y")
from <- c("A","A","A","A","B","C","D","E", "X", "B")
to <- c("B","C","D","E","X","X","Y","Y","Y", "C")
temp <- data.table(from=factor(from, levels=niv),
to=factor(to,levels=niv), col=c(rep("blue",5), rep("black",5)))

nodes <- create_node_df(n=length(niv), label=niv,  width=0.3)
# Add a vector of colors for fontcolor 
edges <- create_edge_df(from=temp$from, to=temp$to, 
         rel="leading_to", label=temp$from, color=temp$col,
         fontcolor=temp$col) 

graph <- create_graph(nodes_df = nodes, edges_df = edges) 

# Delete the default "layout" graph attribute and
# set direction of graph layout
graphAttr <- get_global_graph_attrs(graph)
graphAttr <- rbind(graphAttr[-1,], 
                   c("rankdir", "LR", "graph"))

graph <- set_global_graph_attrs(graph,
      attr = graphAttr$attr,
      value = graphAttr$value,
      attr_type = graphAttr$attr_type)

render_graph(graph)

Json文件从filebeat到Logstash然后再到elasticsearch

2 个答案: