Question

我无法通过组合纬度和经度在Logstash中撰写geo_point。我遵循了其他人的指示，但看起来这些示例都基于旧版本的ELK。由于ELK 2.2对geo_point有重大改变，我不确定我是否以正确的方式执行了所有步骤。在这里，我解释了我的设置。

我使用的ELK版本是：

curl -XGET 'localhost:9200' 
{
"name" : "Artie",
"cluster_name" : "elasticsearch",
"version" : {
"number" : "2.2.1",
"build_hash" : "d045fc29d1932bce18b2e65ab8b297fbf6cd41a1",
"build_timestamp" : "2016-03-09T09:38:54Z",
"build_snapshot" : false,
"lucene_version" : "5.4.1"
},
"tagline" : "You Know, for Search"
}

在Docker容器上使用Elasticsearch，Logstash和Kibana，但这不重要。

这就是我的logstash.conf的样子：

cat logstash.conf
input {
    http_poller {
            urls => {
                   myresource => "myhost/data.json"
            }
            request_timeout => 1
            interval => 1
            # Parse every line captured from data.json as a new event. 
            codec => "line" 
    }
}
filter {
    if [message] !~ /\"hex\":/ { 
        # drop messages without "hex"
        drop {} 
    }
    # Capture "hex":72d5a1
    grok {
            match => { "message" => "\"hex\":\"(?<hex>[^\"]+)\"," }
    }
    mutate {
            convert => { "hex"       => "string"  }
    }
    # Capture "lat":50.047613
    if [message] =~ /\"lat\":/ {
        grok {
                match => { "message" => "\"lat\":(?<latitude>[^,]+),"}
        }
        mutate {
                convert => { "latitude"  => "float"   }
        }
    }
    # Capture "lon":1.702955    
    if [message] =~ /\"lon\":/ {
        grok {
                match => { "message" => "\"lon\":(?<longitude>[^,]+)," }
        }
        mutate {
            convert => { "longitude" => "float"   }
        }
    }
    # convert latitude and longitude into location.
    mutate {
        rename => {
                "longitude" => "[location][lon]"
                "latitude" => "[location][lat]"
        }
    }
    mutate {
        remove_field => [ "message" ]
    }
}
output { 
    elasticsearch { 
        hosts => [ "elasticsearchhost:9200" ]
        index => "logstash-%{+YYYY.MM.dd}" 
    }  
}

重要的是“lon”＆amp; “lat”是从“消息”中捕获的，并且它们被格式化为“位置”字段。

当我查询elasticsearch时，我得到了这种记录：

{
  "_index": "logstash-2016.04.04",
  "_type": "logs",
  "_id": "AVPieJtgVkabtr-H2szZ",
  "_score": null,
  "_source": {
    "@version": "1",
    "@timestamp": "2016-04-04T18:11:07.857Z",
    "hex": "3e37aa",
     "location": {
      "lon": 4.8246,
      "lat": 52.329208
   }
  },
  "fields": {
    "@timestamp": [
      1459793467857
    ]
  },
  "sort": [
    1459793467857
  ]
}

根据我在文档中阅读的内容，符号"location": { "lon": 4.8246, "lat": 52.329208 }看起来不错。但问题是我无法在Kibana中选择字段“location”作为geo_point。

根据ELK文档，我需要确保“location”字段映射到geo_point类型。它要求启用doc_values才能运行。我不确定我是否应该做些什么，因为当我查看我的模板时，默认情况下已经映射了“location”字段："location" : { "type" : "geo_point", "doc_values" : true }

这就是我的模板的样子：

# curl -XGET localhost:9200/_template/logstash?pretty
{
 "logstash" : {
  "order" : 0,
  "template" : "logstash-*",
  "settings" : {
      "index" : {
        "refresh_interval" : "5s"
      }
    },
    "mappings" : {
      "_default_" : {
        "dynamic_templates" : [ {
          "message_field" : {
            "mapping" : {
              "fielddata" : {
                "format" : "disabled"
              },
              "index" : "analyzed",
              "omit_norms" : true,
              "type" : "string"
            },
            "match_mapping_type" : "string",
            "match" : "message"
          }
        }, {
          "string_fields" : {
            "mapping" : {
              "fielddata" : {
                "format" : "disabled"
              },
              "index" : "analyzed",
              "omit_norms" : true,
              "type" : "string",
              "fields" : {
                "raw" : {
                  "ignore_above" : 256,
                  "index" : "not_analyzed",
                  "type" : "string",
                  "doc_values" : true
                }
              }
            },
            "match_mapping_type" : "string",
            "match" : "*"
          }
        }, {
          "float_fields" : {
            "mapping" : {
              "type" : "float",
              "doc_values" : true
            },
            "match_mapping_type" : "float",
            "match" : "*"
          }
        }, {
          "double_fields" : {
            "mapping" : {
              "type" : "double",
              "doc_values" : true
            },
            "match_mapping_type" : "double",
            "match" : "*"
          }
        }, {
          "byte_fields" : {
            "mapping" : {
              "type" : "byte",
              "doc_values" : true
            },
            "match_mapping_type" : "byte",
            "match" : "*"
          }
        }, {
          "short_fields" : {
            "mapping" : {
              "type" : "short",
              "doc_values" : true
            },
            "match_mapping_type" : "short",
            "match" : "*"
          }
        }, {
          "integer_fields" : {
            "mapping" : {
              "type" : "integer",
              "doc_values" : true
            },
            "match_mapping_type" : "integer",
            "match" : "*"
          }
        }, {
          "long_fields" : {
            "mapping" : {
              "type" : "long",
              "doc_values" : true
            },
            "match_mapping_type" : "long",
            "match" : "*"
          }
        }, {
          "date_fields" : {
            "mapping" : {
              "type" : "date",
              "doc_values" : true
            },
            "match_mapping_type" : "date",
            "match" : "*"
          }
        }, {
          "geo_point_fields" : {
            "mapping" : {
              "type" : "geo_point",
              "doc_values" : true
            },
            "match_mapping_type" : "geo_point",
            "match" : "*"
          }
        } ],
        "_all" : {
          "omit_norms" : true,
          "enabled" : true
        },
        "properties" : {
          "@timestamp" : {
            "type" : "date",
            "doc_values" : true
          },
          "geoip" : {
            "dynamic" : true,
            "type" : "object",
            "properties" : {
              "ip" : {
                "type" : "ip",
                "doc_values" : true
              },
              "latitude" : {
                "type" : "float",
                "doc_values" : true
              },
              "location" : {
                "type" : "geo_point",
                "doc_values" : true
              },
              "longitude" : {
                "type" : "float",
                "doc_values" : true
              }
            }
          },
         "@version" : {
            "index" : "not_analyzed",
            "type" : "string",
            "doc_values" : true
          }
        }
      }
    },
    "aliases" : { }
  }
}

我没有在此模板中添加内容。这是在全新安装Logstash和Elastic以及使用我的logstash.conf文件启动Logstash之后的样子。

我的问题是：为了解决我的问题，我需要采取哪些步骤？

非常感谢！

Answer 1

模板中的“位置”字段实际上是“[geoip] [location]”，但您的数据位于“[hex] [location]”中。因此，模板的魔力不适用于您的领域。将数据移动到[geoip] [location]或更改模板。

此外，在一个正则表达式中解析您的消息会更有效，而不是先在条件语句中运行正则表达式，然后再以grok模式再次运行它。

如何从经度和纬度中组合Logstash geo_point

1 个答案: