Question

所以我最近创建了一个使用此网站作为模板的ELK群集。ELK Cluster Setup

我遇到了一个问题，即Logstash处理节点上的json模板没有在实际的Elasticsearch数据节点上使用。我可以看到映射已在HQ中创建，但另一个已创建使用一些动态创建的映射。正确完成的映射在数据节点上称为“Sourcefire”，但它也创建了一个名为“sourcfire”的不正确的映射。

我无法弄清楚这一点，我正在学习这些东西，所以任何帮助都表示赞赏。请参阅下面的代码段。

Logstash.conf

input {
    tcp {
        port => 5170
        type => "sourcefire"
    }
}

filter {

    mutate{
        split => ["message", "|"]
        add_field => {
            "event" => "%{message[5]}"
            "eventSource" => "%{message[1]}"
        }
    }

    kv {
        include_keys => ["dhost", "dst", "dpt", "shost", "src", "spt", "rt"]
    }

    mutate {
        rename => [ "dhost", "destinationHost" ]
        rename => [ "dst", "destinationAddress" ]
        rename => [ "dpt", "destinationPort" ]
        rename => [ "shost", "sourceHost" ]
        rename => [ "src", "sourceAddress" ]
        rename => [ "spt", "sourcePort" ]
    }

    date {
        match => ["rt","UNIX_MS"]
        target => "eventDate"
    }

    geoip {
        add_tag => [ "sourceGeo" ]
        source => "src"
        database => "/opt/logstash/vendor/geoip/GeoLiteCity.dat"
    }

    geoip {
        add_tag => [ "destinationGeo" ]
        source => "src"
        database => "/opt/logstash/vendor/geoip/GeoLiteCity.dat"
    }
}

output {
    if [type] == "sourcefire" {
        elasticsearch {
            cluster => "XXX-cluster"
            flush_size => 1
            manage_template => true
            template => "/opt/logstash/lib/logstash/outputs/elasticsearch/elasticsearch-sourcefire.json"
        }
    }
}

Elasticsearch json模板

{
    "template": "logstash-*",
    "settings": {
        "index.refresh_interval": "5s"
    },
    "mappings": {
        "Sourcefire": {
            "_all": {
                "enabled": true
            },
            "properties": {
                "@timestamp": {
                    "type": "date",
                    "format": "basicDateTimeNoMillis"
                },
                "@version": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "geoip": {
                    "type": "object",
                    "dynamic": true,
                    "path": "full",
                    "properties": {
                        "location": {
                            "type": "geo_point"
                        }
                    }
                },
                "event": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "eventDate": {
                    "type": "date",
                    "format": "basicDateTimeNoMillis"
                },
                "destinationAddress": {
                    "type": "ip"
                },
                "destinationHost": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "destinationPort": {
                    "type": "integer",
                    "index": "not_analyzed"
                },
                "sourceAddress": {
                    "type": "ip"
                },
                "sourceHost": {
                    "type": "string",
                    "index": "not_analyzed"
                },
                "sourcePort": {
                    "type": "integer",
                    "index": "not_analyzed"
                }
            }
        }
    }
}

Answer 1

您可以使用elasticsearch输出的template_overwrite属性。但是，不能保证始终正常工作，特别是如果您有多个同时工作的logstash实例。此外，根据您的elasticsearch映射配置设置，特别是动态映射和默认设置（http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/mapping-dynamic-mapping.html），您可能会得到与预期不同的结果。

根据我的经验，我发现最好手动控制elasticsearch中的索引映射（使用fiddler或elasticsearch-head管理站点等工具）。这是因为当多个logstash实例一起覆盖映射时，我遇到了各种意外结果，禁用了我设置的特殊弹性搜索字段（如_ttl）。

Answer 2

只需删除存储的模板，它应该重新创建它：例如，如果您的模板名称为logstash：

curl -XDELETE localhost:9200/_template/logstash

此外，如果您要写入相同的索引，则无法更改映射。您需要重新创建索引（确保首先停止logstash以防止任何飞行）。

Logstash - ＆gt; Elasticsearch没有正确映射

2 个答案: