ELK设置与兔子在弹性搜索中获取重复记录

Question

我使用ELK和RabbitMQ进行集中式日志记录。所有微服务都有一个AMQP Appender（JAVA，NodeJS，PHP），它们将日志推送到RabbitMQ。从RabbitMQ，Logstash将其消耗掉，然后推入弹性搜索。我正在单节点上运行弹性搜索。

在RabbitMQ中，正如我从RabbitMQ管理用户界面验证的那样，日志被推送一次。但是，当我通过Kibana查看日志时，可以看到重复的日志。

我还放置了Logstash指纹插件，但在这样做之后，它也在elasticsearch中创建了重复的日志。对于第一个记录，document_id被指纹代替，但是对于重复记录，document_id是一些随机数。

请找到logstash conf：

    input {
     rabbitmq {
     user => "guest"
     password => "guest"
     queue => "axon_log_node"
     host => "localhost"
     port => 5672
     durable => true
     }
    }

filter{

    fingerprint {
    source => "[meta]" 
    target => "fingerprint"
    method => "SHA1"
    key => "Centralized Logging"
    base64encode => true
  }

    mutate {
        convert => {"level" => "string"}
        rename => {"[text]" => "message" }
        rename => {"[meta][module]" => "module" }
        rename => {"[meta][correlation_id]" => "correlation_id" }
        rename => {"[meta][appId]" => "component" }
        rename => {"[meta][userId]" => "userId" }
        rename => {"[meta][source]" => "source" }
        rename => {"[meta][action]" => "action" }
        rename => {"[meta][level_name]" => "level_name" }
    }
    mutate {
        convert => {"timestamp" => "string"}
    }
}


output {
      elasticsearch {
        hosts => ["localhost:9200"]
        sniffing => true
        manage_template => false
        document_type => "log"
        document_id => "%{fingerprint}"
        template_overwrite => "true"
      }
    }

Logs in kibana :




 {
     "_index": "logstash-2018.09.03",
     "_type": "log",
     "_id": "tLVzn2UBo4M6JkGUkRaW",
     "_version": 1,
     "_score": null,
     "_source": {
     "@version": "1",
     "action": "Prevalidate",
     "message": "actual perform",
     "component": "Notification",
     "fingerprint": "KqWJGt4m3KabAp8aKO+xWroltQE=",
     "module": "BaseInteractor",
     "correlation_id": "6c9987ed-d714-441f-a7ba-63d52fbcb35b",
     "meta": {},
     "source": "ms-axon-install.local",
     "@timestamp": "2018-09-03T12:41:14.768Z",
     "level_name": "info"
     },
     "fields": {
      "@timestamp": [
       "2018-09-03T12:41:14.768Z"
      ]
    },
    "highlight": {
     "component.keyword": [
      "@kibana-highlighted-field@Notification@/kibana-highlighted-field@"
     ]
   },
   "sort": [
    1535978474768
   ]
 }



 {
  "_index": "logstash-2018.09.03",
  "_type": "log",
  "_id": "KqWJGt4m3KabAp8aKO+xWroltQE=",
  "_version": 4,
  "_score": null,
  "_source": {
    "@version": "1",
    "action": "Prevalidate",
    "message": "actual perform",
    "component": "Notification",
    "fingerprint": "KqWJGt4m3KabAp8aKO+xWroltQE=",
    "module": "BaseInteractor",
    "correlation_id": "6c9987ed-d714-441f-a7ba-63d52fbcb35b",
    "meta": {},
    "source": "ms-axon-install.local",
    "@timestamp": "2018-09-03T12:41:14.768Z",
    "level_name": "info"
  },
  "fields": {
    "@timestamp": [
      "2018-09-03T12:41:14.768Z"
    ]
  },
  "highlight": {
    "component.keyword": [
      "@kibana-highlighted-field@Notification@/kibana-highlighted-field@"
    ]
  },
  "sort": [
    1535978474768
  ]
}

{
  "_index": "logstash-2018.09.03",
  "_type": "log",
  "_id": "zbVzn2UBo4M6JkGUkhbM",
  "_version": 1,
  "_score": null,
  "_source": {
    "@version": "1",
    "action": "Prevalidate",
    "message": "actual perform",
    "component": "Notification",
    "fingerprint": "KqWJGt4m3KabAp8aKO+xWroltQE=",
    "module": "BaseInteractor",
    "correlation_id": "6c9987ed-d714-441f-a7ba-63d52fbcb35b",
    "meta": {},
    "source": "ms-axon-install.local",
    "@timestamp": "2018-09-03T12:41:14.768Z",
    "level_name": "info"
  },
  "fields": {
    "@timestamp": [
      "2018-09-03T12:41:14.768Z"
    ]
  },
  "highlight": {
    "component.keyword": [
      "@kibana-highlighted-field@Notification@/kibana-highlighted-field@"
    ]
  },
  "sort": [
    1535978474768
  ]
}