Question

我有以下Docker compose配置并使用docker-compose up运行它：

version: "2"

services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
    environment:
      - cluster.name=docker-cluster
      - bootstrap.memory_lock=true
      - discovery.type=single-node
      - "ES_JAVA_OPTS=-Xmx2G -Xms2G"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - esdata:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
    networks:
      - elk

  logstash:
    image: docker.elastic.co/logstash/logstash:${ELK_VERSION}
    volumes:
      - ./logstash/config/pipelines.yml:/usr/share/logstash/config/pipelines.yml:ro
      - ./logstash/pipeline:/usr/share/logstash/pipeline
      - ./logstash/drivers:/usr/share/logstash/drivers
      - ./logstash/shared:/usr/share/logstash/shared
    environment:
      - xpack.monitoring.collection.enabled=true
      - "LS_JAVA_OPTS=-Xmx2G -Xms2G"
    networks:
      - elk
    depends_on:
      - elasticsearch

  kibana:
    image: docker.elastic.co/kibana/kibana:${ELK_VERSION}
    environment:
      - server.name=kibana
      - elasticsearch.url=http://elasticsearch:9200
    ports:
      - 5601:5601
    networks:
      - elk
    depends_on:
      - elasticsearch

volumes:
  esdata:
    driver: local

networks:
  elk:

有很多资源分配给它。我运行了两个简单的持久化管道，这些管道的总数据量不足100MB。

但是，只有三分之一的数据被索引。在基巴纳州的监视说：

收到的事件：390.8k
发出的事件：120.1万

Logstash日志也表明没有问题

[2018-12-20T10:12:56,420][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://elasticsearch:9200/]}}
[2018-12-20T10:12:56,443][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>"http://elasticsearch:9200/"}
[2018-12-20T10:12:56,453][INFO ][logstash.outputs.elasticsearch] ES Output version determined {:es_version=>6}
[2018-12-20T10:12:56,453][WARN ][logstash.outputs.elasticsearch] Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type {:es_version=>6}
[2018-12-20T10:12:56,513][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["http://elasticsearch:9200"]}
[2018-12-20T10:12:56,746][INFO ][logstash.pipeline        ] Pipeline started successfully {:pipeline_id=>".monitoring-logstash", :thread=>"#<Thread:0x1d13c506 sleep>"}
[2018-12-20T10:12:56,782][INFO ][logstash.agent           ] Pipelines running {:count=>3, :running_pipelines=>[:"via-brands-categories", :"via-categories", :".monitoring-logstash"], :non_running_pipelines=>[]}
[2018-12-20T10:12:57,731][INFO ][logstash.agent           ] Successfully started Logstash API endpoint {:port=>9600}
[2018-12-20T10:13:00,148][INFO ][logstash.inputs.jdbc     ] (1.116852s)
            SELECT categoryid
              , categoryname
              , industryCode
              , skucount
              , TO_JSON_STRING(permissionIds) as permissionIds
              , TO_JSON_STRING(filters) as filters
              , lastupdate
              , deleted
            FROM migration_search.categories;
[2018-12-20T10:13:00,162][INFO ][logstash.inputs.jdbc     ] (1.114124s)
            SELECT brandid
              , brandname
              , categoryid
              , categoryname
              , industryCode
              , skucount
              , TO_JSON_STRING(permissionIds) as permissionIds
              , TO_JSON_STRING(filters) as filters
              , lastupdate
              , deleted
            FROM migration_search.brands_categories;
[2018-12-20T10:13:50,058][INFO ][logstash.pipeline        ] Pipeline has terminated {:pipeline_id=>"via-categories", :thread=>"#<Thread:0x47925db9@/usr/share/logstash/logstash-core/lib/logstash/pipeline_action/create.rb:51 run>"}
[2018-12-20T10:15:16,179][INFO ][logstash.pipeline        ] Pipeline has terminated {:pipeline_id=>"via-brands-categories", :thread=>"#<Thread:0x7e52add8 run>"}
[2018-12-20T10:15:18,108][INFO ][logstash.pipeline        ] Pipeline has terminated {:pipeline_id=>".monitoring-logstash", :thread=>"#<Thread:0x1d13c506 run>"}

Logstash不会发出所有事件的可能是什么问题？

Answer 1

事实证明，这就是持久队列的工作方式。即使没有从队列中提取所有内容，Logstash也将关闭。

将queue.drain: true添加到我在pipelines.yml中的每个管道中均已解决。 Logstash现在将等待，直到所有数据都被吸收。

Logstash在Docker上提供不到一半的数据

1 个答案: