我们遇到了一个问题，其中一个Elasticsearch数据节点完全在AWS中死亡。即使使用第二个节点，我们也无法将群集重新联机。它最初出现在yellow状态，我们可以检索数据但不能写入数据。
Elasticsearch错误
[2016-09-12 13：16：42,312] [警告] [discovery.zen.ping.unicast] [witness.domain.net]未能将ping发送到[{＃cloud-i-cb397257-0} {1.1.1.1} {1.1.1.1:9300}] RemoteTransportException [[datanode.domain.net] [1.1.1.1:9300] [内部：发现/禅/单播]];嵌套：IllegalStateException [未启动时收到ping请求]; 引起：java.lang.IllegalStateException：未启动时收到ping请求
Logstash错误
{：timestamp =＆gt;＆＃34; 2016-09-12T15：48：15.898000-0400＆＃34;，：message =＆gt;＃900592725，：events_consumed =＆gt; 900732955，：worker_count =＆gt; 4， ：inflight_count =＆gt; 0，：worker_states =＆gt; [{：status =＆gt;＆＃34; sleep＆＃34;，：alive =＆gt; true，：index =＆gt; 0，：inflight_count =＆gt; 0}， {：status =＆gt;＆＃34; sleep＆＃34;，：alive =＆gt; true，：index =＆gt; 1，：inflight_count =＆gt; 0}，{：status =＆gt;＆＃34; sleep＆＃34 ;，：alive =＆gt; true，：index =＆gt; 2，：inflight_count =＆gt; 0}，{：status =＆gt;＆＃34; sleep＆＃34;，：alive =＆gt; true，：index =＆gt; ; 3，：inflight_count =＆gt; 0}]，：output_info =＆gt; [{：type =＆gt;＆＃34; gelf＆＃34;，：config =＆gt; {＆＃34; host＆＃34; =＆gt; ＆＃34; 127.0.0.1＆＃34;，＆＃34; ALLOW_ENV ＆＃34; =＆gt; false}，：is_multi_worker =＆gt; false，：events_received =＆gt; 900592725，：workers =＆gt;＆＃34; 127.0.0.1＆＃34;，codec =＆gt;＆＃34; UTF-8＆＃34;＆gt;，workers =＆gt; 1，port =＆gt; 12201，chunksize =＆gt; 1420， sender =＆gt;＆＃34;％{host}＆＃34;，level =＆gt; [＆＃34;％{severity}＆＃34;，＆＃34; INFO＆＃34;]，ship_metadata =＆gt; true ，ship_tags =＆gt; true，ignore_metadata =＆gt; [＆＃34; @t imestamp＆＃34;，＆＃34; @ version＆＃34;，＆＃34; severity＆＃34;，＆＃34; host＆＃34;，＆＃34; source_host＆＃34;，＆＃34; source_path＆＃34; ，＆＃34; short_message＆＃34;]，full_message =＆gt;＆＃34;％{message}＆＃34;，short_message =＆gt;＆＃34; short_message＆＃34;＆gt;]＆gt;，：busy_workers =＆gt ; 0}]，：thread_info =＆gt; [{＆＃34; thread_id＆＃34; =＆gt; 17，＆＃34; name＆＃34; =＆gt;＆＃34; [main] nil，＆＃34; backtrace＆ ＃34; = GT; [＆＃34; [...] /供应商/束/的jruby / 1.9 /宝石/ JRuby的OpenSSL的-0.9.13-java的/ LIB / jopenssl19 / OpenSSL的/ SSL-internal.rb：106 ：在accept'", "[...]/vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13-java/lib/jopenssl19/openssl/ssl-internal.rb:106:in接受＆＃39;＆＃34;，＆＃34; [...] / vendor / bundle / jruby / 1.9 / gems / logstash-input-tcp-3.0.6 / lib / logstash / inputs / tcp.rb：112：在run_server'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-input-tcp-3.0.6/lib/logstash/inputs/tcp.rb:84:in运行＆＃39;＆＃34;，＆＃34; [...] / vendor / bundle / jruby / 1.9 / gems / logstash-core-2.3.3 -java / lib / logstash / pipeline.rb：342：在inputworker'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:336:in start_input＆＃39;＆＃34;]，＆＃34; blocked_on＆＃34; =＆gt; nil，＆＃34; status＆＃34; =＆gt;＆＃34;睡眠＆＃34;，＆＃34; current_call＆＃34; =＆gt;＆＃34; [...] / vendor / bundle / jruby / 1.9 / gems / jruby-openssl-0.9.13 -java / lib / jopenssl19 / openssl / ssl-internal.rb：106：在accept'"}, {"thread_id"=>18, "name"=>"[main]>worker0", "plugin"=>["LogStash::Filters::Grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{SYSLOGBASE2}"}}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in同步＆＃39;＆＃34;，＆＃34; [...] / vendor / bun dle / jruby / 1.9 / gems / logstash-core-2.3.3-java / lib / logstash / pipeline.rb：309：在inflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:234:in worker_loop＆＃39;＆＃34;，＆＃34; [.. 。] / vendor / bundle / jruby / 1.9 / gems / logstash-core-2.3.3-java / lib / logstash / pipeline.rb：201：in start_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize＆＃39;＆＃34;}，{ ＆＃34; thread_id＆＃34; =＆gt; 19，＆＃34; name＆＃34; =＆gt;＆＃34; [main]＆gt; worker1＆＃34;，＆＃34; plugin＆＃34; =＆gt; [ ＆＃34; LogStash :: Filters :: Grok＆＃34;，{＆＃34; patterns_dir＆＃34; =＆gt;＆＃34; /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns- core-2.0.5 / patterns /＆＃34;，＆＃34;匹配＆＃34; =＆gt; {＆＃34;消息＆＃34; =＆gt;＆＃34;％{SYSLOGBASE2}＆＃34;}} ]，＆＃34; backtrace＆＃34; =＆gt; [＆＃34; [...] / vendor / bundle / jruby / 1.9 / gems / logstash-core-2.3.3-java / lib / logstash / pipeline。 rb：309：在synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in inflight_batches_synchronize＆＃39;＆＃34;，＆＃34; [...] / vendor / bundle / jruby / 1.9 / gems / logstash-core-2.3.3-java / lib /logstash/pipeline.rb:234:in worker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:in start_workers＆＃39;＆＃34;]，＆＃34; blocked_on＆＃34; =＆gt; nil，＆＃34; status＆＃34; =＆gt;＆ ＃34; sleep＆＃34;，＆＃34; current_call＆＃34; =＆gt;＆＃34; [...] / vendor / bundle / jruby / 1.9 / gems / logstash-core -2.3.3-java / lib / logstash / pipeline.rb：309：在synchronize'"}, {"thread_id"=>20, "name"=>"[main]>worker2", "plugin"=>["LogStash::Filters::Grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{SYSLOGBASE2}"}}], "backtrace"=>["[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in同步＆＃39;＆＃34;，＆＃34; [...] / vendor / bundle / jruby / 1.9 / gems / logstash-core-2.3.3-java / lib / logstash / pipeline.rb：309：在inflight_batches_synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:234:in worker_loop＆＃39;＆＃34;，＆＃34; [...] / vendor / bundle /jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:in start_workers'"], "blocked_on"=>nil, "status"=>"sleep", "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in同步＆＃39;＆＃34;}，{＆＃34; thread_id＆＃ 34; =＆gt; 21，＆＃34;名称＆＃34; =＆gt;＆＃34; [主要]＆gt; worker3＆＃34;，＆＃34;插件＆＃34; =＆gt; [＆＃34; LogStash： ：过滤器:: Grok＆＃34;，{＆＃34; patterns_dir＆＃34; =＆gt;＆＃34; /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/模式/＆＃34;，＆＃34;匹配＆＃34; =＆gt; {＆＃34;消息＆＃34; =＆gt;＆＃34;％{SYSLOGBASE2}＆＃34;}}]，＆＃34; backtrace＆＃34; =＆gt; [＆＃34; [...] / vendor / bundle / jruby / 1.9 / gems / logstash-core-2.3.3-java / lib / logstash / pipeline.rb：309：in { {1}} inflight_batches_synchronize＆＃39;＆＃34;，＆＃34; [...] / vendor / bundle / jruby / 1.9 / gems / logstash-core-2.3.3-java / lib / logstash / pipeline.rb ：234：在synchronize'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in start_workers＆＃39;＆＃34;]，＆＃34; blocked_on＆＃34; =＆gt; nil，＆＃34; status＆＃34; =＆gt;＆＃34 ; sleep＆＃34;，＆＃34; current_call＆＃34; =＆gt;＆＃34; [...] / vendor / bundle / jruby / 1.9 / gems / logstash-core-2.3.3-java / lib / logstash /pipeline.rb:309：worker_loop'", "[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:201:in接受＆＃39;＆＃34;}，{＆＃34; thread_id＆＃34; =＆gt; 18，＆＃34;名称＆＃34; =＆gt;＆＃ 34; [main]＆gt; worker0＆＃34;，＆＃34;插件＆＃34; =＆gt; [＆＃34; LogStash :: Filters :: Grok＆＃34;，{＆＃34; patterns_dir＆＃34; =＆gt; ;＆＃34; /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns /" ;,＆＃34;匹配＆＃34; =＆gt; {＆ ＃34;消息＆＃34; =＆gt;＆＃34;％{SYSLOGBASE2}＆＃34;}}]，＆＃34; current_call＆＃34; =＆gt;＆＃34; [...] / vendor / bundle /jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize'"}], :stalling_threads_info=>[{"thread_id"=>17, "name"=>"[main]<tcp", "plugin"=>nil, "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/jruby-openssl-0.9.13-java/lib/jopenssl19/openssl/ssl-internal.rb:106:in同步＆＃39;＆＃34;}，{＆＃34; thread_id＆＃ 34; =＆gt; 20，＆＃34;名称＆＃34; =＆gt;＆＃34; [主要]＆gt; worker2＆＃34;，＆＃34;插件＆＃34; =＆gt; [＆＃34; LogStash： ：过滤器:: Grok＆＃34;，{＆＃34; patterns_dir＆＃34; =＆gt;＆＃34; /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/模式/＆＃34;，＆＃34;匹配＆＃34; =＆gt; {＆＃34;消息＆＃34; =＆gt;＆＃34;％{SYSLOGBASE2}＆＃34;}}]，＆＃34; current_call＆＃34; = GT;＆＃34; [...] / vendor / bundle / jruby / 1.9 / gems / logstash-core-2.3.3-java / lib / logstash / pipeline.rb：309：in synchronize'"}, {"thread_id"=>19, "name"=>"[main]>worker1", "plugin"=>["LogStash::Filters::Grok", {"patterns_dir"=>"/opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/", "match"=>{"message"=>"%{SYSLOGBASE2}"}}], "current_call"=>"[...]/vendor/bundle/jruby/1.9/gems/logstash-core-2.3.3-java/lib/logstash/pipeline.rb:309:in synchronize＆＃39;＆＃34; }]}＆gt;，：level =＆gt;：warn}
答案 0 :(得分：0)
问题最终是我们正在为ES集群使用AWS发现。即使我们从我们使用的安全组中删除了死实例（在AWS中仍然可见，只是无法访问），但我们没有重新启动所有剩余的ES节点。一旦我们这样做了，一切都开始了。