我正在观察Elasticsearch节点中的高磁盘读取I / O.
环境
细分和合并可能是问题之一。但正如link中提到的，我没有看到任何INFO日志说明now throttling indexing。
有人能告诉我可能出现的问题吗？如何调试此问题？
节点统计信息如下所示： -
"indices" : {
"docs" : {
"count" : 42096315,
"deleted" : 16809358
},
"store" : {
"size_in_bytes" : 188061514556,
"throttle_time_in_millis" : 0
},
"indexing" : {
"index_total" : 4971184,
"index_time_in_millis" : 24161898,
"index_current" : 0,
"index_failed" : 0,
"delete_total" : 10317957,
"delete_time_in_millis" : 3323977,
"delete_current" : 0,
"noop_update_total" : 165455,
"is_throttled" : false,
"throttle_time_in_millis" : 0
},
"get" : {
"total" : 3078536,
"time_in_millis" : 4166218,
"exists_total" : 3076266,
"exists_time_in_millis" : 4165295,
"missing_total" : 2270,
"missing_time_in_millis" : 923,
"current" : 0
},
"search" : {
"open_contexts" : 0,
"query_total" : 811510,
"query_time_in_millis" : 49063191,
"query_current" : 0,
"fetch_total" : 55590,
"fetch_time_in_millis" : 2561106,
"fetch_current" : 0,
"scroll_total" : 205,
"scroll_time_in_millis" : 68748,
"scroll_current" : 0
},
"merges" : {
"current" : 0,
"current_docs" : 0,
"current_size_in_bytes" : 0,
"total" : 37047,
"total_time_in_millis" : 29575123,
"total_docs" : 48646732,
"total_size_in_bytes" : 189196722890,
"total_stopped_time_in_millis" : 0,
"total_throttled_time_in_millis" : 7243267,
"total_auto_throttle_in_bytes" : 186360552
},
"refresh" : {
"total" : 253329,
"total_time_in_millis" : 14111583
},
"flush" : {
"total" : 824,
"total_time_in_millis" : 197608
},
"warmer" : {
"current" : 0,
"total" : 479781,
"total_time_in_millis" : 440805
},
"query_cache" : {
"memory_size_in_bytes" : 523777408,
"total_count" : 144964341,
"hit_count" : 5901881,
"miss_count" : 139062460,
"cache_size" : 7773,
"cache_count" : 442280,
"evictions" : 434507
},
"fielddata" : {
"memory_size_in_bytes" : 0,
"evictions" : 0
},
"segments" : {
"count" : 263,
"memory_in_bytes" : 273031904,
"terms_memory_in_bytes" : 203860300,
"stored_fields_memory_in_bytes" : 34899016,
"term_vectors_memory_in_bytes" : 0,
"norms_memory_in_bytes" : 5229120,
"doc_values_memory_in_bytes" : 29043468,
"index_writer_memory_in_bytes" : 0,
"index_writer_max_memory_in_bytes" : 527069180,
"version_map_memory_in_bytes" : 14761,
"fixed_bit_set_memory_in_bytes" : 7048640
},
"translog" : {
"operations" : 137655,
"size_in_bytes" : 122949018
},
"request_cache" : {
"memory_size_in_bytes" : 0,
"evictions" : 0,
"hit_count" : 0,
"miss_count" : 0
}
},
"os" : {
"timestamp" : 1508238172920,
"cpu_percent" : 91,
"load_average" : 22.31,
"mem" : {
"total_in_bytes" : 67543134208,
"free_in_bytes" : 912490496,
"used_in_bytes" : 66630643712,
"free_percent" : 1,
"used_percent" : 99
},
"swap" : {
"total_in_bytes" : 4093636608,
"free_in_bytes" : 1753239552,
"used_in_bytes" : 2340397056
}
},
"process" : {
"timestamp" : 1508238172920,
"open_file_descriptors" : 915,
"max_file_descriptors" : 65535,
"cpu" : {
"percent" : 2,
"total_in_millis" : 99746040
},
"mem" : {
"total_virtual_in_bytes" : 87529877504
}
},
"jvm" : {
"timestamp" : 1508238172921,
"uptime_in_millis" : 292500150,
"mem" : {
"heap_used_in_bytes" : 1868708912,
"heap_used_percent" : 35,
"heap_committed_in_bytes" : 5255331840,
"heap_max_in_bytes" : 5255331840,
"non_heap_used_in_bytes" : 103936064,
"non_heap_committed_in_bytes" : 106307584,
"pools" : {
"young" : {
"used_in_bytes" : 153647352,
"max_in_bytes" : 907345920,
"peak_used_in_bytes" : 907345920,
"peak_max_in_bytes" : 907345920
},
"survivor" : {
"used_in_bytes" : 35321888,
"max_in_bytes" : 113377280,
"peak_used_in_bytes" : 113377280,
"peak_max_in_bytes" : 113377280
},
"old" : {
"used_in_bytes" : 1679739672,
"max_in_bytes" : 4234608640,
"peak_used_in_bytes" : 3660407304,
"peak_max_in_bytes" : 4234608640
}
}
},
"threads" : {
"count" : 199,
"peak_count" : 204
},
"gc" : {
"collectors" : {
"young" : {
"collection_count" : 32655,
"collection_time_in_millis" : 1844356
},
"old" : {
"collection_count" : 611,
"collection_time_in_millis" : 176197
}
}
},
"buffer_pools" : {
"direct" : {
"count" : 258,
"used_in_bytes" : 269582255,
"total_capacity_in_bytes" : 269582255
},
"mapped" : {
"count" : 146,
"used_in_bytes" : 71612833894,
"total_capacity_in_bytes" : 71612833894
}
},
"classes" : {
"current_loaded_count" : 8483,
"total_loaded_count" : 8499,
"total_unloaded_count" : 16
}
},
"thread_pool" : {
"bulk" : {
"threads" : 16,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 16,
"completed" : 44913
},
"fetch_shard_started" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 9,
"completed" : 13
},
"fetch_shard_store" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"flush" : {
"threads" : 5,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 5,
"completed" : 9011
},
"force_merge" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"generic" : {
"threads" : 1,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 6,
"completed" : 29268
},
"get" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"index" : {
"threads" : 16,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 16,
"completed" : 9460079
},
"listener" : {
"threads" : 8,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 8,
"completed" : 1237173
},
"management" : {
"threads" : 4,
"queue" : 0,
"active" : 1,
"rejected" : 0,
"largest" : 4,
"completed" : 44128
},
"percolate" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"refresh" : {
"threads" : 8,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 8,
"completed" : 253330
},
"search" : {
"threads" : 25,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 25,
"completed" : 992032
},
"snapshot" : {
"threads" : 0,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 0,
"completed" : 0
},
"warmer" : {
"threads" : 5,
"queue" : 0,
"active" : 0,
"rejected" : 0,
"largest" : 5,
"completed" : 518307
}
},
"fs" : {
"timestamp" : 1508238172922,
"total" : {
"total_in_bytes" : 1847237029888,
"free_in_bytes" : 921102319616,
"available_in_bytes" : 839458172928
},
"data" : [ {
"path" : "/var/lib/elasticsearch/elasticsearch/nodes/0",
"mount" : "/var (/dev/mapper/vag-var)",
"type" : "ext4",
"total_in_bytes" : 1847237029888,
"free_in_bytes" : 921102319616,
"available_in_bytes" : 839458172928,
"spins" : "false"
} ]
},
"transport" : {
"server_open" : 140,
"rx_count" : 7926335,
"rx_size_in_bytes" : 15511144109,
"tx_count" : 7667433,
"tx_size_in_bytes" : 47171921335
},
"http" : {
"current_open" : 1,
"total_opened" : 63123
},
"breakers" : {
"request" : {
"limit_size_in_bytes" : 2102132736,
"limit_size" : "1.9gb",
"estimated_size_in_bytes" : 0,
"estimated_size" : "0b",
"overhead" : 1.0,
"tripped" : 0
},
"fielddata" : {
"limit_size_in_bytes" : 3153199104,
"limit_size" : "2.9gb",
"estimated_size_in_bytes" : 0,
"estimated_size" : "0b",
"overhead" : 1.03,
"tripped" : 0
},
"parent" : {
"limit_size_in_bytes" : 3678732288,
"limit_size" : "3.4gb",
"estimated_size_in_bytes" : 0,
"estimated_size" : "0b",
"overhead" : 1.0,
"tripped" : 0
}
}
}
}
}
refresh_interval＆amp; translog.flush_threshold_size都设置为默认值。
热线报告如下： - （_nodes/hot_threads?pretty"
）
Hot threads at 2017-10-17T12:45:39.517Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:
71.6% (357.8ms out of 500ms) cpu usage by thread 'elasticsearch[Axum][[denorm][1]: Lucene Merge Thread #6011]'
3/10 snapshots sharing following 13 elements
org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:275)
org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:301)
org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.next(FilterLeafReader.java:195)
org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.write(BlockTreeTermsWriter.java:438)
org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198)
org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:193)
org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95)
org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4075)
org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3655)
org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:588)
org.elasticsearch.index.engine.ElasticsearchConcurrentMergeScheduler.doMerge(ElasticsearchConcurrentMergeScheduler.java:94)
org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:626)