当前，我们有一个缓慢的Elasticsearch聚合查询。我们将数据分为多个索引，每个索引包含一个星期的数据，每个索引包含5个分片。
elasticsearch聚合查询如下所示：
{"profile": true, "size":0,"aggregations":{"whereAggr":{"filters":{"filters":{"combineCondition":{"bool":{"must":[{"bool":{"must":[{"range":{"createdAt":{"from":1569409138454,"to":1569495539454,"include_lower":true,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"term":{"clientId.keyword":{"value":"95ed418f-d255-4847-a432-b7326a8e73da","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}}},"other_bucket":false,"other_bucket_key":"_other_"},"aggregations":{"payoutAmount1Day":{"sum":{"field":"payerAmountUsd"}}}}}}
带有配置文件信息的查询响应：
{
"took": 519,
"timed_out": false,
"_shards": {
"total": 240,
"successful": 240,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 462286,
"max_score": 0,
"hits": []
},
"aggregations": {
"whereAggr": {
"buckets": {
"combineCondition": {
"doc_count": 5,
"payoutAmount1Day": {
"value": 2802.9300000000003
}
}
}
}
},
"profile": {
"shards": [
{
"id": "[-p9A-Zx4TI6JkoA2CZGvvA][payment-2018-07-09][0]",
"searches": [
{
"query": [
{
"type": "MatchAllDocsQuery",
"description": "*:*",
"time_in_nanos": 1619010,
"breakdown": {
"score": 0,
"build_scorer_count": 4,
"match_count": 0,
"create_weight": 3600,
"next_doc": 5487,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 4,
"score_count": 0,
"build_scorer": 1609914,
"advance": 0,
"advance_count": 0
}
},
{
"type": "BooleanQuery",
"description": "+createdAt:[1569409138454 TO 1569495539454] +clientId.keyword:95ed418f-d255-4847-a432-b7326a8e73da",
"time_in_nanos": 1409164,
"breakdown": {
"score": 0,
"build_scorer_count": 2,
"match_count": 0,
"create_weight": 65798,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 1343363,
"advance": 0,
"advance_count": 0
},
"children": [
{
"type": "IndexOrDocValuesQuery",
"description": "createdAt:[1569409138454 TO 1569495539454]",
"time_in_nanos": 14347,
"breakdown": {
"score": 0,
"build_scorer_count": 2,
"match_count": 0,
"create_weight": 1123,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 13221,
"advance": 0,
"advance_count": 0
}
},
{
"type": "TermQuery",
"description": "clientId.keyword:95ed418f-d255-4847-a432-b7326a8e73da",
"time_in_nanos": 20745,
"breakdown": {
"score": 0,
"build_scorer_count": 2,
"match_count": 0,
"create_weight": 1278,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 19464,
"advance": 0,
"advance_count": 0
}
}
]
}
],
"rewrite_time": 444841,
"collector": [
{
"name": "CancellableCollector",
"reason": "search_cancelled",
"time_in_nanos": 1906710,
"children": [
{
"name": "MultiCollector",
"reason": "search_multi",
"time_in_nanos": 1592503,
"children": [
{
"name": "EarlyTerminatingCollector",
"reason": "search_count",
"time_in_nanos": 41650
},
{
"name": "ProfilingAggregator: [whereAggr]",
"reason": "aggregation",
"time_in_nanos": 1530021
}
]
}
]
}
]
}
],
"aggregations": [
{
"type": "FiltersAggregator",
"description": "whereAggr",
"time_in_nanos": 25579,
"breakdown": {
"reduce": 0,
"build_aggregation": 2764,
"build_aggregation_count": 1,
"initialize": 21355,
"initialize_count": 1,
"reduce_count": 0,
"collect": 1456,
"collect_count": 2
},
"children": [
{
"type": "SumAggregator",
"description": "payoutAmount1Day",
"time_in_nanos": 1432,
"breakdown": {
"reduce": 0,
"build_aggregation": 591,
"build_aggregation_count": 1,
"initialize": 839,
"initialize_count": 1,
"reduce_count": 0,
"collect": 0,
"collect_count": 0
}
}
]
}
]
},
{
"id": "[-p9A-Zx4TI6JkoA2CZGvvA][payment-2018-07-09][3]",
"searches": [
{
"query": [
{
"type": "MatchAllDocsQuery",
"description": "*:*",
"time_in_nanos": 7322,
"breakdown": {
"score": 0,
"build_scorer_count": 6,
"match_count": 0,
"create_weight": 1909,
"next_doc": 665,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 6,
"score_count": 0,
"build_scorer": 4735,
"advance": 0,
"advance_count": 0
}
},
{
"type": "BooleanQuery",
"description": "+createdAt:[1569409138454 TO 1569495539454] +clientId.keyword:95ed418f-d255-4847-a432-b7326a8e73da",
"time_in_nanos": 58604,
"breakdown": {
"score": 0,
"build_scorer_count": 3,
"match_count": 0,
"create_weight": 15711,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 42889,
"advance": 0,
"advance_count": 0
},
"children": [
{
"type": "IndexOrDocValuesQuery",
"description": "createdAt:[1569409138454 TO 1569495539454]",
"time_in_nanos": 12802,
"breakdown": {
"score": 0,
"build_scorer_count": 3,
"match_count": 0,
"create_weight": 1265,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 11533,
"advance": 0,
"advance_count": 0
}
},
{
"type": "TermQuery",
"description": "clientId.keyword:95ed418f-d255-4847-a432-b7326a8e73da",
"time_in_nanos": 20187,
"breakdown": {
"score": 0,
"build_scorer_count": 3,
"match_count": 0,
"create_weight": 985,
"next_doc": 0,
"match": 0,
"create_weight_count": 1,
"next_doc_count": 0,
"score_count": 0,
"build_scorer": 19198,
"advance": 0,
"advance_count": 0
}
}
]
}
],
"rewrite_time": 6118,
"collector": [
{
"name": "CancellableCollector",
"reason": "search_cancelled",
"time_in_nanos": 141778,
"children": [
{
"name": "MultiCollector",
"reason": "search_multi",
"time_in_nanos": 138392,
"children": [
{
"name": "EarlyTerminatingCollector",
"reason": "search_count",
"time_in_nanos": 35895
},
{
"name": "ProfilingAggregator: [whereAggr]",
"reason": "aggregation",
"time_in_nanos": 97311
}
]
}
]
}
]
}
],
"aggregations": [
{
"type": "FiltersAggregator",
"description": "whereAggr",
"time_in_nanos": 27128,
"breakdown": {
"reduce": 0,
"build_aggregation": 17543,
"build_aggregation_count": 1,
"initialize": 9223,
"initialize_count": 1,
"reduce_count": 0,
"collect": 357,
"collect_count": 3
},
"children": [
{
"type": "SumAggregator",
"description": "payoutAmount1Day",
"time_in_nanos": 1527,
"breakdown": {
"reduce": 0,
"build_aggregation": 569,
"build_aggregation_count": 1,
"initialize": 956,
"initialize_count": 1,
"reduce_count": 0,
"collect": 0,
"collect_count": 0
}
}
]
}
]
},
{
"id": "[-p9A-Zx4TI6JkoA2CZGvvA][payment-2018-07-09][4]",
.....
}
}
}
从结果观察来看，针对每个索引/分片的每个聚合看起来都不错。但是，它们太多了！查询中涉及240个分片，而查询仅尝试在一天之内聚合数据。随着时间的流逝，情况将变得更糟。
有些想法，我不确定它们是否有意义： 鉴于数据量大，每个索引5个分片可能会过大，但是一旦索引已经启用，是否有办法减少分片的数量。
该查询包含一个日期范围过滤器，因此，如果我们可以过滤掉许多不属于该日期范围的索引，则将很有帮助。还是将过滤器从聚合部分移到查询部分帮助？
或者关于如何提高查询性能还有其他好的建议？