嵌套排序导致使用icu_collat​​ion的日语文本的CircuitBreakingException

Question

我正在使用Elasticsearch 2.4，添加了icu_analysis插件以提供日文文本的排序。它在我的本地环境中工作得很好，该环境具有有限数量的文档，但是当我在更现实的数据集上尝试它时，查询失败并出现以下CircuitBreakingException：

"CircuitBreakingException[[fielddata] Data too large, data for [translations.name.jp_sort] would be larger than limit of [10239895142/9.5gb]]"

我理解当尝试对具有大文档计数的fielddata进行排序并且应该使用docvalues时会发生这种情况 - 但我不确定是否可以在这种情况下完成或者为什么它不会发生。

索引中有大约4.7亿个文档，它们将翻译存储为嵌套文档 - 整个集合中只有大约3500万个文档包含日语翻译。以下是文档的映射：

{
  "settings" : {
    "number_of_shards" : 6,
    "number_of_replicas": 0,
    "analysis": {
        "filter": {
            "trigrams_filter": {
                "type":     "ngram",
                "min_gram": 3,
                "max_gram": 3
            },
          "japanese_ordering": {
            "type":     "icu_collation",
            "language": "ja",
            "country":  "JP"
          }
        },
      "analyzer": {
        "trigrams": {
          "tokenizer": "my_ngram_tokenizer",
          "filter": "lowercase"
        },
        "japanese_ordering": {
          "tokenizer": "keyword",
          "filter":  [ "japanese_ordering" ]
        }
      },
      "tokenizer": {
        "my_ngram_tokenizer": {
          "type": "nGram",
          "min_gram": "3",
          "max_gram": "3",
          "token_chars": [
            "letter",
            "digit",
            "symbol",
            "punctuation"
          ]
        }
      }
    }
  },
  "mappings" : {
    "product" : {
      "_all" : {
        "enabled" : false
      },
      "properties" : {
        "name" : {
          "type" : "string",
          "analyzer": "trigrams",
          "fields": {
            "value" : {
              "type": "string",
              "index": "not_analyzed"
            }
          }
        },
        "record_status" : {
          "type" : "integer"
        },
        "categories" : {
          "type" : "integer"
        },
        "variant_status" : {
          "type" : "integer"
        },
        "visit_count" : {
          "type" : "integer"
        },
        "translations": {
          "type": "nested",
          "properties": {
            "name": {
              "type": "string",
              "fields": {
                "jp_sort": { 
                  "type":     "string",
                  "analyzer": "japanese_ordering"
                }
              }
            },
            "language_id": {
              "type": "short"
            }
          }
        }
      }
    }
  }
}

这是CircuitBreaking的查询：

{
    "from": 0,
    "size": 20,
    "query": {
        "bool": {
            "should": [],
            "must_not": [],
            "must": [{
                "nested": {
                    "path": "translations",
                    "score_mode": "max",
                    "query": {
                        "bool": {
                            "must": [{
                                "match": {
                                    "translations.name": {
                                        "query": "\u30C6\u30B9\u30C8",
                                        "boost": 5
                                    }
                                }
                            }]
                        }
                    }
                }
            }]
        }
    },
    "filter": {
        "bool": {
            "must": [{
                "terms": {
                    "variant_status": ["1"],
                    "_cache": true
                }
            }, {
                "nested": {
                    "path": "translations",
                    "query": {
                        "bool": {
                            "must": [{
                                "term": {
                                    "translations.language_id": 9,
                                    "_cache": true
                                }
                            }]
                        }
                    }
                }
            }, {
                "term": {
                    "record_status": 1,
                    "_cache": true
                }
            }],
            "must_not": [{
                "term": {
                    "product_collections": 0
                }
            }]
        }
    },
    "sort": [{
        "translations.name.jp_sort": {
            "order": "asc",
            "nested_path": "translations"
        }
    }]
}

Answer 1

ES 5.5版本引入了一种名为icu_collation_keyword的新字段类型，它解决了您所面临的问题。

您可以在此处阅读更多内容：https://www.elastic.co/blog/elasticsearch-5-5-0-released