Elasticsearch返回超出限制fielddata

时间:2016-06-27 09:35:00

标签: java elasticsearch spring-data spring-data-elasticsearch

我的排序有问题,即排序工作,但仅限于价格字段。当我尝试按start_date,end_date,uid,cat title排序时,会收到超出限制的消息:

Data too large, the date for [ "name of field here"] would be larger than the limit of [19798897459 / 18.4gb]]

我不知道为什么会发生这样的代码看起来正确的样本查询对于elastica看起来像这样:

映射:

"auctions": {
                "_all": { "enabled": false }, 
                "properties": {
                    "cat": { "store": true,  "type": "long" }, 
                    "curr": { "index": "not_analyzed",  "store": true,  "type": "string" }, 
                    "end_date": { "store": true,  "type": "long" }, 
                    "price": { "store": true,  "type": "long" }, 
                    "start_date": { "store": true,  "type": "long" }, 
                    "tcat": { "store": true,  "type": "long" }, 
                    "title": { "store": true,  "type": "string" }, 
                    "uid": { "store": true,  "type": "long" }
                }
            }, 

请求:

/搜索的uid = 335547&安培;标题= KARTA&安培; ORDERBY = UID&安培; orderDir = ASC

方法:

private NativeSearchQueryBuilder getSearchQuery(AuctionIndexSearchParams searchParams, Pageable pageable) {
        final List<FilterBuilder> filters = Lists.newArrayList();
        final NativeSearchQueryBuilder searchQuery = new NativeSearchQueryBuilder();
        Optional.ofNullable(searchParams.getCategoryId()).ifPresent(v -> filters.add(boolFilter().must(termFilter("cat", v))));
        Optional.ofNullable(searchParams.getCurrency()).ifPresent(v -> filters.add(boolFilter().must(termFilter("curr", v))));
        Optional.ofNullable(searchParams.getTreeCategoryId()).ifPresent(v -> filters.add(boolFilter().must(termFilter("tcat", v))));
        Optional.ofNullable(searchParams.getUid()).ifPresent(v -> filters.add(boolFilter().must(termFilter("uid", v))));
        final BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();

        //access for many uids
        if (searchParams.getUids() != null) {
            if (searchParams.getItemId() != null || searchParams.getTitle() != null) {
                Optional.ofNullable(searchParams.getUids().split(",")).ifPresent(v -> {
                    filters.add(boolFilter().must(termsFilter("uid", v)));
                });
            } else {
                for (String user : searchParams.getUids().split(",")) {
                    boolQueryBuilder.should(queryStringQuery(user).field("uid"));
                }
            }
        }
        //access for many categories
        if (searchParams.getCategories() != null) {
            Optional.ofNullable(searchParams.getCategories().split(",")).ifPresent(v -> {
                filters.add(boolFilter().must(termsFilter("cat", v)));
            });
        }

        if (searchParams.getItemId() != null) {
            boolQueryBuilder.must(queryStringQuery(searchParams.getItemId()).field("_id"));
        }

        if (Optional.ofNullable(searchParams.getTitle()).isPresent()) {
            boolQueryBuilder.must(queryStringQuery(searchParams.getTitle()).analyzeWildcard(true).field("title"));
        }

        if (Optional.ofNullable(searchParams.getStartDateFrom()).isPresent()
                || Optional.ofNullable(searchParams.getStartDateTo()).isPresent()) {

            filters.add(rangeFilter("start_date").from(searchParams.getStartDateFrom()).to(searchParams.getStartDateTo()));
        }

        if (Optional.ofNullable(searchParams.getEndDateFrom()).isPresent()
                || Optional.ofNullable(searchParams.getEndDateTo()).isPresent()) {
            filters.add(rangeFilter("end_date").from(searchParams.getEndDateFrom()).to(searchParams.getEndDateTo()));
        }

        if (Optional.ofNullable(searchParams.getPriceFrom()).isPresent()
                || Optional.ofNullable(searchParams.getPriceTo()).isPresent()) {
            filters.add(rangeFilter("price").from(searchParams.getPriceFrom()).to(searchParams.getPriceTo()));
        }

        searchQuery.withQuery(boolQueryBuilder);

        FilterBuilder[] filterArr = new FilterBuilder[filters.size()];
        filterArr = filters.toArray(filterArr);
        searchQuery.withFilter(andFilter(filterArr));

        if (searchParams.getOrderBy() != null && searchParams.getOrderDir() != null) {
            if (searchParams.getOrderDir().toLowerCase().equals("asc")) {
                searchQuery.withSort(SortBuilders.fieldSort(searchParams.getOrderBy()).order(SortOrder.ASC));
            } else {
                searchQuery.withSort(SortBuilders.fieldSort(searchParams.getOrderBy()).order(SortOrder.DESC));
            }
        }

        if (pageable != null) {
            searchQuery.withPageable(pageable);
        }
        System.out.println(searchQuery.build().getQuery());
        System.out.println(searchQuery.build().getFilter());
        System.out.println(searchQuery.build().getSort());
        return searchQuery;
    }

的System.out.println(searchQuery.build()getQuery());

{
  "bool": {
    "must": {
      "query_string": {
        "query", "card"
        "fields": [ "title"]
        "analyze_wildcard": true
      }
    }
  }
}

System.out.println(searchQuery.build()。getFilter());

{
  "and" {
    "filters": [{
      "bool": {
        "must": {
          "terms": {
            "uid" [ "335", "547"]
          }
        }
      }
    }]
  }
}

的System.out.println(searchQuery.build()getSort());

null

任何可能导致此异常的想法?

我应该补充一点,我已尝试过这些解决方案:

FIELDDATA Data is too large

但效果更糟,然后没有查询没有那么快。

对于任何帮助,我将非常感激!

/ _统计/ fielddata?字段= *

{
    "_shards": {
        "total": 10,
        "successful": 5,
        "failed": 0
    },
    "_all": {
        "primaries": {
            "fielddata": {
                "memory_size_in_bytes": 19466671904,
                "evictions": 0,
                "fields": {
                    "_id": {
                        "memory_size_in_bytes": 0
                    },
                    "cat": {
                        "memory_size_in_bytes": 0
                    },
                    "price": {
                        "memory_size_in_bytes": 3235221240
                    },
                    "title": {
                        "memory_size_in_bytes": 16231450664
                    }
                }
            }
        },
        "total": {
            "fielddata": {
                "memory_size_in_bytes": 19466671904,
                "evictions": 0,
                "fields": {
                    "_id": {
                        "memory_size_in_bytes": 0
                    },
                    "cat": {
                        "memory_size_in_bytes": 0
                    },
                    "price": {
                        "memory_size_in_bytes": 3235221240
                    },
                    "title": {
                        "memory_size_in_bytes": 16231450664
                    }
                }
            }
        }
    },
    "indices": {
        "allek": {
            "primaries": {
                "fielddata": {
                    "memory_size_in_bytes": 19466671904,
                    "evictions": 0,
                    "fields": {
                        "_id": {
                            "memory_size_in_bytes": 0
                        },
                        "cat": {
                            "memory_size_in_bytes": 0
                        },
                        "price": {
                            "memory_size_in_bytes": 3235221240
                        },
                        "title": {
                            "memory_size_in_bytes": 16231450664
                        }
                    }
                }
            },
            "total": {
                "fielddata": {
                    "memory_size_in_bytes": 19466671904,
                    "evictions": 0,
                    "fields": {
                        "_id": {
                            "memory_size_in_bytes": 0
                        },
                        "cat": {
                            "memory_size_in_bytes": 0
                        },
                        "price": {
                            "memory_size_in_bytes": 3235221240
                        },
                        "title": {
                            "memory_size_in_bytes": 16231450664
                        }
                    }
                }
            }
        }
    }

修改

我解决了以下问题:

经过辨别,结果证明我使用的是1.7版本。我在文档中发现,如果要排序或聚合,必须在映射中将doc_values设置为true。字段字符串需要添加另一个字段多字段。

因此,在地图更改为或多或少之后:

{
  "_all": {
    "enabled": false
  },
  "properties": {
    "cat": {
      "store": true,
      "type": "long",
      "doc_values": true
    },
    "curr": {
      "index": "not_analyzed",
      "store": true,
      "type": "string",
      "doc_values": true
    },
    "end_date": {
      "store": true,
      "type": "long",
      "doc_values": true
    },
    "price": {
      "store": true,
      "type": "long",
      "doc_values": true
    },
    "start_date": {
      "store": true,
      "type": "long",
      "doc_values": true
    },
    "tcat": {
      "store": true,
      "type": "long",
      "doc_values": true
    },
    "title": {
      "store": true,
      "type": "string",
      "fields": {
        "raw": {
          "type": "string",
          "index": "not_analyzed",
          "ignore_above": 256,
          "doc_values": true
        }
      }
    },
    "uid": {
      "store": true,
      "type": "long",
      "doc_values": true
    }
  }
}

排序工作,但减慢了整个系统搜索,我会说的很多,虽然文档大约是10-20%。

你还应该记得重新索引数据!

谢谢!

0 个答案:

没有答案