为什么我的Elasticsearch索引占用了这么多磁盘空间?

时间:2019-10-02 13:55:31

标签: elasticsearch diskspace

当前,我在服务器上遇到磁盘空间问题,并且似乎找到了罪魁祸首: 根据某些人的推测,此索引的大小约为40 gb。这听起来很荒谬,因为我保存的平均坐标列表大约包含5个元素。

此索引有11000个文档。

我尝试过的事情:

  1. 删除索引并再次填满,这导致Elasticsearch的索引小得多40GB-> 14MB?这就是为什么我对此是否会非常怀疑的原因。
  2. 试图在没有任何具体解决方案的情况下研究谷歌。

我真的希望有人可能对为什么这么大有一定见识。

我的Elasticsearch版本是:5.6.11

我的设置

{
  "my_polygons_1": {
    "settings": {
      "index": {
        "number_of_shards": "5",
        "provided_name": "my_polygons_1",
        "creation_date": "1514988794703",
        "analysis": {
          "filter": {
            "addresstranslation": {
              "type": "synonym",
              "synonyms": [
                "1e,eerste",
                "2e,tweede",
                "3e,derde",
                "1ste,eerste",
                "2de,tweede",
                "3de,derde"
              ]
            },
            "translation": {
              "token_chars": [
                "letter",
                "digit",
                "whitespace"
              ],
              "min_gram": "3",
              "type": "nGram",
              "max_gram": "50"
            }
          },
          "analyzer": {
            "translation_index_analyzer": {
              "filter": "standard,lowercase,asciifolding,translation",
              "type": "custom",
              "tokenizer": "standard"
            },
            "address_search_analyzer": {
              "filter": "standard,lowercase,asciifolding,addresstranslation",
              "type": "custom",
              "tokenizer": "standard"
            },
            "address_index_analyzer": {
              "filter": "standard,lowercase,asciifolding,addresstranslation,translation",
              "type": "custom",
              "tokenizer": "standard"
            },
            "translation_search_analyzer": {
              "filter": "standard,lowercase,asciifolding",
              "type": "custom",
              "tokenizer": "standard"
            }
          }
        },
        "number_of_replicas": "1",
        "uuid": "lQPnGOZ6TdKx9xZwNPfs2g",
        "version": {
          "created": "5060399"
        }
      }
    }
  }
}

我的_stats:

{
  "_shards": {
    "total": 10,
    "successful": 10,
    "failed": 0
  },
  "_all": {
    "primaries": {
      "docs": {
        "count": 11331,
        "deleted": 646
      },
      "store": {
        "size_in_bytes": 22095974914,
        "throttle_time_in_millis": 0
      },
      "indexing": {
        "index_total": 1184,
        "index_time_in_millis": 1166437,
        "index_current": 0,
        "index_failed": 0,
        "delete_total": 0,
        "delete_time_in_millis": 0,
        "delete_current": 0,
        "noop_update_total": 0,
        "is_throttled": false,
        "throttle_time_in_millis": 0
      },
      "get": {
        "total": 368507,
        "time_in_millis": 159070,
        "exists_total": 354156,
        "exists_time_in_millis": 157586,
        "missing_total": 14351,
        "missing_time_in_millis": 1484,
        "current": 0
      },
      "search": {
        "open_contexts": 0,
        "query_total": 185,
        "query_time_in_millis": 46,
        "query_current": 0,
        "fetch_total": 30,
        "fetch_time_in_millis": 89,
        "fetch_current": 0,
        "scroll_total": 2,
        "scroll_time_in_millis": 2444,
        "scroll_current": 0,
        "suggest_total": 0,
        "suggest_time_in_millis": 0,
        "suggest_current": 0
      },
      "merges": {
        "current": 0,
        "current_docs": 0,
        "current_size_in_bytes": 0,
        "total": 132,
        "total_time_in_millis": 714437,
        "total_docs": 6268,
        "total_size_in_bytes": 9036940847,
        "total_stopped_time_in_millis": 0,
        "total_throttled_time_in_millis": 88138,
        "total_auto_throttle_in_bytes": 73181749
      },
      "refresh": {
        "total": 3971,
        "total_time_in_millis": 1052798,
        "listeners": 0
      },
      "flush": {
        "total": 1063,
        "total_time_in_millis": 10994
      },
      "warmer": {
        "current": 0,
        "total": 3474,
        "total_time_in_millis": 55
      },
      "query_cache": {
        "memory_size_in_bytes": 0,
        "total_count": 0,
        "hit_count": 0,
        "miss_count": 0,
        "cache_size": 0,
        "cache_count": 0,
        "evictions": 0
      },
      "fielddata": {
        "memory_size_in_bytes": 0,
        "evictions": 0
      },
      "completion": {
        "size_in_bytes": 0
      },
      "segments": {
        "count": 123,
        "memory_in_bytes": 597667360,
        "terms_memory_in_bytes": 597553396,
        "stored_fields_memory_in_bytes": 38640,
        "term_vectors_memory_in_bytes": 0,
        "norms_memory_in_bytes": 7872,
        "points_memory_in_bytes": 368,
        "doc_values_memory_in_bytes": 67084,
        "index_writer_memory_in_bytes": 0,
        "version_map_memory_in_bytes": 0,
        "fixed_bit_set_memory_in_bytes": 0,
        "max_unsafe_auto_id_timestamp": -1,
        "file_sizes": {}
      },
      "translog": {
        "operations": 0,
        "size_in_bytes": 215
      },
      "request_cache": {
        "memory_size_in_bytes": 0,
        "evictions": 8,
        "hit_count": 0,
        "miss_count": 100
      },
      "recovery": {
        "current_as_source": 0,
        "current_as_target": 0,
        "throttle_time_in_millis": 0
      }
    },
    "total": {
      "docs": {
        "count": 22662,
        "deleted": 2356
      },
      "store": {
        "size_in_bytes": 46187867347,
        "throttle_time_in_millis": 0
      },
      "indexing": {
        "index_total": 2263,
        "index_time_in_millis": 2216831,
        "index_current": 0,
        "index_failed": 0,
        "delete_total": 0,
        "delete_time_in_millis": 0,
        "delete_current": 0,
        "noop_update_total": 0,
        "is_throttled": false,
        "throttle_time_in_millis": 0
      },
      "get": {
        "total": 657621,
        "time_in_millis": 250947,
        "exists_total": 628867,
        "exists_time_in_millis": 247657,
        "missing_total": 28754,
        "missing_time_in_millis": 3290,
        "current": 0
      },
      "search": {
        "open_contexts": 0,
        "query_total": 375,
        "query_time_in_millis": 128,
        "query_current": 0,
        "fetch_total": 73,
        "fetch_time_in_millis": 219,
        "fetch_current": 0,
        "scroll_total": 5,
        "scroll_time_in_millis": 6110,
        "scroll_current": 0,
        "suggest_total": 0,
        "suggest_time_in_millis": 0,
        "suggest_current": 0
      },
      "merges": {
        "current": 0,
        "current_docs": 0,
        "current_size_in_bytes": 0,
        "total": 252,
        "total_time_in_millis": 1134358,
        "total_docs": 10222,
        "total_size_in_bytes": 13797985517,
        "total_stopped_time_in_millis": 0,
        "total_throttled_time_in_millis": 160514,
        "total_auto_throttle_in_bytes": 155271619
      },
      "refresh": {
        "total": 7539,
        "total_time_in_millis": 1978415,
        "listeners": 0
      },
      "flush": {
        "total": 2030,
        "total_time_in_millis": 21846
      },
      "warmer": {
        "current": 0,
        "total": 6641,
        "total_time_in_millis": 105
      },
      "query_cache": {
        "memory_size_in_bytes": 0,
        "total_count": 0,
        "hit_count": 0,
        "miss_count": 0,
        "cache_size": 0,
        "cache_count": 0,
        "evictions": 0
      },
      "fielddata": {
        "memory_size_in_bytes": 0,
        "evictions": 0
      },
      "completion": {
        "size_in_bytes": 0
      },
      "segments": {
        "count": 261,
        "memory_in_bytes": 1246021155,
        "terms_memory_in_bytes": 1245778862,
        "stored_fields_memory_in_bytes": 81992,
        "term_vectors_memory_in_bytes": 0,
        "norms_memory_in_bytes": 16704,
        "points_memory_in_bytes": 769,
        "doc_values_memory_in_bytes": 142828,
        "index_writer_memory_in_bytes": 0,
        "version_map_memory_in_bytes": 0,
        "fixed_bit_set_memory_in_bytes": 0,
        "max_unsafe_auto_id_timestamp": -1,
        "file_sizes": {}
      },
      "translog": {
        "operations": 0,
        "size_in_bytes": 430
      },
      "request_cache": {
        "memory_size_in_bytes": 0,
        "evictions": 16,
        "hit_count": 0,
        "miss_count": 200
      },
      "recovery": {
        "current_as_source": 0,
        "current_as_target": 0,
        "throttle_time_in_millis": 0
      }
    }
  },
  "indices": {
    "my_polygons_1": {
      "primaries": {
        "docs": {
          "count": 11331,
          "deleted": 646
        },
        "store": {
          "size_in_bytes": 22095974914,
          "throttle_time_in_millis": 0
        },
        "indexing": {
          "index_total": 1184,
          "index_time_in_millis": 1166437,
          "index_current": 0,
          "index_failed": 0,
          "delete_total": 0,
          "delete_time_in_millis": 0,
          "delete_current": 0,
          "noop_update_total": 0,
          "is_throttled": false,
          "throttle_time_in_millis": 0
        },
        "get": {
          "total": 368507,
          "time_in_millis": 159070,
          "exists_total": 354156,
          "exists_time_in_millis": 157586,
          "missing_total": 14351,
          "missing_time_in_millis": 1484,
          "current": 0
        },
        "search": {
          "open_contexts": 0,
          "query_total": 185,
          "query_time_in_millis": 46,
          "query_current": 0,
          "fetch_total": 30,
          "fetch_time_in_millis": 89,
          "fetch_current": 0,
          "scroll_total": 2,
          "scroll_time_in_millis": 2444,
          "scroll_current": 0,
          "suggest_total": 0,
          "suggest_time_in_millis": 0,
          "suggest_current": 0
        },
        "merges": {
          "current": 0,
          "current_docs": 0,
          "current_size_in_bytes": 0,
          "total": 132,
          "total_time_in_millis": 714437,
          "total_docs": 6268,
          "total_size_in_bytes": 9036940847,
          "total_stopped_time_in_millis": 0,
          "total_throttled_time_in_millis": 88138,
          "total_auto_throttle_in_bytes": 73181749
        },
        "refresh": {
          "total": 3971,
          "total_time_in_millis": 1052798,
          "listeners": 0
        },
        "flush": {
          "total": 1063,
          "total_time_in_millis": 10994
        },
        "warmer": {
          "current": 0,
          "total": 3474,
          "total_time_in_millis": 55
        },
        "query_cache": {
          "memory_size_in_bytes": 0,
          "total_count": 0,
          "hit_count": 0,
          "miss_count": 0,
          "cache_size": 0,
          "cache_count": 0,
          "evictions": 0
        },
        "fielddata": {
          "memory_size_in_bytes": 0,
          "evictions": 0
        },
        "completion": {
          "size_in_bytes": 0
        },
        "segments": {
          "count": 123,
          "memory_in_bytes": 597667360,
          "terms_memory_in_bytes": 597553396,
          "stored_fields_memory_in_bytes": 38640,
          "term_vectors_memory_in_bytes": 0,
          "norms_memory_in_bytes": 7872,
          "points_memory_in_bytes": 368,
          "doc_values_memory_in_bytes": 67084,
          "index_writer_memory_in_bytes": 0,
          "version_map_memory_in_bytes": 0,
          "fixed_bit_set_memory_in_bytes": 0,
          "max_unsafe_auto_id_timestamp": -1,
          "file_sizes": {}
        },
        "translog": {
          "operations": 0,
          "size_in_bytes": 215
        },
        "request_cache": {
          "memory_size_in_bytes": 0,
          "evictions": 8,
          "hit_count": 0,
          "miss_count": 100
        },
        "recovery": {
          "current_as_source": 0,
          "current_as_target": 0,
          "throttle_time_in_millis": 0
        }
      },
      "total": {
        "docs": {
          "count": 22662,
          "deleted": 2356
        },
        "store": {
          "size_in_bytes": 46187867347,
          "throttle_time_in_millis": 0
        },
        "indexing": {
          "index_total": 2263,
          "index_time_in_millis": 2216831,
          "index_current": 0,
          "index_failed": 0,
          "delete_total": 0,
          "delete_time_in_millis": 0,
          "delete_current": 0,
          "noop_update_total": 0,
          "is_throttled": false,
          "throttle_time_in_millis": 0
        },
        "get": {
          "total": 657621,
          "time_in_millis": 250947,
          "exists_total": 628867,
          "exists_time_in_millis": 247657,
          "missing_total": 28754,
          "missing_time_in_millis": 3290,
          "current": 0
        },
        "search": {
          "open_contexts": 0,
          "query_total": 375,
          "query_time_in_millis": 128,
          "query_current": 0,
          "fetch_total": 73,
          "fetch_time_in_millis": 219,
          "fetch_current": 0,
          "scroll_total": 5,
          "scroll_time_in_millis": 6110,
          "scroll_current": 0,
          "suggest_total": 0,
          "suggest_time_in_millis": 0,
          "suggest_current": 0
        },
        "merges": {
          "current": 0,
          "current_docs": 0,
          "current_size_in_bytes": 0,
          "total": 252,
          "total_time_in_millis": 1134358,
          "total_docs": 10222,
          "total_size_in_bytes": 13797985517,
          "total_stopped_time_in_millis": 0,
          "total_throttled_time_in_millis": 160514,
          "total_auto_throttle_in_bytes": 155271619
        },
        "refresh": {
          "total": 7539,
          "total_time_in_millis": 1978415,
          "listeners": 0
        },
        "flush": {
          "total": 2030,
          "total_time_in_millis": 21846
        },
        "warmer": {
          "current": 0,
          "total": 6641,
          "total_time_in_millis": 105
        },
        "query_cache": {
          "memory_size_in_bytes": 0,
          "total_count": 0,
          "hit_count": 0,
          "miss_count": 0,
          "cache_size": 0,
          "cache_count": 0,
          "evictions": 0
        },
        "fielddata": {
          "memory_size_in_bytes": 0,
          "evictions": 0
        },
        "completion": {
          "size_in_bytes": 0
        },
        "segments": {
          "count": 261,
          "memory_in_bytes": 1246021155,
          "terms_memory_in_bytes": 1245778862,
          "stored_fields_memory_in_bytes": 81992,
          "term_vectors_memory_in_bytes": 0,
          "norms_memory_in_bytes": 16704,
          "points_memory_in_bytes": 769,
          "doc_values_memory_in_bytes": 142828,
          "index_writer_memory_in_bytes": 0,
          "version_map_memory_in_bytes": 0,
          "fixed_bit_set_memory_in_bytes": 0,
          "max_unsafe_auto_id_timestamp": -1,
          "file_sizes": {}
        },
        "translog": {
          "operations": 0,
          "size_in_bytes": 430
        },
        "request_cache": {
          "memory_size_in_bytes": 0,
          "evictions": 16,
          "hit_count": 0,
          "miss_count": 200
        },
        "recovery": {
          "current_as_source": 0,
          "current_as_target": 0,
          "throttle_time_in_millis": 0
        }
      }
    }
  }
}

和我的映射:

{
  "my_polygons_1": {
    "aliases": {},
    "mappings": {
      "my_polygons_1": {
        "properties": {
          "client_id": {
            "type": "integer"
          },
          "id": {
            "type": "integer"
          },
          "name": {
            "type": "keyword"
          },
          "polygon": {
            "type": "geo_shape",
            "tree": "quadtree",
            "precision": "1.0m"
          }
        }
      }
    },
    "settings": {
      "index": {
        "number_of_shards": "5",
        "provided_name": "my_polygons_1",
        "creation_date": "1514988794703",
        "analysis": {
          "filter": {
            "addresstranslation": {
              "type": "synonym",
              "synonyms": [
                "1e,eerste",
                "2e,tweede",
                "3e,derde",
                "1ste,eerste",
                "2de,tweede",
                "3de,derde"
              ]
            },
            "translation": {
              "token_chars": [
                "letter",
                "digit",
                "whitespace"
              ],
              "min_gram": "3",
              "type": "nGram",
              "max_gram": "50"
            }
          },
          "analyzer": {
            "translation_index_analyzer": {
              "filter": "standard,lowercase,asciifolding,translation",
              "type": "custom",
              "tokenizer": "standard"
            },
            "address_search_analyzer": {
              "filter": "standard,lowercase,asciifolding,addresstranslation",
              "type": "custom",
              "tokenizer": "standard"
            },
            "address_index_analyzer": {
              "filter": "standard,lowercase,asciifolding,addresstranslation,translation",
              "type": "custom",
              "tokenizer": "standard"
            },
            "translation_search_analyzer": {
              "filter": "standard,lowercase,asciifolding",
              "type": "custom",
              "tokenizer": "standard"
            }
          }
        },
        "number_of_replicas": "1",
        "uuid": "lQPnGOZ6TdKx9xZwNPfs2g",
        "version": {
          "created": "5060399"
        }
      }
    }
  }
}

0 个答案:

没有答案