Elasticsearch中的内存分配

时间:2018-09-06 06:01:33

标签: elasticsearch aws-elasticsearch

我正在使用AWS弹性搜索服务,并且每个节点具有16GB RAM,4个核心和160GB EBS。看着

GET _nodes/stats

我发现我的节点只剩下0.27GB的内存。我了解以下事实:这些进程专用于以下数量的内存:

堆:8GB 掉期:2GB Fielddate + Segments:1GB

我不了解其余4-5GB内存会发生什么,哪个进程正在消耗此内存?

以下是我从上述api获得的一个节点的统计信息:

{
      "timestamp": 1536212516381,
      "name": "",
      "roles": [
        "master",
        "data",
        "ingest"
      ],
      "indices": {
        "docs": {
          "count": 40811869,
          "deleted": 12313549
        },
        "store": {
          "size_in_bytes": 21211684802
        },
        "indexing": {
          "index_total": 17141254,
          "index_time_in_millis": 23271050,
          "index_current": 0,
          "index_failed": 0,
          "delete_total": 0,
          "delete_time_in_millis": 0,
          "delete_current": 0,
          "noop_update_total": 0,
          "is_throttled": false,
          "throttle_time_in_millis": 0
        },
        "get": {
          "total": 11,
          "time_in_millis": 4,
          "exists_total": 10,
          "exists_time_in_millis": 4,
          "missing_total": 1,
          "missing_time_in_millis": 0,
          "current": 0
        },
        "search": {
          "open_contexts": 0,
          "query_total": 60495650,
          "query_time_in_millis": 164996296,
          "query_current": 0,
          "fetch_total": 1338,
          "fetch_time_in_millis": 3947,
          "fetch_current": 0,
          "scroll_total": 1541,
          "scroll_time_in_millis": 147381782,
          "scroll_current": 0,
          "suggest_total": 0,
          "suggest_time_in_millis": 0,
          "suggest_current": 0
        },
        "merges": {
          "current": 0,
          "current_docs": 0,
          "current_size_in_bytes": 0,
          "total": 752561,
          "total_time_in_millis": 156448427,
          "total_docs": 6278661063,
          "total_size_in_bytes": 2061451815435,
          "total_stopped_time_in_millis": 0,
          "total_throttled_time_in_millis": 64499,
          "total_auto_throttle_in_bytes": 231968018126
        },
        "refresh": {
          "total": 9621793,
          "total_time_in_millis": 126038098,
          "listeners": 0
        },
        "flush": {
          "total": 135925,
          "total_time_in_millis": 2306674
        },
        "warmer": {
          "current": 0,
          "total": 8576200,
          "total_time_in_millis": 3784313
        },
        "query_cache": {
          "memory_size_in_bytes": 47531137,
          "total_count": 315917952,
          "hit_count": 69088483,
          "miss_count": 246829469,
          "cache_size": 16561,
          "cache_count": 740755,
          "evictions": 724194
        },
        "fielddata": {
          "memory_size_in_bytes": 19006048,
          "evictions": 0
        },
        "completion": {
          "size_in_bytes": 0
        },
        "segments": {
          "count": 2256,
          "memory_in_bytes": 62556325,
          "terms_memory_in_bytes": 50154620,
          "stored_fields_memory_in_bytes": 6587256,
          "term_vectors_memory_in_bytes": 0,
          "norms_memory_in_bytes": 557504,
          "points_memory_in_bytes": 2812593,
          "doc_values_memory_in_bytes": 2444352,
          "index_writer_memory_in_bytes": 0,
          "version_map_memory_in_bytes": 0,
          "fixed_bit_set_memory_in_bytes": 6743824,
          "max_unsafe_auto_id_timestamp": -1,
          "file_sizes": {}
        },
        "translog": {
          "operations": 322600,
          "size_in_bytes": 511751430,
          "uncommitted_operations": 22528,
          "uncommitted_size_in_bytes": 35107521
        },
        "request_cache": {
          "memory_size_in_bytes": 84792534,
          "evictions": 13922143,
          "hit_count": 7667542,
          "miss_count": 35693456
        },
        "recovery": {
          "current_as_source": 0,
          "current_as_target": 0,
          "throttle_time_in_millis": 952628
        }
      },
      "os": {
        "timestamp": 1536212516872,
        "cpu": {
          "percent": 23,
          "load_average": {
            "1m": 1.29,
            "5m": 1.19,
            "15m": 1.16
          }
        },
        "mem": {
          "total_in_bytes": 16823488512,
          "free_in_bytes": 297766912,
          "used_in_bytes": 16525721600,
          "free_percent": 2,
          "used_percent": 98
        },
        "swap": {
          "total_in_bytes": 2147479552,
          "free_in_bytes": 2143141888,
          "used_in_bytes": 4337664
        }
      },
      "process": {
        "timestamp": 1536212516872,
        "open_file_descriptors": 2573,
        "max_file_descriptors": 128000,
        "cpu": {
          "percent": 19,
          "total_in_millis": 4177826450
        },
        "mem": {
          "total_virtual_in_bytes": 34029547520
        }
      },
      "jvm": {
        "timestamp": 1536212516874,
        "uptime_in_millis": 6132906547,
        "mem": {
          "heap_used_in_bytes": 5277036968,
          "heap_used_percent": 61,
          "heap_committed_in_bytes": 8555069440,
          "heap_max_in_bytes": 8555069440,
          "non_heap_used_in_bytes": 278172640,
          "non_heap_committed_in_bytes": 304259072,
          "pools": {
            "young": {
              "used_in_bytes": 246875928,
              "max_in_bytes": 279183360,
              "peak_used_in_bytes": 279183360,
              "peak_max_in_bytes": 279183360
            },
            "survivor": {
              "used_in_bytes": 6813256,
              "max_in_bytes": 34865152,
              "peak_used_in_bytes": 34865152,
              "peak_max_in_bytes": 34865152
            },
            "old": {
              "used_in_bytes": 5023347784,
              "max_in_bytes": 8241020928,
              "peak_used_in_bytes": 6966292552,
              "peak_max_in_bytes": 8241020928
            }
          }
        },
        "threads": {
          "count": 234,
          "peak_count": 250
        },
        "gc": {
          "collectors": {
            "young": {
              "collection_count": 4474572,
              "collection_time_in_millis": 127468649
            },
            "old": {
              "collection_count": 920,
              "collection_time_in_millis": 325448
            }
          }
        },
        "buffer_pools": {
          "direct": {
            "count": 463,
            "used_in_bytes": 154976141,
            "total_capacity_in_bytes": 154976139
          },
          "mapped": {
            "count": 4914,
            "used_in_bytes": 21027708516,
            "total_capacity_in_bytes": 21027708516
          }
        },
        "classes": {
          "current_loaded_count": 26154,
          "total_loaded_count": 26712,
          "total_unloaded_count": 558
        }
      },
      "thread_pool": {
        "bulk": {
          "threads": 4,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 4,
          "completed": 17141525
        },
        "fetch_shard_started": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "fetch_shard_store": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "flush": {
          "threads": 2,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 2,
          "completed": 271881
        },
        "force_merge": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "generic": {
          "threads": 32,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 128,
          "completed": 43205055
        },
        "get": {
          "threads": 4,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 4,
          "completed": 8
        },
        "index": {
          "threads": 3,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 3,
          "completed": 3
        },
        "listener": {
          "threads": 2,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 2,
          "completed": 4
        },
        "management": {
          "threads": 5,
          "queue": 0,
          "active": 1,
          "rejected": 0,
          "largest": 5,
          "completed": 28115022
        },
        "refresh": {
          "threads": 2,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 2,
          "completed": 1023804661
        },
        "search": {
          "threads": 7,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 7,
          "completed": 161702240
        },
        "snapshot": {
          "threads": 1,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 2,
          "completed": 21133
        },
        "warmer": {
          "threads": 2,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 2,
          "completed": 95146934
        }
      },
      "fs": {
        "timestamp": 1536212516874,
        "total": {
          "total_in_bytes": 168968957952,
          "free_in_bytes": 147126472704,
          "available_in_bytes": 138519760896
        },
        "data": [
          {
            "type": "ext4",
            "total_in_bytes": 168968957952,
            "free_in_bytes": 147126472704,
            "available_in_bytes": 138519760896
          }
        ],
        "io_stats": {
          "devices": [
            {
              "device_name": "xvdf",
              "operations": 133367878,
              "read_operations": 1456763,
              "write_operations": 131911115,
              "read_kilobytes": 104740824,
              "write_kilobytes": 3178855500
            }
          ],
          "total": {
            "operations": 133367878,
            "read_operations": 1456763,
            "write_operations": 131911115,
            "read_kilobytes": 104740824,
            "write_kilobytes": 3178855500
          }
        }
      },
      "breakers": {
        "request": {
          "limit_size_in_bytes": 5133041664,
          "limit_size": "4.7gb",
          "estimated_size_in_bytes": 0,
          "estimated_size": "0b",
          "overhead": 1,
          "tripped": 0
        },
        "fielddata": {
          "limit_size_in_bytes": 5133041664,
          "limit_size": "4.7gb",
          "estimated_size_in_bytes": 19006048,
          "estimated_size": "18.1mb",
          "overhead": 1.03,
          "tripped": 0
        },
        "in_flight_requests": {
          "limit_size_in_bytes": 8555069440,
          "limit_size": "7.9gb",
          "estimated_size_in_bytes": 1657,
          "estimated_size": "1.6kb",
          "overhead": 1,
          "tripped": 0
        },
        "accounting": {
          "limit_size_in_bytes": 8555069440,
          "limit_size": "7.9gb",
          "estimated_size_in_bytes": 62556325,
          "estimated_size": "59.6mb",
          "overhead": 1,
          "tripped": 0
        },
        "parent": {
          "limit_size_in_bytes": 5988548608,
          "limit_size": "5.5gb",
          "estimated_size_in_bytes": 81564030,
          "estimated_size": "77.7mb",
          "overhead": 1,
          "tripped": 0
        }
      },
      "script": {
        "compilations": 5,
        "cache_evictions": 0
      },
      "discovery": {
        "cluster_state_queue": {
          "total": 0,
          "pending": 0,
          "committed": 0
        },
        "published_cluster_states": {
          "full_states": 1,
          "incompatible_diffs": 0,
          "compatible_diffs": 70303
        }
      },
      "ingest": {
        "total": {
          "count": 0,
          "time_in_millis": 0,
          "current": 0,
          "failed": 0
        },
        "pipelines": {}
      },
      "adaptive_selection": {
        "qhqf-YADRF2gS1Vu6EV_8Q": {
          "outgoing_searches": 0,
          "avg_queue_size": 0,
          "avg_service_time_ns": 6520537,
          "avg_response_time_ns": 15403676,
          "rank": "15.4"
        },
        "I21NiLksQqCNV3-vP3uCNA": {
          "outgoing_searches": 0,
          "avg_queue_size": 0,
          "avg_service_time_ns": 3079216,
          "avg_response_time_ns": 1115616,
          "rank": "1.1"
        },
        "T2_cLfzrQBKd7WTR0p6jWQ": {
          "outgoing_searches": 0,
          "avg_queue_size": 0,
          "avg_service_time_ns": 12493125,
          "avg_response_time_ns": 1141514,
          "rank": "1.1"
        },
        "ZZYgl3WARsKE-80HXhRbVw": {
          "outgoing_searches": 0,
          "avg_queue_size": 0,
          "avg_service_time_ns": 3919880,
          "avg_response_time_ns": 6063593,
          "rank": "6.1"
        }
      }
    }

任何人都可以帮助解释此节点中的内存分配吗?可以只有0.27GB的可用内存空间,还是应该怎么做才能在节点中提供更多可用内存?

0 个答案:

没有答案