es查询报错circuit_breaking_exception","reason":"[parent] Data too large,异常
Elasticsearch | 作者 myc | 发布于2020年07月17日 | 阅读数:10225
查询数据大概在两百万左右,语句如下:
{
"_source": ["_id"],
"query": {
"bool":{
"must": [
{
"match": {
"deleteStatus": "1"
}
}
#if($sourceIp!='' && $sourceIp),
{
"terms": {
"sourceIp": #[sourceIp,serialJson=true]
}
}
#end
#if($purposeIp!='' && $purposeIp),
{
"terms": {
"purposeIp": #[purposeIp,serialJson=true]
}
}
#end
]
}
},
"size": 20000000,
"track_total_hits": true
}
报错如下:
{"error":{"root_cause":[{"type":"circuit_breaking_exception","reason":"[parent] Data too large, data for [<http_request>] would be [8333415566/7.7gb], which is larger than the limit of [8160437862/7.5gb], real usage: [8333414416/7.7gb], new bytes reserved: [1150/1.1kb], usages [request=0/0b, fielddata=89417291/85.2mb, in_flight_requests=72424/70.7kb, accounting=52032679/49.6mb]","bytes_wanted":8333415566,"bytes_limit":8160437862,"durability":"PERMANENT"}],"type":"circuit_breaking_exception","reason":"[parent] Data too large, data for [<http_request>] would be [8333415566/7.7gb], which is larger than the limit of [8160437862/7.5gb], real usage: [8333414416/7.7gb], new bytes reserved: [1150/1.1kb], usages [request=0/0b, fielddata=89417291/85.2mb, in_flight_requests=72424/70.7kb, accounting=52032679/49.6mb]","bytes_wanted":8333415566,"bytes_limit":8160437862,"durability":"PERMANENT"},"status":429}
解决思路:认为是请求时,内存不足,导致的请求熔断,之后采用GET _nodes/stats请求拿到返回数据进行查看
返回数据如下:
{
"_nodes": {
"total": 1,
"successful": 1,
"failed": 0
},
"cluster_name": "ES-Cluster",
"nodes": {
"d9a2cTVrRbCm943bCqZ8Yg": {
"timestamp": 1594897062572,
"name": "node-1",
"transport_address": "10.64.47.207:29300",
"host": "10.64.47.207",
"ip": "10.64.47.207:29300",
"roles": [
"ingest",
"master",
"data"
],
"attributes": {
"ml.machine_memory": "66967392256",
"xpack.installed": "true",
"ml.max_open_jobs": "20"
},
"indices": {
"docs": {
"count": 80230047,
"deleted": 8250
},
"store": {
"size_in_bytes": 66501602593
},
"indexing": {
"index_total": 5917,
"index_time_in_millis": 3616,
"index_current": 0,
"index_failed": 0,
"delete_total": 0,
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"get": {
"total": 0,
"time_in_millis": 0,
"exists_total": 0,
"exists_time_in_millis": 0,
"missing_total": 0,
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 85597,
"query_time_in_millis": 3190705,
"query_current": 0,
"fetch_total": 13696,
"fetch_time_in_millis": 6111290,
"fetch_current": 0,
"scroll_total": 50,
"scroll_time_in_millis": 150,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 66,
"total_time_in_millis": 28030,
"total_docs": 334883,
"total_size_in_bytes": 343508670,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 9619,
"total_auto_throttle_in_bytes": 4230527534
},
"refresh": {
"total": 1131,
"total_time_in_millis": 9375,
"external_total": 1068,
"external_total_time_in_millis": 9783,
"listeners": 0
},
"flush": {
"total": 219,
"periodic": 0,
"total_time_in_millis": 419
},
"warmer": {
"current": 0,
"total": 866,
"total_time_in_millis": 46
},
"query_cache": {
"memory_size_in_bytes": 6114069,
"total_count": 655404,
"hit_count": 33057,
"miss_count": 622347,
"cache_size": 1491,
"cache_count": 1491,
"evictions": 0
},
"fielddata": {
"memory_size_in_bytes": 150090244,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 781,
"memory_in_bytes": 52002851,
"terms_memory_in_bytes": 24570023,
"stored_fields_memory_in_bytes": 20399080,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 1441472,
"points_memory_in_bytes": 4822608,
"doc_values_memory_in_bytes": 769668,
"index_writer_memory_in_bytes": 3738564,
"version_map_memory_in_bytes": 68527,
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": -1,
"file_sizes": {
}
},
"translog": {
"operations": 8354,
"size_in_bytes": 18720589,
"uncommitted_operations": 1331,
"uncommitted_size_in_bytes": 3154180,
"earliest_last_modified_age": 0
},
"request_cache": {
"memory_size_in_bytes": 132918,
"evictions": 0,
"hit_count": 1316,
"miss_count": 2487
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 0
}
},
"os": {
"timestamp": 1594897062682,
"cpu": {
"percent": 0,
"load_average": {
"1m": 0.04,
"5m": 0.21,
"15m": 1.21
}
},
"mem": {
"total_in_bytes": 66967392256,
"free_in_bytes": 7783120896,
"used_in_bytes": 59184271360,
"free_percent": 12,
"used_percent": 88
},
"swap": {
"total_in_bytes": 33621536768,
"free_in_bytes": 33621536768,
"used_in_bytes": 0
},
"cgroup": {
"cpuacct": {
"control_group": "/",
"usage_nanos": 5001563861438754
},
"cpu": {
"control_group": "/",
"cfs_period_micros": 100000,
"cfs_quota_micros": -1,
"stat": {
"number_of_elapsed_periods": 0,
"number_of_times_throttled": 0,
"time_throttled_nanos": 0
}
},
"memory": {
"control_group": "/",
"limit_in_bytes": "9223372036854771712",
"usage_in_bytes": "55168942080"
}
}
},
"process": {
"timestamp": 1594897062682,
"open_file_descriptors": 2470,
"max_file_descriptors": 65536,
"cpu": {
"percent": 0,
"total_in_millis": 17025610
},
"mem": {
"total_virtual_in_bytes": 76974538752
}
},
"jvm": {
"timestamp": 1594897062684,
"uptime_in_millis": 3192457,
"mem": {
"heap_used_in_bytes": 3339824552,
"heap_used_percent": 38,
"heap_committed_in_bytes": 8589934592,
"heap_max_in_bytes": 8589934592,
"non_heap_used_in_bytes": 166834672,
"non_heap_committed_in_bytes": 182095872,
"pools": {
"young": {
"used_in_bytes": 276824064,
"max_in_bytes": 0,
"peak_used_in_bytes": 4303355904,
"peak_max_in_bytes": 0
},
"old": {
"used_in_bytes": 2962337192,
"max_in_bytes": 8589934592,
"peak_used_in_bytes": 8585457664,
"peak_max_in_bytes": 8589934592
},
"survivor": {
"used_in_bytes": 100663296,
"max_in_bytes": 0,
"peak_used_in_bytes": 536870912,
"peak_max_in_bytes": 0
}
}
},
"threads": {
"count": 236,
"peak_count": 241
},
"gc": {
"collectors": {
"young": {
"collection_count": 4295,
"collection_time_in_millis": 267812
},
"old": {
"collection_count": 18,
"collection_time_in_millis": 8075
}
}
},
"buffer_pools": {
"mapped": {
"count": 1219,
"used_in_bytes": 44295384446,
"total_capacity_in_bytes": 44295384446
},
"direct": {
"count": 292,
"used_in_bytes": 1348319977,
"total_capacity_in_bytes": 1348319976
}
},
"classes": {
"current_loaded_count": 18072,
"total_loaded_count": 18800,
"total_unloaded_count": 728
}
},
"thread_pool": {
"analyze": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"ccr": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"data_frame_indexing": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"fetch_shard_started": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 80,
"completed": 202
},
"fetch_shard_store": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"flush": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 5,
"completed": 252
},
"force_merge": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"generic": {
"threads": 12,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 12,
"completed": 12012
},
"get": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"listener": {
"threads": 4,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 4,
"completed": 4
},
"management": {
"threads": 5,
"queue": 0,
"active": 1,
"rejected": 0,
"largest": 5,
"completed": 26971
},
"ml_datafeed": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"ml_job_comms": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"ml_utility": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 1,
"completed": 1
},
"refresh": {
"threads": 10,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 10,
"completed": 18980
},
"rollup_indexing": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"search": {
"threads": 61,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 61,
"completed": 116402
},
"search_throttled": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"security-token-key": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"snapshot": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"warmer": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"watcher": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"write": {
"threads": 40,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 40,
"completed": 595
}
},
"fs": {
"timestamp": 1594897062684,
"total": {
"total_in_bytes": 510356619264,
"free_in_bytes": 436794892288,
"available_in_bytes": 436794892288
},
"least_usage_estimate": {
"path": "/home/axyh/data/elasticsearch/nodes/0",
"total_in_bytes": 510356619264,
"available_in_bytes": 436795650048,
"used_disk_percent": 14.413640666027689
},
"most_usage_estimate": {
"path": "/home/axyh/data/elasticsearch/nodes/0",
"total_in_bytes": 510356619264,
"available_in_bytes": 436795650048,
"used_disk_percent": 14.413640666027689
},
"data": [
{
"path": "/home/axyh/data/elasticsearch/nodes/0",
"mount": "/home (/dev/mapper/cl-home)",
"type": "xfs",
"total_in_bytes": 510356619264,
"free_in_bytes": 436794892288,
"available_in_bytes": 436794892288
}
],
"io_stats": {
"devices": [
{
"device_name": "dm-2",
"operations": 22284,
"read_operations": 110,
"write_operations": 22174,
"read_kilobytes": 5628,
"write_kilobytes": 1420959
}
],
"total": {
"operations": 22284,
"read_operations": 110,
"write_operations": 22174,
"read_kilobytes": 5628,
"write_kilobytes": 1420959
}
}
},
"transport": {
"server_open": 0,
"rx_count": 8,
"rx_size_in_bytes": 1036,
"tx_count": 8,
"tx_size_in_bytes": 1036
},
"http": {
"current_open": 44,
"total_opened": 156
},
"breakers": {
"request": {
"limit_size_in_bytes": 5153960755,
"limit_size": "4.7gb",
"estimated_size_in_bytes": 0,
"estimated_size": "0b",
"overhead": 1,
"tripped": 0
},
"fielddata": {
"limit_size_in_bytes": 3435973836,
"limit_size": "3.1gb",
"estimated_size_in_bytes": 150090244,
"estimated_size": "143.1mb",
"overhead": 1.03,
"tripped": 0
},
"in_flight_requests": {
"limit_size_in_bytes": 8589934592,
"limit_size": "8gb",
"estimated_size_in_bytes": 0,
"estimated_size": "0b",
"overhead": 2,
"tripped": 0
},
"accounting": {
"limit_size_in_bytes": 8589934592,
"limit_size": "8gb",
"estimated_size_in_bytes": 52002851,
"estimated_size": "49.5mb",
"overhead": 1,
"tripped": 0
},
"parent": {
"limit_size_in_bytes": 8160437862,
"limit_size": "7.5gb",
"estimated_size_in_bytes": 3339824552,
"estimated_size": "3.1gb",
"overhead": 1,
"tripped": 80
}
},
"script": {
"compilations": 3,
"cache_evictions": 0,
"compilation_limit_triggered": 0
},
"discovery": {
"cluster_state_queue": {
"total": 0,
"pending": 0,
"committed": 0
},
"published_cluster_states": {
"full_states": 2,
"incompatible_diffs": 0,
"compatible_diffs": 106
}
},
"ingest": {
"total": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
},
"pipelines": {
"xpack_monitoring_6": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"script": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
},
{
"gsub": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
]
},
"xpack_monitoring_7": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
]
}
}
},
"adaptive_selection": {
"d9a2cTVrRbCm943bCqZ8Yg": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 90049721,
"avg_response_time_ns": 79921719,
"rank": "79.9"
}
}
}
}
}经查看发现breakers中并没有超标,在上面的dsl中每几秒会请求一次,但是从节点状态中又查不出是不是内存不够用,故此不知道应该怎么排查,望各位,给予建议
{
"_source": ["_id"],
"query": {
"bool":{
"must": [
{
"match": {
"deleteStatus": "1"
}
}
#if($sourceIp!='' && $sourceIp),
{
"terms": {
"sourceIp": #[sourceIp,serialJson=true]
}
}
#end
#if($purposeIp!='' && $purposeIp),
{
"terms": {
"purposeIp": #[purposeIp,serialJson=true]
}
}
#end
]
}
},
"size": 20000000,
"track_total_hits": true
}
报错如下:
{"error":{"root_cause":[{"type":"circuit_breaking_exception","reason":"[parent] Data too large, data for [<http_request>] would be [8333415566/7.7gb], which is larger than the limit of [8160437862/7.5gb], real usage: [8333414416/7.7gb], new bytes reserved: [1150/1.1kb], usages [request=0/0b, fielddata=89417291/85.2mb, in_flight_requests=72424/70.7kb, accounting=52032679/49.6mb]","bytes_wanted":8333415566,"bytes_limit":8160437862,"durability":"PERMANENT"}],"type":"circuit_breaking_exception","reason":"[parent] Data too large, data for [<http_request>] would be [8333415566/7.7gb], which is larger than the limit of [8160437862/7.5gb], real usage: [8333414416/7.7gb], new bytes reserved: [1150/1.1kb], usages [request=0/0b, fielddata=89417291/85.2mb, in_flight_requests=72424/70.7kb, accounting=52032679/49.6mb]","bytes_wanted":8333415566,"bytes_limit":8160437862,"durability":"PERMANENT"},"status":429}
解决思路:认为是请求时,内存不足,导致的请求熔断,之后采用GET _nodes/stats请求拿到返回数据进行查看
返回数据如下:
{
"_nodes": {
"total": 1,
"successful": 1,
"failed": 0
},
"cluster_name": "ES-Cluster",
"nodes": {
"d9a2cTVrRbCm943bCqZ8Yg": {
"timestamp": 1594897062572,
"name": "node-1",
"transport_address": "10.64.47.207:29300",
"host": "10.64.47.207",
"ip": "10.64.47.207:29300",
"roles": [
"ingest",
"master",
"data"
],
"attributes": {
"ml.machine_memory": "66967392256",
"xpack.installed": "true",
"ml.max_open_jobs": "20"
},
"indices": {
"docs": {
"count": 80230047,
"deleted": 8250
},
"store": {
"size_in_bytes": 66501602593
},
"indexing": {
"index_total": 5917,
"index_time_in_millis": 3616,
"index_current": 0,
"index_failed": 0,
"delete_total": 0,
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"get": {
"total": 0,
"time_in_millis": 0,
"exists_total": 0,
"exists_time_in_millis": 0,
"missing_total": 0,
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 85597,
"query_time_in_millis": 3190705,
"query_current": 0,
"fetch_total": 13696,
"fetch_time_in_millis": 6111290,
"fetch_current": 0,
"scroll_total": 50,
"scroll_time_in_millis": 150,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 66,
"total_time_in_millis": 28030,
"total_docs": 334883,
"total_size_in_bytes": 343508670,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 9619,
"total_auto_throttle_in_bytes": 4230527534
},
"refresh": {
"total": 1131,
"total_time_in_millis": 9375,
"external_total": 1068,
"external_total_time_in_millis": 9783,
"listeners": 0
},
"flush": {
"total": 219,
"periodic": 0,
"total_time_in_millis": 419
},
"warmer": {
"current": 0,
"total": 866,
"total_time_in_millis": 46
},
"query_cache": {
"memory_size_in_bytes": 6114069,
"total_count": 655404,
"hit_count": 33057,
"miss_count": 622347,
"cache_size": 1491,
"cache_count": 1491,
"evictions": 0
},
"fielddata": {
"memory_size_in_bytes": 150090244,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 781,
"memory_in_bytes": 52002851,
"terms_memory_in_bytes": 24570023,
"stored_fields_memory_in_bytes": 20399080,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 1441472,
"points_memory_in_bytes": 4822608,
"doc_values_memory_in_bytes": 769668,
"index_writer_memory_in_bytes": 3738564,
"version_map_memory_in_bytes": 68527,
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": -1,
"file_sizes": {
}
},
"translog": {
"operations": 8354,
"size_in_bytes": 18720589,
"uncommitted_operations": 1331,
"uncommitted_size_in_bytes": 3154180,
"earliest_last_modified_age": 0
},
"request_cache": {
"memory_size_in_bytes": 132918,
"evictions": 0,
"hit_count": 1316,
"miss_count": 2487
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 0
}
},
"os": {
"timestamp": 1594897062682,
"cpu": {
"percent": 0,
"load_average": {
"1m": 0.04,
"5m": 0.21,
"15m": 1.21
}
},
"mem": {
"total_in_bytes": 66967392256,
"free_in_bytes": 7783120896,
"used_in_bytes": 59184271360,
"free_percent": 12,
"used_percent": 88
},
"swap": {
"total_in_bytes": 33621536768,
"free_in_bytes": 33621536768,
"used_in_bytes": 0
},
"cgroup": {
"cpuacct": {
"control_group": "/",
"usage_nanos": 5001563861438754
},
"cpu": {
"control_group": "/",
"cfs_period_micros": 100000,
"cfs_quota_micros": -1,
"stat": {
"number_of_elapsed_periods": 0,
"number_of_times_throttled": 0,
"time_throttled_nanos": 0
}
},
"memory": {
"control_group": "/",
"limit_in_bytes": "9223372036854771712",
"usage_in_bytes": "55168942080"
}
}
},
"process": {
"timestamp": 1594897062682,
"open_file_descriptors": 2470,
"max_file_descriptors": 65536,
"cpu": {
"percent": 0,
"total_in_millis": 17025610
},
"mem": {
"total_virtual_in_bytes": 76974538752
}
},
"jvm": {
"timestamp": 1594897062684,
"uptime_in_millis": 3192457,
"mem": {
"heap_used_in_bytes": 3339824552,
"heap_used_percent": 38,
"heap_committed_in_bytes": 8589934592,
"heap_max_in_bytes": 8589934592,
"non_heap_used_in_bytes": 166834672,
"non_heap_committed_in_bytes": 182095872,
"pools": {
"young": {
"used_in_bytes": 276824064,
"max_in_bytes": 0,
"peak_used_in_bytes": 4303355904,
"peak_max_in_bytes": 0
},
"old": {
"used_in_bytes": 2962337192,
"max_in_bytes": 8589934592,
"peak_used_in_bytes": 8585457664,
"peak_max_in_bytes": 8589934592
},
"survivor": {
"used_in_bytes": 100663296,
"max_in_bytes": 0,
"peak_used_in_bytes": 536870912,
"peak_max_in_bytes": 0
}
}
},
"threads": {
"count": 236,
"peak_count": 241
},
"gc": {
"collectors": {
"young": {
"collection_count": 4295,
"collection_time_in_millis": 267812
},
"old": {
"collection_count": 18,
"collection_time_in_millis": 8075
}
}
},
"buffer_pools": {
"mapped": {
"count": 1219,
"used_in_bytes": 44295384446,
"total_capacity_in_bytes": 44295384446
},
"direct": {
"count": 292,
"used_in_bytes": 1348319977,
"total_capacity_in_bytes": 1348319976
}
},
"classes": {
"current_loaded_count": 18072,
"total_loaded_count": 18800,
"total_unloaded_count": 728
}
},
"thread_pool": {
"analyze": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"ccr": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"data_frame_indexing": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"fetch_shard_started": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 80,
"completed": 202
},
"fetch_shard_store": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"flush": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 5,
"completed": 252
},
"force_merge": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"generic": {
"threads": 12,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 12,
"completed": 12012
},
"get": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"listener": {
"threads": 4,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 4,
"completed": 4
},
"management": {
"threads": 5,
"queue": 0,
"active": 1,
"rejected": 0,
"largest": 5,
"completed": 26971
},
"ml_datafeed": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"ml_job_comms": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"ml_utility": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 1,
"completed": 1
},
"refresh": {
"threads": 10,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 10,
"completed": 18980
},
"rollup_indexing": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"search": {
"threads": 61,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 61,
"completed": 116402
},
"search_throttled": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"security-token-key": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"snapshot": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"warmer": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"watcher": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"write": {
"threads": 40,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 40,
"completed": 595
}
},
"fs": {
"timestamp": 1594897062684,
"total": {
"total_in_bytes": 510356619264,
"free_in_bytes": 436794892288,
"available_in_bytes": 436794892288
},
"least_usage_estimate": {
"path": "/home/axyh/data/elasticsearch/nodes/0",
"total_in_bytes": 510356619264,
"available_in_bytes": 436795650048,
"used_disk_percent": 14.413640666027689
},
"most_usage_estimate": {
"path": "/home/axyh/data/elasticsearch/nodes/0",
"total_in_bytes": 510356619264,
"available_in_bytes": 436795650048,
"used_disk_percent": 14.413640666027689
},
"data": [
{
"path": "/home/axyh/data/elasticsearch/nodes/0",
"mount": "/home (/dev/mapper/cl-home)",
"type": "xfs",
"total_in_bytes": 510356619264,
"free_in_bytes": 436794892288,
"available_in_bytes": 436794892288
}
],
"io_stats": {
"devices": [
{
"device_name": "dm-2",
"operations": 22284,
"read_operations": 110,
"write_operations": 22174,
"read_kilobytes": 5628,
"write_kilobytes": 1420959
}
],
"total": {
"operations": 22284,
"read_operations": 110,
"write_operations": 22174,
"read_kilobytes": 5628,
"write_kilobytes": 1420959
}
}
},
"transport": {
"server_open": 0,
"rx_count": 8,
"rx_size_in_bytes": 1036,
"tx_count": 8,
"tx_size_in_bytes": 1036
},
"http": {
"current_open": 44,
"total_opened": 156
},
"breakers": {
"request": {
"limit_size_in_bytes": 5153960755,
"limit_size": "4.7gb",
"estimated_size_in_bytes": 0,
"estimated_size": "0b",
"overhead": 1,
"tripped": 0
},
"fielddata": {
"limit_size_in_bytes": 3435973836,
"limit_size": "3.1gb",
"estimated_size_in_bytes": 150090244,
"estimated_size": "143.1mb",
"overhead": 1.03,
"tripped": 0
},
"in_flight_requests": {
"limit_size_in_bytes": 8589934592,
"limit_size": "8gb",
"estimated_size_in_bytes": 0,
"estimated_size": "0b",
"overhead": 2,
"tripped": 0
},
"accounting": {
"limit_size_in_bytes": 8589934592,
"limit_size": "8gb",
"estimated_size_in_bytes": 52002851,
"estimated_size": "49.5mb",
"overhead": 1,
"tripped": 0
},
"parent": {
"limit_size_in_bytes": 8160437862,
"limit_size": "7.5gb",
"estimated_size_in_bytes": 3339824552,
"estimated_size": "3.1gb",
"overhead": 1,
"tripped": 80
}
},
"script": {
"compilations": 3,
"cache_evictions": 0,
"compilation_limit_triggered": 0
},
"discovery": {
"cluster_state_queue": {
"total": 0,
"pending": 0,
"committed": 0
},
"published_cluster_states": {
"full_states": 2,
"incompatible_diffs": 0,
"compatible_diffs": 106
}
},
"ingest": {
"total": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
},
"pipelines": {
"xpack_monitoring_6": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"script": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
},
{
"gsub": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
]
},
"xpack_monitoring_7": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
]
}
}
},
"adaptive_selection": {
"d9a2cTVrRbCm943bCqZ8Yg": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 90049721,
"avg_response_time_ns": 79921719,
"rank": "79.9"
}
}
}
}
}经查看发现breakers中并没有超标,在上面的dsl中每几秒会请求一次,但是从节点状态中又查不出是不是内存不够用,故此不知道应该怎么排查,望各位,给予建议
2 个回复
Charele - Cisco4321
赞同来自:
Data too large, data for [<http_request>] would be [8333415566/7.7gb], //Awhich is larger than the limit of [8160437862/7.5gb], //B
real usage: [8333414416/7.7gb], //C
new bytes reserved: [1150/1.1kb] //D
这里有4个数值。
B处的数就是上限,超过这个就报错。(缺省是它是ES最大内存的95%,所以我判断你的-Xmx应该是8g)
C处的数值是你的本机上ES进程已使用的内存大小,
D处的数值1150就是你本次操作(或者说执行当前的任务)所需要内存,
C + D = A > B,所以报错了。
解决方法你可以增大-Xmx量(如果物理内存足够的话)等等。
当然,最省事的做法就是关闭CircuitBreaker检查。
indices.breaker.type: none
pony_maggie - 公众号:犀牛饲养员的技术笔记
赞同来自: