绊脚石乃是进身之阶。

es查询报错circuit_breaking_exception","reason":"[parent] Data too large,异常

Elasticsearch | 作者 myc | 发布于2020年07月17日 | 阅读数:9528

查询数据大概在两百万左右,语句如下:
{
        "_source": ["_id"],
            "query": {
              "bool":{
                "must": [
                  {
                     "match": {
                  "deleteStatus": "1"
                }
                  }
                  #if($sourceIp!='' && $sourceIp),
                   {
                  "terms": {
                        "sourceIp": #[sourceIp,serialJson=true]
                     }
                }
                #end
                #if($purposeIp!='' && $purposeIp),
                   {
                  "terms": {
                        "purposeIp": #[purposeIp,serialJson=true]
                     }
                }
                #end
                ]
              }
                
            },
             "size": 20000000,
            "track_total_hits": true
            }
报错如下:
{"error":{"root_cause":[{"type":"circuit_breaking_exception","reason":"[parent] Data too large, data for [<http_request>] would be [8333415566/7.7gb], which is larger than the limit of [8160437862/7.5gb], real usage: [8333414416/7.7gb], new bytes reserved: [1150/1.1kb], usages [request=0/0b, fielddata=89417291/85.2mb, in_flight_requests=72424/70.7kb, accounting=52032679/49.6mb]","bytes_wanted":8333415566,"bytes_limit":8160437862,"durability":"PERMANENT"}],"type":"circuit_breaking_exception","reason":"[parent] Data too large, data for [<http_request>] would be [8333415566/7.7gb], which is larger than the limit of [8160437862/7.5gb], real usage: [8333414416/7.7gb], new bytes reserved: [1150/1.1kb], usages [request=0/0b, fielddata=89417291/85.2mb, in_flight_requests=72424/70.7kb, accounting=52032679/49.6mb]","bytes_wanted":8333415566,"bytes_limit":8160437862,"durability":"PERMANENT"},"status":429}
 
解决思路:认为是请求时,内存不足,导致的请求熔断,之后采用GET _nodes/stats请求拿到返回数据进行查看
返回数据如下:
{
  "_nodes": {
    "total": 1,
    "successful": 1,
    "failed": 0
  },
  "cluster_name": "ES-Cluster",
  "nodes": {
    "d9a2cTVrRbCm943bCqZ8Yg": {
      "timestamp": 1594897062572,
      "name": "node-1",
      "transport_address": "10.64.47.207:29300",
      "host": "10.64.47.207",
      "ip": "10.64.47.207:29300",
      "roles": [
        "ingest",
        "master",
        "data"
      ],
      "attributes": {
        "ml.machine_memory": "66967392256",
        "xpack.installed": "true",
        "ml.max_open_jobs": "20"
      },
      "indices": {
        "docs": {
          "count": 80230047,
          "deleted": 8250
        },
        "store": {
          "size_in_bytes": 66501602593
        },
        "indexing": {
          "index_total": 5917,
          "index_time_in_millis": 3616,
          "index_current": 0,
          "index_failed": 0,
          "delete_total": 0,
          "delete_time_in_millis": 0,
          "delete_current": 0,
          "noop_update_total": 0,
          "is_throttled": false,
          "throttle_time_in_millis": 0
        },
        "get": {
          "total": 0,
          "time_in_millis": 0,
          "exists_total": 0,
          "exists_time_in_millis": 0,
          "missing_total": 0,
          "missing_time_in_millis": 0,
          "current": 0
        },
        "search": {
          "open_contexts": 0,
          "query_total": 85597,
          "query_time_in_millis": 3190705,
          "query_current": 0,
          "fetch_total": 13696,
          "fetch_time_in_millis": 6111290,
          "fetch_current": 0,
          "scroll_total": 50,
          "scroll_time_in_millis": 150,
          "scroll_current": 0,
          "suggest_total": 0,
          "suggest_time_in_millis": 0,
          "suggest_current": 0
        },
        "merges": {
          "current": 0,
          "current_docs": 0,
          "current_size_in_bytes": 0,
          "total": 66,
          "total_time_in_millis": 28030,
          "total_docs": 334883,
          "total_size_in_bytes": 343508670,
          "total_stopped_time_in_millis": 0,
          "total_throttled_time_in_millis": 9619,
          "total_auto_throttle_in_bytes": 4230527534
        },
        "refresh": {
          "total": 1131,
          "total_time_in_millis": 9375,
          "external_total": 1068,
          "external_total_time_in_millis": 9783,
          "listeners": 0
        },
        "flush": {
          "total": 219,
          "periodic": 0,
          "total_time_in_millis": 419
        },
        "warmer": {
          "current": 0,
          "total": 866,
          "total_time_in_millis": 46
        },
        "query_cache": {
          "memory_size_in_bytes": 6114069,
          "total_count": 655404,
          "hit_count": 33057,
          "miss_count": 622347,
          "cache_size": 1491,
          "cache_count": 1491,
          "evictions": 0
        },
        "fielddata": {
          "memory_size_in_bytes": 150090244,
          "evictions": 0
        },
        "completion": {
          "size_in_bytes": 0
        },
        "segments": {
          "count": 781,
          "memory_in_bytes": 52002851,
          "terms_memory_in_bytes": 24570023,
          "stored_fields_memory_in_bytes": 20399080,
          "term_vectors_memory_in_bytes": 0,
          "norms_memory_in_bytes": 1441472,
          "points_memory_in_bytes": 4822608,
          "doc_values_memory_in_bytes": 769668,
          "index_writer_memory_in_bytes": 3738564,
          "version_map_memory_in_bytes": 68527,
          "fixed_bit_set_memory_in_bytes": 0,
          "max_unsafe_auto_id_timestamp": -1,
          "file_sizes": {
            
          }
        },
        "translog": {
          "operations": 8354,
          "size_in_bytes": 18720589,
          "uncommitted_operations": 1331,
          "uncommitted_size_in_bytes": 3154180,
          "earliest_last_modified_age": 0
        },
        "request_cache": {
          "memory_size_in_bytes": 132918,
          "evictions": 0,
          "hit_count": 1316,
          "miss_count": 2487
        },
        "recovery": {
          "current_as_source": 0,
          "current_as_target": 0,
          "throttle_time_in_millis": 0
        }
      },
      "os": {
        "timestamp": 1594897062682,
        "cpu": {
          "percent": 0,
          "load_average": {
            "1m": 0.04,
            "5m": 0.21,
            "15m": 1.21
          }
        },
        "mem": {
          "total_in_bytes": 66967392256,
          "free_in_bytes": 7783120896,
          "used_in_bytes": 59184271360,
          "free_percent": 12,
          "used_percent": 88
        },
        "swap": {
          "total_in_bytes": 33621536768,
          "free_in_bytes": 33621536768,
          "used_in_bytes": 0
        },
        "cgroup": {
          "cpuacct": {
            "control_group": "/",
            "usage_nanos": 5001563861438754
          },
          "cpu": {
            "control_group": "/",
            "cfs_period_micros": 100000,
            "cfs_quota_micros": -1,
            "stat": {
              "number_of_elapsed_periods": 0,
              "number_of_times_throttled": 0,
              "time_throttled_nanos": 0
            }
          },
          "memory": {
            "control_group": "/",
            "limit_in_bytes": "9223372036854771712",
            "usage_in_bytes": "55168942080"
          }
        }
      },
      "process": {
        "timestamp": 1594897062682,
        "open_file_descriptors": 2470,
        "max_file_descriptors": 65536,
        "cpu": {
          "percent": 0,
          "total_in_millis": 17025610
        },
        "mem": {
          "total_virtual_in_bytes": 76974538752
        }
      },
      "jvm": {
        "timestamp": 1594897062684,
        "uptime_in_millis": 3192457,
        "mem": {
          "heap_used_in_bytes": 3339824552,
          "heap_used_percent": 38,
          "heap_committed_in_bytes": 8589934592,
          "heap_max_in_bytes": 8589934592,
          "non_heap_used_in_bytes": 166834672,
          "non_heap_committed_in_bytes": 182095872,
          "pools": {
            "young": {
              "used_in_bytes": 276824064,
              "max_in_bytes": 0,
              "peak_used_in_bytes": 4303355904,
              "peak_max_in_bytes": 0
            },
            "old": {
              "used_in_bytes": 2962337192,
              "max_in_bytes": 8589934592,
              "peak_used_in_bytes": 8585457664,
              "peak_max_in_bytes": 8589934592
            },
            "survivor": {
              "used_in_bytes": 100663296,
              "max_in_bytes": 0,
              "peak_used_in_bytes": 536870912,
              "peak_max_in_bytes": 0
            }
          }
        },
        "threads": {
          "count": 236,
          "peak_count": 241
        },
        "gc": {
          "collectors": {
            "young": {
              "collection_count": 4295,
              "collection_time_in_millis": 267812
            },
            "old": {
              "collection_count": 18,
              "collection_time_in_millis": 8075
            }
          }
        },
        "buffer_pools": {
          "mapped": {
            "count": 1219,
            "used_in_bytes": 44295384446,
            "total_capacity_in_bytes": 44295384446
          },
          "direct": {
            "count": 292,
            "used_in_bytes": 1348319977,
            "total_capacity_in_bytes": 1348319976
          }
        },
        "classes": {
          "current_loaded_count": 18072,
          "total_loaded_count": 18800,
          "total_unloaded_count": 728
        }
      },
      "thread_pool": {
        "analyze": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "ccr": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "data_frame_indexing": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "fetch_shard_started": {
          "threads": 1,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 80,
          "completed": 202
        },
        "fetch_shard_store": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "flush": {
          "threads": 1,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 5,
          "completed": 252
        },
        "force_merge": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "generic": {
          "threads": 12,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 12,
          "completed": 12012
        },
        "get": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "listener": {
          "threads": 4,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 4,
          "completed": 4
        },
        "management": {
          "threads": 5,
          "queue": 0,
          "active": 1,
          "rejected": 0,
          "largest": 5,
          "completed": 26971
        },
        "ml_datafeed": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "ml_job_comms": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "ml_utility": {
          "threads": 1,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 1,
          "completed": 1
        },
        "refresh": {
          "threads": 10,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 10,
          "completed": 18980
        },
        "rollup_indexing": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "search": {
          "threads": 61,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 61,
          "completed": 116402
        },
        "search_throttled": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "security-token-key": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "snapshot": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "warmer": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "watcher": {
          "threads": 0,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 0,
          "completed": 0
        },
        "write": {
          "threads": 40,
          "queue": 0,
          "active": 0,
          "rejected": 0,
          "largest": 40,
          "completed": 595
        }
      },
      "fs": {
        "timestamp": 1594897062684,
        "total": {
          "total_in_bytes": 510356619264,
          "free_in_bytes": 436794892288,
          "available_in_bytes": 436794892288
        },
        "least_usage_estimate": {
          "path": "/home/axyh/data/elasticsearch/nodes/0",
          "total_in_bytes": 510356619264,
          "available_in_bytes": 436795650048,
          "used_disk_percent": 14.413640666027689
        },
        "most_usage_estimate": {
          "path": "/home/axyh/data/elasticsearch/nodes/0",
          "total_in_bytes": 510356619264,
          "available_in_bytes": 436795650048,
          "used_disk_percent": 14.413640666027689
        },
        "data": [
          {
            "path": "/home/axyh/data/elasticsearch/nodes/0",
            "mount": "/home (/dev/mapper/cl-home)",
            "type": "xfs",
            "total_in_bytes": 510356619264,
            "free_in_bytes": 436794892288,
            "available_in_bytes": 436794892288
          }
        ],
        "io_stats": {
          "devices": [
            {
              "device_name": "dm-2",
              "operations": 22284,
              "read_operations": 110,
              "write_operations": 22174,
              "read_kilobytes": 5628,
              "write_kilobytes": 1420959
            }
          ],
          "total": {
            "operations": 22284,
            "read_operations": 110,
            "write_operations": 22174,
            "read_kilobytes": 5628,
            "write_kilobytes": 1420959
          }
        }
      },
      "transport": {
        "server_open": 0,
        "rx_count": 8,
        "rx_size_in_bytes": 1036,
        "tx_count": 8,
        "tx_size_in_bytes": 1036
      },
      "http": {
        "current_open": 44,
        "total_opened": 156
      },
      "breakers": {
        "request": {
          "limit_size_in_bytes": 5153960755,
          "limit_size": "4.7gb",
          "estimated_size_in_bytes": 0,
          "estimated_size": "0b",
          "overhead": 1,
          "tripped": 0
        },
        "fielddata": {
          "limit_size_in_bytes": 3435973836,
          "limit_size": "3.1gb",
          "estimated_size_in_bytes": 150090244,
          "estimated_size": "143.1mb",
          "overhead": 1.03,
          "tripped": 0
        },
        "in_flight_requests": {
          "limit_size_in_bytes": 8589934592,
          "limit_size": "8gb",
          "estimated_size_in_bytes": 0,
          "estimated_size": "0b",
          "overhead": 2,
          "tripped": 0
        },
        "accounting": {
          "limit_size_in_bytes": 8589934592,
          "limit_size": "8gb",
          "estimated_size_in_bytes": 52002851,
          "estimated_size": "49.5mb",
          "overhead": 1,
          "tripped": 0
        },
        "parent": {
          "limit_size_in_bytes": 8160437862,
          "limit_size": "7.5gb",
          "estimated_size_in_bytes": 3339824552,
          "estimated_size": "3.1gb",
          "overhead": 1,
          "tripped": 80
        }
      },
      "script": {
        "compilations": 3,
        "cache_evictions": 0,
        "compilation_limit_triggered": 0
      },
      "discovery": {
        "cluster_state_queue": {
          "total": 0,
          "pending": 0,
          "committed": 0
        },
        "published_cluster_states": {
          "full_states": 2,
          "incompatible_diffs": 0,
          "compatible_diffs": 106
        }
      },
      "ingest": {
        "total": {
          "count": 0,
          "time_in_millis": 0,
          "current": 0,
          "failed": 0
        },
        "pipelines": {
          "xpack_monitoring_6": {
            "count": 0,
            "time_in_millis": 0,
            "current": 0,
            "failed": 0,
            "processors": [
              {
                "script": {
                  "count": 0,
                  "time_in_millis": 0,
                  "current": 0,
                  "failed": 0
                }
              },
              {
                "gsub": {
                  "count": 0,
                  "time_in_millis": 0,
                  "current": 0,
                  "failed": 0
                }
              }
            ]
          },
          "xpack_monitoring_7": {
            "count": 0,
            "time_in_millis": 0,
            "current": 0,
            "failed": 0,
            "processors": [
              
            ]
          }
        }
      },
      "adaptive_selection": {
        "d9a2cTVrRbCm943bCqZ8Yg": {
          "outgoing_searches": 0,
          "avg_queue_size": 0,
          "avg_service_time_ns": 90049721,
          "avg_response_time_ns": 79921719,
          "rank": "79.9"
        }
      }
    }
  }
}经查看发现breakers中并没有超标,在上面的dsl中每几秒会请求一次,但是从节点状态中又查不出是不是内存不够用,故此不知道应该怎么排查,望各位,给予建议
 
 
已邀请:

Charele - Cisco4321

赞同来自:

出错这种错误的确是内存方面的原因,来看下,
 
Data too large, data for [<http_request>] would be [8333415566/7.7gb], //Awhich is larger than the limit of [8160437862/7.5gb], //B
real usage: [8333414416/7.7gb], //C
new bytes reserved: [1150/1.1kb] //D
 
这里有4个数值。
B处的数就是上限,超过这个就报错。(缺省是它是ES最大内存的95%,所以我判断你的-Xmx应该是8g)
C处的数值是你的本机上ES进程已使用的内存大小,
D处的数值1150就是你本次操作(或者说执行当前的任务)所需要内存,
C + D = A > B,所以报错了。
 
解决方法你可以增大-Xmx量(如果物理内存足够的话)等等。
当然,最省事的做法就是关闭CircuitBreaker检查。
indices.breaker.type: none
 
 
 
 
 
 
  
 

pony_maggie - 公众号:犀牛饲养员的技术笔记

赞同来自:

从监控看下平时的查询一般占用多少堆内存。

要回复问题请先登录注册