居然是你

elasticsearch创建索引后无法写入数据

Elasticsearch | 作者 minix | 发布于2020年09月14日 | 阅读数:159

es版本是7.7.1,运行在centos7上
es有3个masterabhk, 14个数据节点。
每个数据节点的硬盘有38T左右,目前最多一个节点只使用了2T。硬盘空间的足够的。
 
集群的设置如下:
GET /_cluster/settings

{
"persistent" : {
"cluster" : {
"routing" : {
"rebalance" : {
"enable" : "all"
},
"allocation" : {
"include" : {
"_ip" : "10.128.108.28"
},
"node_concurrent_recoveries" : "5",
"exclude" : {
"_name" : "10.128.108.28",
"_ip" : "10.128.108.28"
},
"balance" : {
"index" : "0.55f"
},
"total_shards_per_node" : "-1"
}
},
"max_shards_per_node" : "10000"
},
"indices" : {
"breaker" : {
"fielddata" : {
"limit" : "40%"
}
},
"recovery" : {
"max_bytes_per_sec" : "1000mb"
}
},
"xpack" : {
"monitoring" : {
"collection" : {
"enabled" : "true"
}
}
}
},
"transient" : {
"cluster" : {
"routing" : {
"allocation" : {
"include" : {
"_ip" : "10.128.108.28"
},
"enable" : "all"
}
},
"max_shards_per_node" : "10000"
}
}
}

 昨天发现es的状态是red, 索引创建了但数据无法写入索引。
从logstash一直在报下面的错误:
[2020-09-14T15:20:37,184][INFO ][logstash.outputs.elasticsearch] retrying failed action with response code: 503 ({"type"=>"unavailable_shards_exception", "reason"=>"[logstash-jetty-ouc-2020.09.12][0] primary shard is not active Timeout: [1m], request: [BulkShardRequest [[logstash-XXXXXX-2020.09.12][0]] containing [5] requests]"})
而且有多个索引报这错误。试过删除报错的索引,但过一会后依然报这种错误。
通过ES的 explain API得到下面的信息。
GET /_cluster/allocation/explain


{
"index" : "logstash-XXXX-2020.09.12",
"shard" : 1,
"primary" : true,
"current_state" : "unassigned",
"unassigned_info" : {
"reason" : "INDEX_CREATED",
"at" : "2020-09-14T06:46:46.769Z",
"last_allocation_status" : "no"
},
"can_allocate" : "no",
"allocate_explanation" : "cannot allocate because allocation is not permitted to any of the nodes",
"node_allocation_decisions" : [
{
"node_id" : "ttyEqlerTEaVuu7jpnqsgg",
"node_name" : "node1",
"transport_address" : "node1:9300",
"node_attributes" : {
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"weight_ranking" : 1,
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not cluster setting [cluster.routing.allocation.include] filters [_ip:"10.128.108.28"]"""
}
]
},
{
"node_id" : "JAjGqiLKSiqlwzkyrhQXSw",
"node_name" : "node2",
"transport_address" : "node2:9300",
"node_attributes" : {
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"weight_ranking" : 2,
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not cluster setting [cluster.routing.allocation.include] filters [_ip:"10.128.108.28"]"""
}
]
},
{
"node_id" : "w7dmKXr9ST-E6_8f83usWA",
"node_name" : "node3",
"transport_address" : "node3:9300",
"node_attributes" : {
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"weight_ranking" : 3,
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not cluster setting [cluster.routing.allocation.include] filters [_ip:"10.128.108.28"]"""
}
]
},
{
"node_id" : "lH47uf4bQHW8tTMkRcz9-g",
"node_name" : "node4",
"transport_address" : "node4:9300",
"node_attributes" : {
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"weight_ranking" : 4,
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not cluster setting [cluster.routing.allocation.include] filters [_ip:"10.128.108.28"]"""
}
]
},
{
"node_id" : "nnAW_YxlS_uFBgbKIJ6Osg",
"node_name" : "node5",
"transport_address" : "node5:9300",
"node_attributes" : {
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"weight_ranking" : 5,
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not cluster setting [cluster.routing.allocation.include] filters [_ip:"10.128.108.28"]"""
}
]
},
{
"node_id" : "ewADoYRgSdaZy5xEKhQ_gw",
"node_name" : "node6",
"transport_address" : "node6:9300",
"node_attributes" : {
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"weight_ranking" : 6,
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not cluster setting [cluster.routing.allocation.include] filters [_ip:"10.128.108.28"]"""
}
]
},
{
"node_id" : "WCPbdX1oR2eFJ5GlG1AGpQ",
"node_name" : "node7",
"transport_address" : "node7:9300",
"node_attributes" : {
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"weight_ranking" : 7,
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not cluster setting [cluster.routing.allocation.include] filters [_ip:"10.128.108.28"]"""
}
]
},
{
"node_id" : "HYHIGaR1SxmRBewtOt797Q",
"node_name" : "node8",
"transport_address" : "node8:9300",
"node_attributes" : {
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"weight_ranking" : 8,
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not cluster setting [cluster.routing.allocation.include] filters [_ip:"10.128.108.28"]"""
}
]
},
{
"node_id" : "t5qj6EWfSSCPXAD2eONCDg",
"node_name" : "10.128.52.83",
"transport_address" : "10.128.52.83:9300",
"node_attributes" : {
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"weight_ranking" : 9,
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not cluster setting [cluster.routing.allocation.include] filters [_ip:"10.128.108.28"]"""
}
]
},
{
"node_id" : "jiXUUWtVSu2wdgUvlqwuzA",
"node_name" : "node9",
"transport_address" : "node9:9300",
"node_attributes" : {
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"weight_ranking" : 10,
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not cluster setting [cluster.routing.allocation.include] filters [_ip:"10.128.108.28"]"""
}
]
},
{
"node_id" : "wuNzcYveQxap-QBd9d6V2Q",
"node_name" : "node10",
"transport_address" : "node10:9300",
"node_attributes" : {
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"weight_ranking" : 11,
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not cluster setting [cluster.routing.allocation.include] filters [_ip:"10.128.108.28"]"""
}
]
},
{
"node_id" : "XuOcOzKbST-lGucU5sc-Lg",
"node_name" : "node11",
"transport_address" : "node11:9300",
"node_attributes" : {
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"weight_ranking" : 12,
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not cluster setting [cluster.routing.allocation.include] filters [_ip:"10.128.108.28"]"""
}
]
},
{
"node_id" : "0YhfPsxYSGaGks9HQ2mTcw",
"node_name" : "node12",
"transport_address" : "node12:9300",
"node_attributes" : {
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"weight_ranking" : 13,
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node does not cluster setting [cluster.routing.allocation.include] filters [_ip:"10.128.108.28"]"""
}
]
},
{
"node_id" : "gM_g0DftSKukS7JIU76n9A",
"node_name" : "node13",
"transport_address" : "node13:9300",
"node_attributes" : {
"ml.machine_memory" : "134773764096",
"ml.max_open_jobs" : "20",
"xpack.installed" : "true",
"transform.node" : "true"
},
"node_decision" : "no",
"weight_ranking" : 14,
"deciders" : [
{
"decider" : "filter",
"decision" : "NO",
"explanation" : """node matches cluster setting [cluster.routing.allocation.exclude] filters [_name:"10.128.108.28",_ip:"10.128.108.28"]"""
}
]
}
]
}

对于explain API的提示,试过运行
GET /logstash-XXXXX-2020.09.12/_settings
{
"index.routing.allocation.exclude._name": null
}


结果一样没变。
 
请问各位,有遇到这问题吗?
已邀请:

Ombres

赞同来自:

看提示好像这两个参数都有问题。
GET /_cluster/settings先看看集群配置
 
"cluster.routing.allocation.include._name": null 
"cluster.routing.allocation.include._ip": null 
 

zqc0512 - andy zhou

赞同来自:

cluster.routing.allocation.include
cluster.routing.allocation.exclude
你集群有包含 排除节点……
有什么需求么? 这里面的值一样了……

要回复问题请先登录注册