无论才能、知识多么卓著,如果缺乏热情,则无异纸上画饼充饥,无补于事。

ES内存使用过高

Elasticsearch | 作者 ziyou | 发布于2017年11月06日 | 阅读数:19579

ES的集群的配置:
三台服务器,每台服务器的内存是18G,硬盘800G,
分配ES内存是8G
数据量在白天半小时的数据量是480万,算下来每秒钟会有2660的数据量。ES 集群在运行的时候内存的使用率居高不下,下面是集群的信息:
{
"_nodes" : {
"total" : 3,
"successful" : 3,
"failed" : 0
},
"cluster_name" : "ES",
"timestamp" : 1509951969945,
"status" : "green",
"indices" : {
"count" : 99,
"shards" : {
"total" : 982,
"primaries" : 491,
"replication" : 1.0,
"index" : {
"shards" : {
"min" : 2,
"max" : 10,
"avg" : 9.919191919191919
},
"primaries" : {
"min" : 1,
"max" : 5,
"avg" : 4.959595959595959
},
"replication" : {
"min" : 1.0,
"max" : 1.0,
"avg" : 1.0
}
}
},
"docs" : {
"count" : 1013971510,
"deleted" : 66960
},
"store" : {
"size" : "1.2tb",
"size_in_bytes" : 1373088950761,
"throttle_time" : "0s",
"throttle_time_in_millis" : 0
},
"fielddata" : {
"memory_size" : "452.6mb",
"memory_size_in_bytes" : 474654080,
"evictions" : 0
},
"query_cache" : {
"memory_size" : "619.7mb",
"memory_size_in_bytes" : 649838032,
"total_count" : 3010207,
"hit_count" : 395960,
"miss_count" : 2614247,
"cache_size" : 45228,
"cache_count" : 62364,
"evictions" : 17136
},
"completion" : {
"size" : "0b",
"size_in_bytes" : 0
},
"segments" : {
"count" : 12777,
"memory" : "3.3gb",
"memory_in_bytes" : 3619016398,
"terms_memory" : "2.9gb",
"terms_memory_in_bytes" : 3143711676,
"stored_fields_memory" : "346.6mb",
"stored_fields_memory_in_bytes" : 363456936,
"term_vectors_memory" : "0b",
"term_vectors_memory_in_bytes" : 0,
"norms_memory" : "19.8mb",
"norms_memory_in_bytes" : 20776576,
"points_memory" : "31.6mb",
"points_memory_in_bytes" : 33171794,
"doc_values_memory" : "55.2mb",
"doc_values_memory_in_bytes" : 57899416,
"index_writer_memory" : "86.9mb",
"index_writer_memory_in_bytes" : 91152660,
"version_map_memory" : "703kb",
"version_map_memory_in_bytes" : 719948,
"fixed_bit_set" : "0b",
"fixed_bit_set_memory_in_bytes" : 0,
"max_unsafe_auto_id_timestamp" : 1509928479068,
"file_sizes" : { }
}
},
"nodes" : {
"count" : {
"total" : 3,
"data" : 3,
"coordinating_only" : 0,
"master" : 3,
"ingest" : 3
},
"versions" : [
"5.2.2"
],
"os" : {
"available_processors" : 18,
"allocated_processors" : 18,
"names" : [
{
"name" : "Linux",
"count" : 3
}
],
"mem" : {
"total" : "52.4gb",
"total_in_bytes" : 56268460032,
"free" : "878.2mb",
"free_in_bytes" : 920948736,
"used" : "51.5gb",
"used_in_bytes" : 55347511296,
"free_percent" : 2,
"used_percent" : 98
}
},
"process" : {
"cpu" : {
"percent" : 43
},
"open_file_descriptors" : {
"min" : 963,
"max" : 1071,
"avg" : 1031
}
},
"jvm" : {
"max_uptime" : "4.7d",
"max_uptime_in_millis" : 406672392,
"versions" : [
{
"version" : "1.8.0_131",
"vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
"vm_version" : "25.131-b11",
"vm_vendor" : "Oracle Corporation",
"count" : 3
}
],
"mem" : {
"heap_used" : "12.3gb",
"heap_used_in_bytes" : 13294911944,
"heap_max" : "23.8gb",
"heap_max_in_bytes" : 25612910592
},
"threads" : 283
},
"fs" : {
"total" : "2.3tb",
"total_in_bytes" : 2576948920320,
"free" : "1tb",
"free_in_bytes" : 1178129113088,
"available" : "1tb",
"available_in_bytes" : 1178129113088,
"spins" : "true"
},
"plugins" : [ ],
"network_types" : {
"transport_types" : {
"netty4" : 3
},
"http_types" : {
"netty4" : 3
}
}
}
}
我现在有两个问题:
1、是不是这个配置低了?有那方面可以调优?
2、下面这段,摘自上面的jvm里面,堆内存其实只是用了一般左右,但是总的内存是使用到了98%,ES默认的堆内存是锁死吗?
"mem" : { "heap_used" : "12.3gb", "heap_used_in_bytes" : 13294911944, "heap_max" : "23.8gb", "heap_max_in_bytes" : 25612910592 },
 
已邀请:

zhangg7723

赞同来自: ziyou

清理一下query_cache和fielddata缓存,定时执行段合并优化

ziyou - 一个学习ELK的Java程序员

赞同来自:

关于内存的信息,推荐读一下下面这个社区内的日报
https://elasticsearch.cn/article/32
 

redhat

赞同来自:

把你生成client的代码发下

要回复问题请先登录注册