好的想法是十分钱一打,真正无价的是能够实现这些想法的人。

请教es7在大量使用bulk 写入、更新、删除的优化问题

Elasticsearch | 作者 whalewhale | 发布于2020年09月04日 | 阅读数:2431

我使用canal-adapter 1.1.5的syncES模块同步mysql数据到es,执行速度很快,但是发现一个现象应该是不对的,这是head中该索引的信息:

size: 51.0Mi (97.0Mi)
docs: 4,846 (124,774)
 
 
查看索引的状态发现应该是大量的deleted数据导致,请问这种更新、删除频繁的索引有什么优化建议吗?
 
 
附索引状态
 


{"uuid": "D269gAufRvKuLNPs_U8qqQ",
"primaries": {

"docs": {

"count": 8003,
"deleted": 97770

},
"store": {

"size_in_bytes": 83355986

},
"indexing": {

"index_total": 207677,
"index_time_in_millis": 37310,
"index_current": 1,
"index_failed": 0,
"delete_total": 95,
"delete_time_in_millis": 3,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time_in_millis": 0

},
"get": {

"total": 426242,
"time_in_millis": 7679,
"exists_total": 199641,
"exists_time_in_millis": 5802,
"missing_total": 226601,
"missing_time_in_millis": 1877,
"current": 0

},
"search": {

"open_contexts": 0,
"query_total": 33,
"query_time_in_millis": 2,
"query_current": 0,
"fetch_total": 32,
"fetch_time_in_millis": 18,
"fetch_current": 0,
"scroll_total": 0,
"scroll_time_in_millis": 0,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0

},
"merges": {

"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 94,
"total_time_in_millis": 31990,
"total_docs": 1142096,
"total_size_in_bytes": 485085515,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 0,
"total_auto_throttle_in_bytes": 20971520

},
"refresh": {

"total": 865,
"total_time_in_millis": 23412,
"external_total": 859,
"external_total_time_in_millis": 24862,
"listeners": 0

},
"flush": {

"total": 1,
"periodic": 0,
"total_time_in_millis": 131

},
"warmer": {

"current": 0,
"total": 858,
"total_time_in_millis": 18

},
"query_cache": {

"memory_size_in_bytes": 0,
"total_count": 0,
"hit_count": 0,
"miss_count": 0,
"cache_size": 0,
"cache_count": 0,
"evictions": 0

},
"fielddata": {

"memory_size_in_bytes": 0,
"evictions": 0

},
"completion": {

"size_in_bytes": 0

},
"segments": {

"count": 11,
"memory_in_bytes": 250153,
"terms_memory_in_bytes": 158867,
"stored_fields_memory_in_bytes": 57832,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 29312,
"points_memory_in_bytes": 0,
"doc_values_memory_in_bytes": 4142,
"index_writer_memory_in_bytes": 532240,
"version_map_memory_in_bytes": 4160,
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": -1,
"file_sizes": { }

},
"translog": {

"operations": 102493,
"size_in_bytes": 281930598,
"uncommitted_operations": 102493,
"uncommitted_size_in_bytes": 281930598,
"earliest_last_modified_age": 0

},
"request_cache": {

"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 0,
"miss_count": 0

},
"recovery": {

"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 0

}

},
"total": {

"docs": {

"count": 14352,
"deleted": 151758

},
"store": {

"size_in_bytes": 163073396

},
"indexing": {

"index_total": 415325,
"index_time_in_millis": 71300,
"index_current": 1,
"index_failed": 0,
"delete_total": 190,
"delete_time_in_millis": 5,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time_in_millis": 0

},
"get": {

"total": 426242,
"time_in_millis": 7679,
"exists_total": 199641,
"exists_time_in_millis": 5802,
"missing_total": 226601,
"missing_time_in_millis": 1877,
"current": 0

},
"search": {

"open_contexts": 0,
"query_total": 59,
"query_time_in_millis": 4,
"query_current": 0,
"fetch_total": 58,
"fetch_time_in_millis": 32,
"fetch_current": 0,
"scroll_total": 0,
"scroll_time_in_millis": 0,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0

},
"merges": {

"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 105,
"total_time_in_millis": 35570,
"total_docs": 1351947,
"total_size_in_bytes": 568951543,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 0,
"total_auto_throttle_in_bytes": 41943040

},
"refresh": {

"total": 968,
"total_time_in_millis": 27013,
"external_total": 960,
"external_total_time_in_millis": 28571,
"listeners": 0

},
"flush": {

"total": 2,
"periodic": 0,
"total_time_in_millis": 253

},
"warmer": {

"current": 0,
"total": 958,
"total_time_in_millis": 20

},
"query_cache": {

"memory_size_in_bytes": 0,
"total_count": 0,
"hit_count": 0,
"miss_count": 0,
"cache_size": 0,
"cache_count": 0,
"evictions": 0

},
"fielddata": {

"memory_size_in_bytes": 0,
"evictions": 0

},
"completion": {

"size_in_bytes": 0

},
"segments": {

"count": 13,
"memory_in_bytes": 328630,
"terms_memory_in_bytes": 198272,
"stored_fields_memory_in_bytes": 89088,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 36352,
"points_memory_in_bytes": 0,
"doc_values_memory_in_bytes": 4918,
"index_writer_memory_in_bytes": 10691340,
"version_map_memory_in_bytes": 491012,
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": -1,
"file_sizes": { }

},
"translog": {

"operations": 204923,
"size_in_bytes": 563687562,
"uncommitted_operations": 204923,
"uncommitted_size_in_bytes": 563687562,
"earliest_last_modified_age": 0

},
"request_cache": {

"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 0,
"miss_count": 0

},
"recovery": {

"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 0

}

}}


已邀请:

chenyaming

赞同来自:

。。。
 

要回复问题请先登录注册