ELK，萌萌哒

大部分节点 cpu 占用较高，是否因为内存不足造成

Elasticsearch | 作者 harry2019 | 发布于2020年05月08日 | 阅读数：3136
分享到：QQ空间新浪微博微信QQ好友印象笔记有道云笔记
背景补充：
软件版本：6.3.2；
centos 7;
使用默认5个分片，2个副本集合，15个数据节点，16c16g；
查询中无聚合查询，排序较多，个别情况出现 cpu 100%，响应延迟；
  86.4% (432.2ms out of 500ms) cpu usage by thread 'elasticsearch[192.x.x.x][search][T#18]'

     2/10 snapshots sharing following 30 elements

       org.apache.lucene.codecs.blocktree.IntersectTermsEnum.next(IntersectTermsEnum.java:497)

       org.apache.lucene.search.MultiTermQueryConstantScoreWrapper$1.rewrite(MultiTermQueryConstantScoreWrapper.java:175)

       org.apache.lucene.search.MultiTermQueryConstantScoreWrapper$1.scorer(MultiTermQueryConstantScoreWrapper.java:207)

       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:113)

       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:329)

       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:329)

       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:329)

       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:295)

       org.apache.lucene.search.Weight.bulkScorer(Weight.java:147)

       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:289)

       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:657)

       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)

       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)

       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:266)

       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)

       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:443)

       org.elasticsearch.action.search.SearchTransportService$7.messageReceived(SearchTransportService.java:400)

       org.elasticsearch.action.search.SearchTransportService$7.messageReceived(SearchTransportService.java:397)

       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:259)

       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)

       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:317)

       org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)

       org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1592)

       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:725)

       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)

       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)

       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)

       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

       java.lang.Thread.run(Thread.java:748)

     2/10 snapshots sharing following 28 elements

       org.apache.lucene.search.MultiTermQueryConstantScoreWrapper$1.scorer(MultiTermQueryConstantScoreWrapper.java:207)

       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:113)

       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:329)

       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:329)

       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:329)

       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:295)

       org.apache.lucene.search.Weight.bulkScorer(Weight.java:147)

       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:289)

       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:657)

       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)

       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)

       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:266)

       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)

       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:443)

       org.elasticsearch.action.search.SearchTransportService$7.messageReceived(SearchTransportService.java:400)

       org.elasticsearch.action.search.SearchTransportService$7.messageReceived(SearchTransportService.java:397)

       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:259)

       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)

       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:317)

       org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)

       org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1592)

       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:725)

       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)

       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)

       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)

       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

       java.lang.Thread.run(Thread.java:748)



::: {192.x.x.x}{eBZz1fGNTWehhXlDDMGqjA}{yIRnwUZxQDCNciPrcxZEGA}{192.x.x.x}{192.x.x.x:9300}{ml.machine_memory=3975241728, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true}

   Hot threads at 2020-05-08T09:02:18.870Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:



::: {192.x.x.x}{gKO8mmwBRTWZP25mO1n4-Q}{NGRK1rAFSx6yvJyKzYxTlA}{192.x.x.x}{192.x.x.x:9300}{ml.machine_memory=16657248256, ml.max_open_jobs=20, xpack.installed=true, ml.enabled=true}

   Hot threads at 2020-05-08T09:02:18.796Z, interval=500ms, busiestThreads=3, ignoreIdleThreads=true:



   95.8% (479.1ms out of 500ms) cpu usage by thread 'elasticsearch[192.x.x.x][search][T#18]'

     10/10 snapshots sharing following 2 elements

       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

       java.lang.Thread.run(Thread.java:748)



   90.2% (450.9ms out of 500ms) cpu usage by thread 'elasticsearch[192.x.x.x][search][T#20]'

     2/10 snapshots sharing following 31 elements

       org.apache.lucene.codecs.blocktree.IntersectTermsEnum._next(IntersectTermsEnum.java:647)

       org.apache.lucene.codecs.blocktree.IntersectTermsEnum.next(IntersectTermsEnum.java:497)

       org.apache.lucene.search.MultiTermQueryConstantScoreWrapper$1.rewrite(MultiTermQueryConstantScoreWrapper.java:175)

       org.apache.lucene.search.MultiTermQueryConstantScoreWrapper$1.scorer(MultiTermQueryConstantScoreWrapper.java:207)

       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:113)

       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:329)

       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:329)

       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:329)

       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:295)

       org.apache.lucene.search.Weight.bulkScorer(Weight.java:147)

       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:289)

       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:657)

       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)

       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)

       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:266)

       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)

       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:443)

       org.elasticsearch.action.search.SearchTransportService$7.messageReceived(SearchTransportService.java:400)

       org.elasticsearch.action.search.SearchTransportService$7.messageReceived(SearchTransportService.java:397)

       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:259)

       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)

       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:317)

       org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)

       org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1592)

       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:725)

       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)

       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)

       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)

       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

       java.lang.Thread.run(Thread.java:748)

     2/10 snapshots sharing following 30 elements

       org.apache.lucene.codecs.blocktree.IntersectTermsEnum.postings(IntersectTermsEnum.java:232)

       org.apache.lucene.search.MultiTermQueryConstantScoreWrapper$1.rewrite(MultiTermQueryConstantScoreWrapper.java:173)

       org.apache.lucene.search.MultiTermQueryConstantScoreWrapper$1.scorer(MultiTermQueryConstantScoreWrapper.java:207)

       org.apache.lucene.search.Weight.scorerSupplier(Weight.java:113)

       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:329)

       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:329)

       org.apache.lucene.search.BooleanWeight.scorerSupplier(BooleanWeight.java:329)

       org.apache.lucene.search.BooleanWeight.scorer(BooleanWeight.java:295)

       org.apache.lucene.search.Weight.bulkScorer(Weight.java:147)

       org.apache.lucene.search.BooleanWeight.bulkScorer(BooleanWeight.java:289)

       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:657)

       org.elasticsearch.search.internal.ContextIndexSearcher.search(ContextIndexSearcher.java:191)

       org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:462)

       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:266)

       org.elasticsearch.search.query.QueryPhase.execute(QueryPhase.java:107)

       org.elasticsearch.search.SearchService.executeQueryPhase(SearchService.java:443)

       org.elasticsearch.action.search.SearchTransportService$7.messageReceived(SearchTransportService.java:400)

       org.elasticsearch.action.search.SearchTransportService$7.messageReceived(SearchTransportService.java:397)

       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:259)

       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)

       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:317)

       org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66)

       org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1592)

       org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:725)

       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)

       org.elasticsearch.common.util.concurrent.TimedRunnable.doRun(TimedRunnable.java:41)

       org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)

       java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)

       java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)

       java.lang.Thread.run(Thread.java:748)
2020-05-08 添加评论

没有找到相关结果

    已邀请:

    byx313 - BLOG:https://www.jianshu.com/u/43fd06f9589c

    赞同来自:

    看样子是查询压力太大了。如果写入压力不大，配置上index.sort 会预先排序好
    2020-05-08 0 3
    分享

    zqc0512 - andy zhou

    赞同来自:

    把分片多搞点。查询可以调优的。
    2020-05-09 0 3
    分享
    为什么被折叠? 0 个回复被折叠

    要回复问题请先登录或注册

    Copyright © 2024 · CC BY-NC-SA 3.0

    本站服务器及带宽由 提供赞助