es 偶发查询缓慢

Elasticsearch | 作者 wolaiey1010 | 发布于2019年01月24日 | 阅读数：7105

所有query 都是，时而快时而慢，大部分时间很快，偶发的比例大概5%左右
数据量 300w
机器配置：

服务器类型机型:物理机
OS:CentOS release 6.6 (Final)
内存:128GB
CPU：12 × 2 cores
硬盘：6 × 300 GB
机械硬盘

es 启动参数：
rg.elasticsearch.bootstrap.Elasticsearch -d -Xms31g -Xmx31g -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+AlwaysPreTouch -Xss1m -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djna.nosys=true -XX:-OmitStackTraceInFastThrow -Dio.netty.noUnsafe=true -Dio.netty.noKeySetOptimization=true -Dio.netty.recycler.maxCapacityPerThread=0 -Dlog4j.shutdownHookEnabled=false -Dlog4j2.disable.jmx=true -Djava.io.tmpdir=/tmp/elasticsearch.kJHoxXhA -XX:+HeapDumpOnOutOfMemoryError -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -Xloggc:logs/gc.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=32 -XX:GCLogFileSize=64m -Des.path.home=/opt/soft/elasticsearch/current -Des.path.conf=/opt/soft/elasticsearch/current/config

eg：
很简单的一个get 查询
http://10.126.101.91:9200/dbww ... earch

慢的时候 800毫秒

快的时候8毫秒

15 个回复

wolaiey1010

赞同来自: derobukal 、battlescars

@all, 最后升级 ssd 硬盘问题解决（原来是hdd 的机械盘）

elasticStack - 90后it大数据男

dsl呢？你给了一个内网地址？？

weizijun - elasticsearch fan

是不是你的jvm配置太小，gc导致的抖动？

wolaiey1010

gc 情况如下：

kennywu76 - Wood

打开ES的GC log，捕捉一下耗时高的GC日志，看是否偶尔有数百秒的GC停顿。因为你的heap划的比较大，有31GB，又是用的默认的CMS GC，在查询量比较多的时候，有可能出现长时间的GC停顿。我们线上遇到过类似的问题，改用G1以后这种偶发的长时间停顿几乎就看不到了。

wolaiey1010

@kennywu76
gc日志如下：

zqc0512 - andy zhou

change to G1GC and try CMS in 31G JVM as my test is not good .
also use the croding node
as the flow setting in elasticsearch.yml file

node.master: false

node.data: false

node.ingest: false

search.remote.connect: false

laoyang360 - 《一本书讲透Elasticsearch》作者，Elastic认证工程师 [死磕Elasitcsearch]知识星球地址：http://t.cn/RmwM3N9；微信公众号：铭毅天下; 博客：https://elastic.blog.csdn.net

看到各位大佬别的维度分析的，查询维度建议发一下dsl

wolaiey1010

@zqc0512 ,
做了分离，集群共7台，分出2台作为查询客户端，上线后，效果不大

wolaiey1010

以下是几个 query Dsl，

{

  "from":0,

  "size":10,

  "query": {

      "bool": {

      "filter": [

        {"term":{"review_status":1}},

        {"term":{"delete_flag":1}},

        {"terms":{"basis_id" :[109641688,102109196]}}

      ]

    }

  }

}

{

   "from":0,

   "size":3,

   "query":{

     "function_score": {

       "query": {

          "bool": {

              "filter": [

                {"term":{"tags":"全部"}},

                {"term":{"tags":"好评"}},

                {"term":{"b_user_id":11188980363527}},

                {"term":{"cate_id":1060}},

                {"term":{"type":1}},

                {"term":{"review_status":1}},

                {"term":{"delete_flag":1}}

              ],

              "must_not" : [

                {"term":{"appeal_state":2}}

                ,{"terms":{"id":[]}}

              ]

            }

        },

         "functions":[

              {"gauss": { "comment_time": { "scale": "45d", "offset": "0", "decay" : 0.5 } }},

  {"field_value_factor": { "field": "sort" ,"factor": 0.266666666 }}

         ],

         "boost_mode": "replace",

         "score_mode": "sum"

     }

   },

    "sort" : [

        {"comment_score" : {"order" : "desc"}},

        {"_score" : {"order" : "desc"}},

        {"comment_time" : {"order" : "desc"}}

    ]

}

{

 "from":0,

 "size":3,

 "query": {

    "bool": {

    "filter": [

      {"term":{"tags":"全部"}},

      {"term":{"b_user_id":58188497000469}},

      {"term":{"cate_id":252}},

      {"range":{"comment_time":{"gte":"2018-11-16 20:36:43.534"}}},

      {"term":{"is_chosen":1}},

      {"term":{"type":1}},

      {"term":{"review_status":1}},

      {"term":{"delete_flag":1}}

    ],

    "must_not" :[

      {"term":{"appeal_state":2}}



    ]

}

  },

  "sort" : [

      {"comment_time" : {"order" : "desc"}}

  ]

}

wolaiey1010

mapping 如下：

{

  "dynamic": "strict",

  "properties": {

    "id": {

      "type": "long"

    },

    "basis_id": {

      "type": "long"

    },

    "user_id": {

      "type": "long"

    },

    "info_id": {

      "type": "long"

    },

    "b_user_id": {

      "type": "long"

    },

    "commodity_id": {

      "type": "long"

    },

    "comment_content": {

      "type": "text",

      "analyzer":"ik_smart",

      "fields": {"raw": {"type": "keyword"}}

    },

    "comment_score": {

      "type": "byte"

    },

    "comment_label": {

      "type": "text",

      "analyzer":"ik_smart",

      "fields": {"raw": {"type": "keyword"}}

    },

    "comment_pictures": {

      "type": "keyword"

    },

    "comment_time": {

      "type": "date",

       "format": "yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"

    },

    "comment_add_content":{

        "type":"text",

        "analyzer":"ik_smart",

        "fields": {"raw": {"type": "keyword"}}

    },

    "comment_add_time":{

        "type": "date",

        "format": "yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"

    },

    "comment_reply_content":{

         "type":"text",

         "analyzer":"ik_smart",

         "fields": {"raw": {"type": "keyword"}}

    },

    "comment_reply_datetime":{

        "type": "date",

        "format": "yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"

    },

    "comment_reply_add_content":{

        "type":"text",

        "analyzer":"ik_smart",

        "fields": {"raw": {"type": "keyword"}}

    },

    "comment_reply_add_datetime":{

         "type": "date",

         "format": "yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"

    },

    "comment_source": {

      "type": "byte"

    },

    "delete_flag": {

      "type": "byte"

    },

    "referer": {

      "type": "keyword"

    },

    "review_status": {

      "type": "byte"

    },

    "review_reason": {

      "type": "text",

      "analyzer":"ik_smart",

      "fields": {"raw": {"type": "keyword"}}

    },

    "type": {

      "type": "byte"

    },

    "be_commented_obj_id": {

      "type": "long"

    },

    "cate_id": {

      "type": "integer"

    },

    "appeal_state": {

      "type": "byte"

    },

    "is_comment_reply": {

      "type": "byte"

    },

    "update_time": {

      "type": "date",

       "format": "yyyy-MM-dd HH:mm:ss.SSS||yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"

    },

    "mobile": {

      "type": "keyword"

    },

    "service_status": {

      "type": "byte"

    },

     "tags":{

        "type":"keyword"

    },

    "is_chosen": {

     "type": "byte"

    },

    "sort": {

     "type": "double"

    },

    "comp_score": {

      "type": "keyword"

    },

    "is_recommend": {

      "type": "byte"

    },

    "price": {

      "type": "double"

    },

    "evaluation_type": {

       "type": "byte"

    }

  }

}

lsyoung

是不是查询任务并发多？queue怎么设置的？

zqc0512 - andy zhou

@wolaiey1010 你没有用负载均衡节点啊。用的的话node.role那里应该是- 你用的ingest节点…… date master ingest都分开测试下。

xiaowoniu - java工程师

原来用的云主机吧？隔离做的不好，io负载高

dy.huang@hotmail.com

楼主解决问题了吗？

要回复问题请先登录或注册

es 偶发查询缓慢

15 个回复

发起人

相关问题

问题状态

es 偶发查询缓慢

与内容相关的链接

15 个回复

发起人

相关问题

问题状态