title: Elasticsearch query speed varies greatly on two nodes with almost the same configuration
body: <p>We have setup two single-node elasticsearch(5.1.1) clusters and indexed almost the same data (about 200G) on them. But when I execute the same query, the speed varies greatly. One is almost twice as fast as the other. Could anybody help me figure out the reason?</p>
  
  <p>Here are some details：
  <strong>The query:</strong></p>
  
  <pre><code>GET /trend/_search
  {  
     "profile":true,
     "size":0,
     "aggs":{  
        "date_hist_aggr":{  
           "date_histogram":{  
              "field":"publish_time",
              "interval":"day",
              "time_zone":"+08:00"
           }
        }
     },
     "query":{  
        "bool":{  
           "filter":[  
              {  
                 "bool":{  
                    "should":[  
                       {  
                          "bool":{  
                             "should":[  
                                {  
                                   "match":{  
                                      "title":{  
                                         "type":"phrase",
                                         "query":"search_text"
                                      }
                                   }
                                },
                                {  
                                   "match":{  
                                      "text":{  
                                         "type":"phrase",
                                         "query":"search_text"
                                      }
                                   }
                                }
                             ],
                             "filter":[  
                                {  
                                   "term":{  
                                      "document_source":{  
                                         "value":"choice_news_full"
                                      }
                                   }
                                }
                             ]
                          }
                       }
  
                       // more queries ....
                    ]
                 }
              }
           ]
        }
     }
  }
  </code></pre>
  
  <p><strong>Profile result of the slower one:</strong></p>
  
  <pre><code>{
    "took": 6990,
    "timed_out": false,
    "_shards": {
      "total": 5,
      "successful": 5,
      "failed": 0
    },
    "hits": {
      "total": 10703,
      "max_score": 0,
      "hits": []
    },
    "profile": {
      "shards": [
        // other shards are similary.
        {
          "id": "[EqbHLX5_SQuKPQ-ZZw7Pmg][trend][1]",
          "searches": [
            {
              "query": [
                {
                  "type": "BoostQuery",
                  "description": "(ConstantScore((title:\"some_text_to_search\" text:\"some_text_to_search\" #document_source:choice_news_full)~1))^0.0",
                  "time": "5132.414299ms",
                  "breakdown": {
                    "score": 0,
                    "build_scorer_count": 34,
                    "match_count": 12149,
                    "create_weight": 1506900476,
                    "next_doc": 834988530,
                    "match": 2783243786,
                    "create_weight_count": 1,
                    "next_doc_count": 12181,
                    "score_count": 0,
                    "build_scorer": 7257142,
                    "advance": 0,
                    "advance_count": 0
                  },
                  "children": [
                    {
                      "type": "BooleanQuery",
                      "description": "(title:\"some_text_to_search\" text:\"some_text_to_search\" #document_source:choice_news_full)~1",
                      "time": "5128.086301ms",
                      "breakdown": {
                        "score": 0,
                        "build_scorer_count": 34,
                        "match_count": 12149,
                        "create_weight": 1506894285,
                        "next_doc": 832868797,
                        "match": 2781145088,
                        "create_weight_count": 1,
                        "next_doc_count": 12181,
                        "score_count": 0,
                        "build_scorer": 7153766,
                        "advance": 0,
                        "advance_count": 0
                      },
                      "children": [
                        {
                          "type": "MultiPhraseQuery",
                          "description": "title:\"some_text_to_search\"",
                          "time": "1123.789370ms",
                          "breakdown": {
                            "score": 0,
                            "build_scorer_count": 34,
                            "match_count": 376,
                            "create_weight": 285785210,
                            "next_doc": 160281857,
                            "match": 673293683,
                            "create_weight_count": 1,
                            "next_doc_count": 402,
                            "score_count": 0,
                            "build_scorer": 3966744,
                            "advance": 460643,
                            "advance_count": 420
                          }
                        },
                        {
                          "type": "MultiPhraseQuery",
                          "description": "text:\"some_text_to_search\"",
                          "time": "3986.896197ms",
                          "breakdown": {
                            "score": 0,
                            "build_scorer_count": 34,
                            "match_count": 11795,
                            "create_weight": 1221076820,
                            "next_doc": 655575011,
                            "match": 2103910865,
                            "create_weight_count": 1,
                            "next_doc_count": 12040,
                            "score_count": 0,
                            "build_scorer": 767627,
                            "advance": 5531401,
                            "advance_count": 10603
                          }
                        },
                        {
                          "type": "TermQuery",
                          "description": "document_source:choice_news_full",
                          "time": "5.050457000ms",
                          "breakdown": {
                            "score": 0,
                            "build_scorer_count": 34,
                            "match_count": 0,
                            "create_weight": 1493,
                            "next_doc": 0,
                            "match": 0,
                            "create_weight_count": 1,
                            "next_doc_count": 0,
                            "score_count": 0,
                            "build_scorer": 233508,
                            "advance": 4792856,
                            "advance_count": 22565
                          }
                        }
                      ]
                    }
                  ]
                }
              ],
              "rewrite_time": 22447,
              "collector": [
                {
                  "name": "CancellableCollector",
                  "reason": "search_cancelled",
                  "time": "3.101963000ms",
                  "children": [
                    {
                      "name": "MultiCollector",
                      "reason": "search_multi",
                      "time": "2.694232000ms",
                      "children": [
                        {
                          "name": "TotalHitCountCollector",
                          "reason": "search_count",
                          "time": "0.2663410000ms"
                        },
                        {
                          "name": "ProfilingAggregator: [org.elasticsearch.search.profile.aggregation.ProfilingAggregator@70caba80]",
                          "reason": "aggregation",
                          "time": "1.778954000ms"
                        }
                      ]
                    }
                  ]
                }
              ]
            }
          ],
          "aggregations": [
            {
              "type": "org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregator",
              "description": "date_hist_aggr",
              "time": "1.326995000ms",
              "breakdown": {
                "reduce": 0,
                "build_aggregation": 265919,
                "build_aggregation_count": 1,
                "initialize": 4584,
                "initialize_count": 1,
                "reduce_count": 0,
                "collect": 1054442,
                "collect_count": 2048
              }
            }
          ]
        }
      ]
    }
  }
  </code></pre>
  
  <p><strong>Profile result of the faster one:</strong></p>
  
  <pre><code>{
    "took": 4622,
    "timed_out": false,
    "_shards": {
      "total": 5,
      "successful": 5,
      "failed": 0
    },
    "hits": {
      "total": 10589,
      "max_score": 0,
      "hits": []
    },
    "profile": {
      "shards": [
        // other shards are similary.
        {
          "id": "[3zJGuDH9TQ-WOLgI2ZKfOA][trend][0]",
          "searches": [
            {
              "query": [
                {
                  "type": "BoostQuery",
                  "description": "(ConstantScore((title:\"some_text_to_search\" text:\"some_text_to_search\" #document_source:choice_news_full)~1))^0.0",
                  "time": "2965.483800ms",
                  "breakdown": {
                    "score": 0,
                    "build_scorer_count": 30,
                    "match_count": 11901,
                    "create_weight": 1548877683,
                    "next_doc": 201685172,
                    "match": 1213525903,
                    "create_weight_count": 1,
                    "next_doc_count": 11919,
                    "score_count": 0,
                    "build_scorer": 1371191,
                    "advance": 0,
                    "advance_count": 0
                  },
                  "children": [
                    {
                      "type": "BooleanQuery",
                      "description": "(title:\"some_text_to_search\" text:\"some_text_to_search\" #document_source:choice_news_full)~1",
                      "time": "2961.314893ms",
                      "breakdown": {
                        "score": 0,
                        "build_scorer_count": 30,
                        "match_count": 11901,
                        "create_weight": 1548873529,
                        "next_doc": 199676450,
                        "match": 1211468384,
                        "create_weight_count": 1,
                        "next_doc_count": 11919,
                        "score_count": 0,
                        "build_scorer": 1272679,
                        "advance": 0,
                        "advance_count": 0
                      },
                      "children": [
                        {
                          "type": "MultiPhraseQuery",
                          "description": "title:\"some_text_to_search\"",
                          "time": "377.6214820ms",
                          "breakdown": {
                            "score": 0,
                            "build_scorer_count": 30,
                            "match_count": 443,
                            "create_weight": 357702051,
                            "next_doc": 9996950,
                            "match": 9403763,
                            "create_weight_count": 1,
                            "next_doc_count": 461,
                            "score_count": 0,
                            "build_scorer": 502110,
                            "advance": 15670,
                            "advance_count": 3
                          }
                        },
                        {
                          "type": "MultiPhraseQuery",
                          "description": "text:\"some_text_to_search\"",
                          "time": "2573.946724ms",
                          "breakdown": {
                            "score": 0,
                            "build_scorer_count": 30,
                            "match_count": 11486,
                            "create_weight": 1191134981,
                            "next_doc": 183139344,
                            "match": 1199396032,
                            "create_weight_count": 1,
                            "next_doc_count": 11747,
                            "score_count": 0,
                            "build_scorer": 196634,
                            "advance": 56460,
                            "advance_count": 9
                          }
                        },
                        {
                          "type": "TermQuery",
                          "description": "document_source:choice_news_full",
                          "time": "3.243158000ms",
                          "breakdown": {
                            "score": 0,
                            "build_scorer_count": 30,
                            "match_count": 0,
                            "create_weight": 1382,
                            "next_doc": 0,
                            "match": 0,
                            "create_weight_count": 1,
                            "next_doc_count": 0,
                            "score_count": 0,
                            "build_scorer": 211093,
                            "advance": 3018723,
                            "advance_count": 11929
                          }
                        }
                      ]
                    }
                  ]
                }
              ],
              "rewrite_time": 10115,
              "collector": [
                {
                  "name": "CancellableCollector",
                  "reason": "search_cancelled",
                  "time": "2.942374000ms",
                  "children": [
                    {
                      "name": "MultiCollector",
                      "reason": "search_multi",
                      "time": "2.540610000ms",
                      "children": [
                        {
                          "name": "TotalHitCountCollector",
                          "reason": "search_count",
                          "time": "0.2359210000ms"
                        },
                        {
                          "name": "ProfilingAggregator: [org.elasticsearch.search.profile.aggregation.ProfilingAggregator@63c09142]",
                          "reason": "aggregation",
                          "time": "1.663939000ms"
                        }
                      ]
                    }
                  ]
                }
              ]
            }
          ],
          "aggregations": [
            {
              "type": "org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregator",
              "description": "date_hist_aggr",
              "time": "1.451517000ms",
              "breakdown": {
                "reduce": 0,
                "build_aggregation": 288404,
                "build_aggregation_count": 1,
                "initialize": 1827,
                "initialize_count": 1,
                "reduce_count": 0,
                "collect": 1159177,
                "collect_count": 2107
              }
            }
          ]
        }
      ]
    }
  </code></pre>