docstr: target: test scale out querynode when load multi replicas
  method: 1.Deploy cluster with 5 querynodes
          2.Create collection with 2 shards
          3.Insert 10 segments and flushed
          4.Load collection with 2 replicas
          5.Scale out querynode from 5 to 6 while search and insert growing data
  expected: Verify search succ rate is 100%
docstr_tok: ["target",":","test","scale","out","querynode","when","load","multi","replicas","method",":","1.Deploy","cluster","with","5","querynodes","2.Create","collection","with","2","shards","3.Insert","10","segments","and","flushed","4.Load","collection","with","2","replicas","5.Scale","out","querynode","from","5","to","6","while","search","and","insert","growing","data","expected",":","Verify","search","succ","rate","is","100","%"]
code: def test_scale_query_node_replicas(self):
      """
      target: test scale out querynode when load multi replicas
      method: 1.Deploy cluster with 5 querynodes
              2.Create collection with 2 shards
              3.Insert 10 segments and flushed
              4.Load collection with 2 replicas
              5.Scale out querynode from 5 to 6 while search and insert growing data
      expected: Verify search succ rate is 100%
      """
      release_name = "scale-replica"
      image_tag = get_latest_tag()
      image = f"{constants.IMAGE_REPOSITORY}:{image_tag}"
      query_config = {
          "metadata.namespace": constants.NAMESPACE,
          "metadata.name": release_name,
          "spec.mode": "cluster",
          "spec.components.image": image,
          "spec.components.proxy.serviceType": "LoadBalancer",
          "spec.components.queryNode.replicas": 5,
          "spec.config.common.retentionDuration": 60,
      }
      mic = MilvusOperator()
      mic.install(query_config)
      if mic.wait_for_healthy(
          release_name, constants.NAMESPACE, timeout=1800
      ):
          host = mic.endpoint(release_name, constants.NAMESPACE).split(
              ":"
          )[0]
      else:
          raise MilvusException(message=f"Milvus healthy timeout 1800s")
  
      try:
          scale_querynode = random.choice([6, 7, 4, 3])
          connections.connect("scale-replica", host=host, port=19530)
  
          collection_w = ApiCollectionWrapper()
          collection_w.init_collection(
              name=cf.gen_unique_str("scale_out"),
              schema=cf.gen_default_collection_schema(),
              using="scale-replica",
              shards_num=3,
          )
  
          # insert 10 sealed segments
          for i in range(5):
              df = cf.gen_default_dataframe_data(nb=nb, start=i * nb)
              collection_w.insert(df)
              assert collection_w.num_entities == (i + 1) * nb
  
          collection_w.load(replica_number=2)
  
          @counter
          def do_search():
              """do search"""
              search_res, is_succ = collection_w.search(
                  cf.gen_vectors(1, ct.default_dim),
                  ct.default_float_vec_field_name,
                  ct.default_search_params,
                  ct.default_limit,
                  check_task=CheckTasks.check_nothing,
              )
              assert len(search_res) == 1
              return search_res, is_succ
  
          def loop_search():
              """continuously search"""
              while True:
                  do_search()
  
          threading.Thread(
              target=loop_search, args=(), daemon=True
          ).start()
  
          # scale out
          mic.upgrade(
              release_name,
              {"spec.components.queryNode.replicas": scale_querynode},
              constants.NAMESPACE,
          )
          mic.wait_for_healthy(release_name, constants.NAMESPACE)
          wait_pods_ready(
              constants.NAMESPACE,
              f"app.kubernetes.io/instance={release_name}",
          )
          log.debug("Scale out querynode success")
  
          time.sleep(100)
          scale_common.check_succ_rate(do_search)
          log.debug("Scale out test finished")
  
      except Exception as e:
          raise Exception(str(e))
  
      finally:
          label = f"app.kubernetes.io/instance={release_name}"
          log.info("Start to export milvus pod logs")
          read_pod_log(
              namespace=constants.NAMESPACE,
              label_selector=label,
              release_name=release_name,
          )
          mic.uninstall(release_name, namespace=constants.NAMESPACE)
  
imports: import random
  import threading
  import time
  import pytest
  from base.collection_wrapper import ApiCollectionWrapper
  from base.utility_wrapper import ApiUtilityWrapper
  from common.common_type import CaseLabel, CheckTasks
  from common.milvus_sys import MilvusSys
  from customize.milvus_operator import MilvusOperator
  from common import common_func
  from common import common_type
  from scale import constants, scale_common
  from pymilvus import Index, connections, MilvusException
  from utils.util_log import test_log
  from utils.util_k8s import wait_pods_ready, read_pod_log
  from utils.util_pymilvus import get_latest_tag
  from utils.wrapper import counter
label_desc: Use image node_id 2 for calling the TestQueryNodeScale obj's underlying member method code with example usage: obj.test_scale_query_node_replicas() and returns: search_res,  is_succ