text: import time
  import pytest
  
  from helpers.cluster import ClickHouseCluster
  from helpers.cluster import ClickHouseKiller
  from helpers.test_tools import assert_eq_with_retry
  
  def fill_nodes(nodes):
      for node in nodes:
          node.query(
          '''
              CREATE TABLE test(n UInt32)
              ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/', '{replica}')
              ORDER BY n PARTITION BY n % 10;
          '''.format(replica=node.name))
  
  cluster = ClickHouseCluster(__file__)
  configs =["configs/remote_servers.xml"]
  
  node_1 = cluster.add_instance('replica1', with_zookeeper=True, main_configs=configs)
  node_2 = cluster.add_instance('replica2', with_zookeeper=True, main_configs=configs)
  node_3 = cluster.add_instance('replica3', with_zookeeper=True, main_configs=configs)
  nodes = [node_1, node_2, node_3]
  
  def fill_table():
      node_1.query("TRUNCATE TABLE test")
  
      for node in nodes:
          node.query("SYSTEM SYNC REPLICA test")
  
      check_data(0, 0)
  
      # it will create multiple parts in each partition and probably cause merges
      node_1.query("INSERT INTO test SELECT number + 0 FROM numbers(200)")
      node_1.query("INSERT INTO test SELECT number + 200 FROM numbers(200)")
      node_1.query("INSERT INTO test SELECT number + 400 FROM numbers(200)")
      node_1.query("INSERT INTO test SELECT number + 600 FROM numbers(200)")
      node_1.query("INSERT INTO test SELECT number + 800 FROM numbers(200)")
      check_data(499500, 1000)
  
  @pytest.fixture(scope="module")
  def start_cluster():
      try:
          cluster.start()
          fill_nodes(nodes)
          yield cluster
  
      except Exception as ex:
          print(ex)
  
      finally:
          cluster.shutdown()
  
  def check_data(_sum: int, count: int) -> None:
      res = "{}\t{}\n".format(_sum, count)
      assert_eq_with_retry(node_1, "SELECT sum(n), count() FROM test", res)
      assert_eq_with_retry(node_2, "SELECT sum(n), count() FROM test", res)
      assert_eq_with_retry(node_3, "SELECT sum(n), count() FROM test", res)
  
  def check_after_restoration():
      check_data(1999000, 2000)
  
      for node in nodes:
          node.query_and_get_error("SYSTEM RESTORE REPLICA test")
  
  def test_restore_replica_invalid_tables(start_cluster):
      print("Checking the invocation on non-existent and non-replicated tables")
      node_1.query_and_get_error("SYSTEM RESTORE REPLICA i_dont_exist_42")
      node_1.query_and_get_error("SYSTEM RESTORE REPLICA no_db.i_dont_exist_42")
      node_1.query_and_get_error("SYSTEM RESTORE REPLICA system.numbers")
  
  def test_restore_replica_sequential(start_cluster):
      zk = cluster.get_kazoo_client('zoo1')
      fill_table()
  
      print("Deleting root ZK path metadata")
      zk.delete("/clickhouse/tables/test", recursive=True)
      assert zk.exists("/clickhouse/tables/test") is None
  
      node_1.query("SYSTEM RESTART REPLICA test")
      node_1.query_and_get_error("INSERT INTO test SELECT number AS num FROM numbers(1000,2000) WHERE num % 2 = 0")
  
      print("Restoring replica1")
  
      node_1.query("SYSTEM RESTORE REPLICA test")
      assert zk.exists("/clickhouse/tables/test")
      check_data(499500, 1000)
  
      node_1.query("INSERT INTO test SELECT number + 1000 FROM numbers(1000)")
  
      print("Restoring other replicas")
  
      node_2.query("SYSTEM RESTART REPLICA test")
      node_2.query("SYSTEM RESTORE REPLICA test")
  
      node_3.query("SYSTEM RESTART REPLICA test")
      node_3.query("SYSTEM RESTORE REPLICA test")
  
      node_2.query("SYSTEM SYNC REPLICA test")
      node_3.query("SYSTEM SYNC REPLICA test")
  
      check_after_restoration()
  
  def test_restore_replica_parallel(start_cluster):
      zk = cluster.get_kazoo_client('zoo1')
      fill_table()
  
      print("Deleting root ZK path metadata")
      zk.delete("/clickhouse/tables/test", recursive=True)
      assert zk.exists("/clickhouse/tables/test") is None
  
      node_1.query("SYSTEM RESTART REPLICA test")
      node_1.query_and_get_error("INSERT INTO test SELECT number AS num FROM numbers(1000,2000) WHERE num % 2 = 0")
  
      print("Restoring replicas in parallel")
  
      node_2.query("SYSTEM RESTART REPLICA test")
      node_3.query("SYSTEM RESTART REPLICA test")
  
      node_1.query("SYSTEM RESTORE REPLICA test ON CLUSTER test_cluster")
  
      assert zk.exists("/clickhouse/tables/test")
      check_data(499500, 1000)
  
      node_1.query("INSERT INTO test SELECT number + 1000 FROM numbers(1000)")
  
      check_after_restoration()
  
  def test_restore_replica_alive_replicas(start_cluster):
      zk = cluster.get_kazoo_client('zoo1')
      fill_table()
  
      print("Deleting replica2 path, trying to restore replica1")
      zk.delete("/clickhouse/tables/test/replicas/replica2", recursive=True)
      assert zk.exists("/clickhouse/tables/test/replicas/replica2") is None
      node_1.query_and_get_error("SYSTEM RESTORE REPLICA test")
  
      print("Deleting replica1 path, trying to restore replica1")
      zk.delete("/clickhouse/tables/test/replicas/replica1", recursive=True)
      assert zk.exists("/clickhouse/tables/test/replicas/replica1") is None
  
      node_1.query("SYSTEM RESTART REPLICA test")
      node_1.query("SYSTEM RESTORE REPLICA test")
  
      node_2.query("SYSTEM RESTART REPLICA test")
      node_2.query("SYSTEM RESTORE REPLICA test")
  
      check_data(499500, 1000)
  
      node_1.query("INSERT INTO test SELECT number + 1000 FROM numbers(1000)")
  
      node_2.query("SYSTEM SYNC REPLICA test")
      node_3.query("SYSTEM SYNC REPLICA test")
  
      check_after_restoration()
  