class_docstr: Fault tolerance tests for distributed save/load.
class_docstr_tok: ["Fault","tolerance","tests","for","distributed","save\/load","."]
code: def test_add_worker_midjob(
      self,
      load_repetitions: int,
      sharding_policy: data_service_ops.ShardingPolicy,
  ):
      num_elements = 2 * multiprocessing.cpu_count() + 100
      cluster = data_service_test_base.TestCluster(num_workers=1)
      snapshot_dir = data_service_test_base.TempDir()
      dataset = dataset_ops.Dataset.range(num_elements)
      self.evaluate(
          distributed_save_op.distributed_save(
              dataset,
              snapshot_dir.full_path,
              cluster.dispatcher_address(),
          )
      )
  
      dataset = load_op._load_with_retry(snapshot_dir.full_path)
      dataset = dataset.repeat(load_repetitions)
      dataset = dataset.apply(
          data_service_ops.distribute(
              sharding_policy,
              cluster.dispatcher_address(),
              max_outstanding_requests=1,
          )
      )
      expected = list(range(num_elements)) * load_repetitions
      if sharding_policy == data_service_ops.ShardingPolicy.OFF:
          expected *= 2
  
      # Reads halfway through the dataset.
      iterator = self.getNext(dataset)
      output = [
          self.evaluate(iterator()) for _ in range(num_elements // 2)
      ]
  
      # Waits for a new worker to register with the dispatcher.
      cluster.add_worker()
      while cluster.num_registered_workers() < 2:
          time.sleep(10 / 1000)  # 10ms
  
      output.extend(self.getIteratorOutput(iterator))
      self.assertCountEqual(output, expected)
  
file_path: tensorflow/tensorflow/python/data/experimental/kernel_tests/service/distributed_save_load_ft_test.py
imports: import multiprocessing
  import time
  from absl.testing import parameterized
  from tensorflow.python.data.experimental.kernel_tests.service import test_base
  from tensorflow.python.data.experimental.ops import data_service_ops
  from tensorflow.python.data.experimental.ops import distributed_save_op
  from tensorflow.python.data.kernel_tests import test_base
  from tensorflow.python.data.ops import dataset_ops
  from tensorflow.python.data.ops import load_op
  from tensorflow.python.framework import combinations
  from tensorflow.python.platform import test
label_desc: Use image node_id 3 for calling the DistributedSaveLoadFtTest obj's underlying member method code with example usage: obj.test_add_worker_midjob(load_repetitions, sharding_policy) without return types