class_docstr: Fault tolerance tests for distributed save/load.
class_docstr_tok: ["Fault","tolerance","tests","for","distributed","save\/load","."]
code: def test_dispatcher_restart(
      self,
      num_workers: int,
      num_elements: int,
      load_repetitions: int,
      sharding_policy: data_service_ops.ShardingPolicy,
  ):
      cluster = data_service_test_base.TestCluster(
          num_workers=num_workers
      )
      snapshot_dir = data_service_test_base.TempDir()
      dataset = dataset_ops.Dataset.range(num_elements)
      self.evaluate(
          distributed_save_op.distributed_save(
              dataset,
              snapshot_dir.full_path,
              cluster.dispatcher_address(),
          )
      )
  
      dataset = load_op._load_with_retry(snapshot_dir.full_path)
      dataset = dataset.repeat(load_repetitions)
      dataset = dataset.apply(
          data_service_ops.distribute(
              sharding_policy,
              cluster.dispatcher_address(),
              max_outstanding_requests=1,
          )
      )
  
      iterator = self.getNext(dataset)
      output = [self.evaluate(iterator())]
      cluster.restart_dispatcher()
      output.extend(self.getIteratorOutput(iterator))
  
      # For no sharding, dispatcher restarts do not affect data processing
      # happening at the workers.
      if sharding_policy == data_service_ops.ShardingPolicy.OFF:
          expected = (
              list(range(num_elements)) * load_repetitions * num_workers
          )
          self.assertCountEqual(output, expected)
  
      # Dynamic sharding may lose splits if the dispatcher fails.
      if sharding_policy == data_service_ops.ShardingPolicy.DYNAMIC:
          self.assertNotEmpty(output)
          self.assertContainsSubset(output, range(num_elements))
  
file_path: tensorflow/tensorflow/python/data/experimental/kernel_tests/service/distributed_save_load_ft_test.py
imports: import multiprocessing
  import time
  from absl.testing import parameterized
  from tensorflow.python.data.experimental.kernel_tests.service import test_base
  from tensorflow.python.data.experimental.ops import data_service_ops
  from tensorflow.python.data.experimental.ops import distributed_save_op
  from tensorflow.python.data.kernel_tests import test_base
  from tensorflow.python.data.ops import dataset_ops
  from tensorflow.python.data.ops import load_op
  from tensorflow.python.framework import combinations
  from tensorflow.python.platform import test
label_desc: Use image node_id 1 for calling the DistributedSaveLoadFtTest obj's underlying member method code with example usage: obj.test_dispatcher_restart(num_workers, num_elements, load_repetitions, sharding_policy) without return types