docstr: Test that Tune run can recover from a failed node.
  
  When `max_failures` is set to larger than zero.
docstr_tok: ["Test","that","Tune","run","can","recover","from","a","failed","node",".","When","`","max_failures","`","is","set","to","larger","than","zero","."]
code: def testFaultTolerance(self):
      """Test that Tune run can recover from a failed node.
  
      When `max_failures` is set to larger than zero.
      """
      self.cluster.update_config(
          {
              "provider": {"head_resources": {"CPU": 4, "GPU": 0}},
              "available_node_types": {
                  "ray.worker.cpu": {
                      "resources": {"CPU": 4},
                      "min_workers": 0,  # No minimum nodes
                      "max_workers": 2,
                  },
                  "ray.worker.gpu": {
                      "min_workers": 0,
                      "max_workers": 0,  # No GPU nodes
                  },
              },
          }
      )
      self.cluster.start()
      self.cluster.connect(client=True, timeout=120)
      remote_api = self.cluster.remote_execution_api()
  
      def train_fn(config):
          time.sleep(120)
          train.report({"_metric": 1.0})
  
      class FailureInjectionCallback(Callback):
          def __init__(self):
              self._killed = False
  
          def on_step_begin(self, iteration, trials, **info):
              if (
                  not self._killed
                  and len(trials) == 3
                  and all(
                      trial.status == Trial.RUNNING for trial in trials
                  )
              ):
                  remote_api.kill_node(num=2)
                  self._killed = True
  
      tune.run(
          train_fn,
          num_samples=3,
          resources_per_trial={"cpu": 4},
          max_failures=1,
          callbacks=[FailureInjectionCallback()],
      )
  
file_path: ray/python/ray/tune/tests/test_multinode_sync.py
imports: import sys
  import time
  import unittest
  import ray
  from ray import train, tune
  from ray.air.util.node import _force_on_node
  from ray.autoscaler._private.fake_multi_node.test_utils import DockerCluster
  from ray.tune.callback import Callback
  from ray.tune.experiment import Trial
  from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
label_desc: Use image node_id 5 for calling the MultiNodeSyncTest obj's underlying member method code with example usage: obj.testFaultTolerance() without return types