code: def test_cluster_down_full(start_connected_cluster, tmpdir):
      """Tests that run_experiment restoring works on cluster shutdown."""
      cluster = start_connected_cluster
  
      base_dict = dict(run="__fake", stop=dict(training_iteration=3))
  
      exp1_args = base_dict
      exp2_args = dict(
          base_dict.items(),
          checkpoint_config=CheckpointConfig(checkpoint_frequency=1),
      )
      exp3_args = dict(base_dict.items(), config=dict(mock_error=True))
      exp4_args = dict(
          base_dict.items(),
          config=dict(mock_error=True),
          checkpoint_config=CheckpointConfig(checkpoint_frequency=1),
      )
  
      all_experiments = {
          "exp1": exp1_args,
          "exp2": exp2_args,
          "exp3": exp3_args,
          "exp4": exp4_args,
      }
  
      tune.run_experiments(all_experiments, raise_on_failed_trial=False)
  
      ray.shutdown()
      cluster.shutdown()
      cluster = _start_new_cluster()
  
      trials = tune.run_experiments(
          all_experiments,
          resume=True,
          raise_on_failed_trial=False,
      )
  
      assert len(trials) == 4
      assert all(
          t.status in [Trial.TERMINATED, Trial.ERROR] for t in trials
      )
      ray.shutdown()
      cluster.shutdown()
  
imports: import time
  import os
  import pytest
  import sys
  from unittest.mock import MagicMock
  import ray
  from ray import tune
  from ray.train import CheckpointConfig
  from ray.cluster_utils import Cluster
  from ray.train._internal.storage import StorageContext
  from ray.tune.error import TuneError
  from ray.tune.search import BasicVariantGenerator
  from ray.tune.experiment import Trial
  from ray.tune.execution.tune_controller import TuneController
label_desc: Use image node_id 12 for calling a global function with example usage: test_cluster_down_full(start_connected_cluster, tmpdir) without return types