code: def test_noop_pause_hpc() -> None:
      # The original configuration file, which we will need to modify for HPC
      # clusters. We choose a configuration file that will create an experiment
      # that runs long enough to allow us to pause it after the first check
      # point is recorded.  If we choose an experiment that completes too
      # quickly, then by the time we try to pause it, the experiment may be over,
      # so we won't be able to activate (restart) it.
      config_file = conf.fixtures_path("no_op/single-hpc.yaml")
  
      # Walk through starting, pausing, and resuming a single no-op experiment.
      experiment_id = exp.create_experiment(
          config_file, conf.fixtures_path("no_op"), None
      )
      exp.wait_for_experiment_state(
          experiment_id, bindings.experimentv1State.RUNNING
      )
  
      # Wait for the only trial to get scheduled.
      exp.wait_for_experiment_active_workload(experiment_id)
  
      # Wait for the only trial to show progress, indicating the image is built and running.
      exp.wait_for_experiment_workload_progress(experiment_id)
  
      # If we pause the experiment before it gets to write at least one checkpoint,
      # then we're really not testing whether the experiment can pick up from where
      # it left off when it's activated.  In which case, the "activate" simply
      # starts from the beginning upon finding that are no checkpoints to start
      # from.  Therefore, wait a while to give the experiment a chance to write at
      # least one checkpoint.
      exp.wait_for_at_least_one_checkpoint(experiment_id)
  
      # Pause the experiment. Note that Determined does not currently differentiate
      # between a "stopping paused" and a "paused" state, so we follow this check
      # up by ensuring the experiment cleared all scheduled workloads.
      exp.pause_experiment(experiment_id)
      exp.wait_for_experiment_state(
          experiment_id, bindings.experimentv1State.PAUSED
      )
  
      # Wait at most 420 seconds for the experiment to clear all workloads (each
      # train step should take 5 seconds).
      for _ in range(420):
          workload_active = exp.experiment_has_active_workload(
              experiment_id
          )
          if not workload_active:
              break
          else:
              time.sleep(1)
      check.true(
          not workload_active,
          "The experiment cannot be paused within 420 seconds.",
      )
  
      # Resume the experiment and wait for completion.
      exp.activate_experiment(experiment_id)
      exp.wait_for_experiment_state(
          experiment_id, bindings.experimentv1State.COMPLETED
      )
  
file_path: determined/e2e_tests/tests/experiment/test_noop_hpc.py
imports: import os
  import tempfile
  import time
  import pytest
  from determined.common import check, yaml
  from determined.common.api import bindings
  from tests import config
  from tests import experiment
label_desc: Use image node_id 1 for calling a global function with example usage: test_noop_pause_hpc() without return types