code: def test_continue_batches() -> None:
      # Experiment fails before first checkpoint.
      exp_id = exp.create_experiment(
          conf.fixtures_path("mnist_pytorch/failable.yaml"),
          conf.fixtures_path("mnist_pytorch"),
          [
              "--config",
              "environment.environment_variables=['FAIL_AT_BATCH=2']",
          ],
      )
      exp.wait_for_experiment_state(
          exp_id, bindings.experimentv1State.ERROR
      )
  
      sess = api_utils.determined_test_session()
      trials = exp.experiment_trials(exp_id)
      assert len(trials) == 1
      trial_id = trials[0].trial.id
  
      def assert_exited_at(batch_idx: int) -> None:
          assert f"failed at this batch {batch_idx}" in "\n".join(
              exp.trial_logs(trial_id)
          )
  
      assert_exited_at(2)
  
      def get_metric_list() -> List[bindings.v1MetricsReport]:
          resp_list = bindings.get_GetValidationMetrics(
              sess, trialIds=[trial_id]
          )
          return [
              metric for resp in resp_list for metric in resp.metrics
          ]
  
      metrics = get_metric_list()
      assert len(metrics) == 2
  
      first_metric_ids = []
      i = 1
      for m in metrics:
          first_metric_ids.append(m.id)
          assert m.totalBatches == i
          i += 1
  
      # Experiment has to start over since we didn't checkpoint.
      # We must invalidate all previous reported metrics.
      # This time experiment makes it a validation after the first checkpoint.
      det_cmd(
          [
              "e",
              "continue",
              str(exp_id),
              "--config",
              "environment.environment_variables=['FAIL_AT_BATCH=5']",
          ],
          check=True,
      )
      exp.wait_for_experiment_state(
          exp_id, bindings.experimentv1State.ERROR
      )
      assert_exited_at(5)
  
      second_metric_ids = []
      metrics = get_metric_list()
      assert len(metrics) == 5
      i = 1
      for m in metrics:
          assert (
              m.id not in first_metric_ids
          )  # Invalidated first metrics.
          second_metric_ids.append(m.id)
          assert m.totalBatches == i
          i += 1
  
      # We lose one metric since we are continuing from first checkpoint.
      # We correctly stop at total_batches.
      det_cmd(
          [
              "e",
              "continue",
              str(exp_id),
              "--config",
              "environment.environment_variables=['FAIL_AT_BATCH=-1']",
          ],
          check=True,
      )
      exp.wait_for_experiment_state(
          exp_id, bindings.experimentv1State.COMPLETED
      )
  
      metrics = get_metric_list()
      assert len(metrics) == 8
      i = 1
      for m in metrics:
          if m.totalBatches <= 3:
              assert m.id in second_metric_ids
          else:
              assert m.id not in second_metric_ids
          assert m.totalBatches == i
          i += 1
  
file_path: determined/e2e_tests/tests/cluster/test_exp_continue.py
imports: import subprocess
  import tempfile
  from typing import Any, List, Tuple
  import pytest
  from determined.common import util
  from determined.common.api import bindings
  from tests import api_utils
  from tests import config
  from tests import experiment
label_desc: Use image node_id 7 for calling a global function with example usage: test_continue_batches() and returns: unknown