class_docstr: The Slurm Evaluator is a class designed to continuously monitor VISSL pretrainings
  and launch evaluations as checkpoints become available. The method takes a
  config dictionary consisting of the training checkpoint directory, an array of
  benchmarks, and information on how often to evaluate the trainings.
code: def _evaluate_checkpoint(self, checkpoint_str, benchmarks):
      for benchmark in benchmarks:
          retry_job = self._retry_job(benchmark)
          if benchmark["job_id"] and not retry_job:
              continue
  
          if retry_job:
              self.evaluation_jobs_finished.remove(benchmark["job_id"])
              # Log the job retry.
              job_id, slurm_state = (
                  benchmark["job_id"],
                  benchmark["slurm_state"],
              )
              logging.info(
                  f"Retrying job: { job_id } in state: { slurm_state }"
              )
  
          args, config = self._generate_config(
              benchmark["config_files"]
          )
          job = self._launch_slurm_job(args, config)
  
          time.sleep(
              10
          )  # Wait for slurm job status to be reliably updated.
  
          # Set checkpoint status
          benchmark["job_id"] = job.job_id
          benchmark["num_retries"] += 1
          benchmark["slurm_log_dir"] = config.SLURM.LOG_FOLDER
          benchmark["slurm_checkpoint_dir"] = config.CHECKPOINT.DIR
          benchmark[
              "weights_init_params_file"
          ] = config.MODEL.WEIGHTS_INIT.PARAMS_FILE
          benchmark["slurm_state"] = job.state
  
          current_time = datetime.now().strftime("%H:%M:%S %z")
          log = f"""
                  Launched Slurm Evaluation job. Time: { current_time }
                  job_id: { job.job_id }, num_retries: { benchmark["num_retries"] }
                  evaluation_name: { benchmark["evaluation_name"] }
                  checkpoint_str: { checkpoint_str }
                  state_prev: None, state_current: { job.state }
              """
  
          logging.info(log)
  
          # Save evaluation results to json file.
          self.save_evaluation_benchmarks()
  
imports: import collections
  import json
  import logging
  import os
  import time
  from datetime import datetime
  from pathlib import Path
  from typing import List
  import submitit
  from iopath.common.file_io import g_pathmgr
  from vissl.config.attr_dict import AttrDict
  from vissl.utils.distributed_launcher import launch_distributed_on_slurm
  from vissl.utils.hydra_config import compose_hydra_configuration, convert_to_attrdict
  from vissl.utils.io import load_file, makedir
  from vissl.utils.misc import flatten_dict, retry
label_desc: Use image node_id 11 for calling the BenchmarkSuiteScheduler obj's underlying member method code with example usage: obj._evaluate_checkpoint(checkpoint_str, benchmarks) without return types