class_docstr: The Slurm Evaluator is a class designed to continuously monitor VISSL pretrainings
  and launch evaluations as checkpoints become available. The method takes a
  config dictionary consisting of the training checkpoint directory, an array of
  benchmarks, and information on how often to evaluate the trainings.
docstr: Args:
  
  training_checkpoint_dir: (str). Checkpoint directory of the training.
                          This should match the trainings CHECKPOINT.dir
  benchmarks: (list[dict]) Benchmarks with the following structure:
      "config_files": [
              {
                  # Path to config file.
                  "config=test/integration_test/quick_eval_in1k_linear.yaml"
                  # Config overrides.
                  "config.TRAIN.DATA_LIMIT=1000",
                  ...
              },
              ...
          ]
  evaluate_final_phase: (bool, optional, default=True). Whether or not to evaluate the
                          final phase of the training.
  evaluation_phase_freq: (int, optional, default=-1) How often to evaluate phases.
                          Training checkpoint phase freq must evenly
                          divide evaluation_phase_freq.
  evaluation_iter_freq: (int, optional, default=-1) How often to evaluate iterations.
                         Training checkpoint iteration freq must evenly
                         divide evaluation_iter_freq.
  autoload_slurm_evaluator_checkpoint: (bool, optional, default=False) Whether or not to
                                        autoload slurm_evaluator Checkpoint.
                                        This is useful when slurm evaluator job
                                        is preempted for example.
  slurm_evaluator_checkpoint: (str, optional, default=None) String of
                              slurm_evaluator checkpoint directory.
  retry_evaluation_job_ids: (List[int], optional, default=[]) List of job_ids to retry.
  auto_retry_evaluations: (bool, optional, default=False) Whether or not to automatically
                          retry all failed jobs.
  max_retries: (int, optional, default=3). Maximum number of retries.
  pytorch_ports: (list[int], optional, default=[40050]). Ports to cycle through as
                  you are launching your trainings.
docstr_tok: ["Args",":","training_checkpoint_dir",":","(","str",")",".","Checkpoint","directory","of","the","training",".","This","should","match","the","trainings","CHECKPOINT.dir","benchmarks",":","(","list","[","dict","]",")","Benchmarks","with","the","following","structure",":","``","config_files","''",":","[","{","#","Path","to","config","file",".","``","config=test\/integration_test\/quick_eval_in1k_linear.yaml","''","#","Config","overrides",".","``","config.TRAIN.DATA_LIMIT=1000","''",",","...","}",",","...","]","evaluate_final_phase",":","(","bool",",","optional",",","default=True",")",".","Whether","or","not","to","evaluate","the","final","phase","of","the","training",".","evaluation_phase_freq",":","(","int",",","optional",",","default=-1",")","How","often","to","evaluate","phases",".","Training","checkpoint","phase","freq","must","evenly","divide","evaluation_phase_freq",".","evaluation_iter_freq",":","(","int",",","optional",",","default=-1",")","How","often","to","evaluate","iterations",".","Training","checkpoint","iteration","freq","must","evenly","divide","evaluation_iter_freq",".","autoload_slurm_evaluator_checkpoint",":","(","bool",",","optional",",","default=False",")","Whether","or","not","to","autoload","slurm_evaluator","Checkpoint",".","This","is","useful","when","slurm","evaluator","job","is","preempted","for","example",".","slurm_evaluator_checkpoint",":","(","str",",","optional",",","default=None",")","String","of","slurm_evaluator","checkpoint","directory",".","retry_evaluation_job_ids",":","(","List","[","int","]",",","optional",",","default=","[","]",")","List","of","job_ids","to","retry",".","auto_retry_evaluations",":","(","bool",",","optional",",","default=False",")","Whether","or","not","to","automatically","retry","all","failed","jobs",".","max_retries",":","(","int",",","optional",",","default=3",")",".","Maximum","number","of","retries",".","pytorch_ports",":","(","list","[","int","]",",","optional",",","default=","[","40050","]",")",".","Ports","to","cycle","through","as","you","are","launching","your","trainings","."]
code: def __init__(
      self,
      training_checkpoint_dir: str,
      benchmarks: List,
      evaluate_final_phase=True,
      evaluation_phase_freq: int = -1,
      evaluation_iter_freq: int = -1,
      autoload_slurm_evaluator_checkpoint=False,
      slurm_evaluator_checkpoint: str = None,
      retry_evaluation_job_ids: List[int] = None,
      auto_retry_evaluations=False,
      max_retries=3,
      pytorch_ports=None,
  ):
      """
      Args:
  
      training_checkpoint_dir: (str). Checkpoint directory of the training.
                              This should match the trainings CHECKPOINT.dir
      benchmarks: (list[dict]) Benchmarks with the following structure:
          "config_files": [
                  {
                      # Path to config file.
                      "config=test/integration_test/quick_eval_in1k_linear.yaml"
                      # Config overrides.
                      "config.TRAIN.DATA_LIMIT=1000",
                      ...
                  },
                  ...
              ]
      evaluate_final_phase: (bool, optional, default=True). Whether or not to evaluate the
                              final phase of the training.
      evaluation_phase_freq: (int, optional, default=-1) How often to evaluate phases.
                              Training checkpoint phase freq must evenly
                              divide evaluation_phase_freq.
      evaluation_iter_freq: (int, optional, default=-1) How often to evaluate iterations.
                             Training checkpoint iteration freq must evenly
                             divide evaluation_iter_freq.
      autoload_slurm_evaluator_checkpoint: (bool, optional, default=False) Whether or not to
                                            autoload slurm_evaluator Checkpoint.
                                            This is useful when slurm evaluator job
                                            is preempted for example.
      slurm_evaluator_checkpoint: (str, optional, default=None) String of
                                  slurm_evaluator checkpoint directory.
      retry_evaluation_job_ids: (List[int], optional, default=[]) List of job_ids to retry.
      auto_retry_evaluations: (bool, optional, default=False) Whether or not to automatically
                              retry all failed jobs.
      max_retries: (int, optional, default=3). Maximum number of retries.
      pytorch_ports: (list[int], optional, default=[40050]). Ports to cycle through as
                      you are launching your trainings.
      """
      self.evaluation_jobs_finished = set()
  
      # Required Arguments
      self.training_checkpoint_dir = training_checkpoint_dir
      self.training_checkpoint_file = os.path.join(
          self.training_checkpoint_dir, "train_config.yaml"
      )
      self.benchmarks = benchmarks
  
      # Optional Arguments
      self.evaluate_final_phase = evaluate_final_phase
      self.evaluation_phase_freq = evaluation_phase_freq
      self.evaluation_iter_freq = evaluation_iter_freq
      self.autoload_slurm_evaluator_checkpoint = (
          autoload_slurm_evaluator_checkpoint
      )
      self.slurm_evaluator_checkpoint = slurm_evaluator_checkpoint
      self.retry_evaluation_job_ids = retry_evaluation_job_ids or []
      self.auto_retry_evaluations = auto_retry_evaluations
      self.max_retries = max_retries
      self.pytorch_ports = pytorch_ports or _DEFAULT_PYTORCH_PORTS
      self.pytorch_ports_iterable = iter(self.pytorch_ports)
  
      self.validate()
  
      # Will be set in #evaluate, once training_checkpoint_dir becomes available.
      self.training_config = None
      self.evaluation_results = None
  
imports: import collections
  import json
  import logging
  import os
  import time
  from datetime import datetime
  from pathlib import Path
  from typing import List
  import submitit
  from iopath.common.file_io import g_pathmgr
  from vissl.config.attr_dict import AttrDict
  from vissl.utils.distributed_launcher import launch_distributed_on_slurm
  from vissl.utils.hydra_config import compose_hydra_configuration, convert_to_attrdict
  from vissl.utils.io import load_file, makedir
  from vissl.utils.misc import flatten_dict, retry
label_desc: Use image node_id 1 to create a new BenchmarkSuiteScheduler object  with example: obj = BenchmarkSuiteScheduler(training_checkpoint_dir, benchmarks, evaluate_final_phase, evaluation_phase_freq, evaluation_iter_freq, autoload_slurm_evaluator_checkpoint, slurm_evaluator_checkpoint, retry_evaluation_job_ids, auto_retry_evaluations, max_retries, pytorch_ports)