class_docstr: The DeepSpeed Autotuner automatically discovers the optimal DeepSpeed configuration that delivers good training speed. The Autotuner uses model information, system information, and heuristics to efficiently tune system knobs that affect compute and memory efficiencies, such as ZeRO optimization stages, micro-batch sizes, and many other ZeRO optimization configurations. It not only reduces the time and resources user spend on tuning, but also can discover configurations better than hand-tuned methods.
  Autotuning with DeepSpeed requires no code change from DeepSpeed users. Please refer to the README for usage details.
class_docstr_tok: ["The","DeepSpeed","Autotuner","automatically","discovers","the","optimal","DeepSpeed","configuration","that","delivers","good","training","speed",".","The","Autotuner","uses","model","information",",","system","information",",","and","heuristics","to","efficiently","tune","system","knobs","that","affect","compute","and","memory","efficiencies",",","such","as","ZeRO","optimization","stages",",","micro-batch","sizes",",","and","many","other","ZeRO","optimization","configurations",".","It","not","only","reduces","the","time","and","resources","user","spend","on","tuning",",","but","also","can","discover","configurations","better","than","hand-tuned","methods",".","Autotuning","with","DeepSpeed","requires","no","code","change","from","DeepSpeed","users",".","Please","refer","to","the","README","for","usage","details","."]
code: def __init__(self, args, active_resources):
      self.args = args
      self.selected_exp_dir = None
  
      assert (
          tabulate is not None
      ), "Missing required package `tabulate`, please install with `pip install deepspeed[autotuning]`."
  
      logger.debug(f"autotunning args={args}")
  
      self.user_config = self._get_user_config(args.user_args)
      assert (
          self.user_config is not None
      ), "DeepSpeed configuration is not provided"
  
      self.autotuning_config = DeepSpeedAutotuningConfig(
          self.user_config
      )
      if self.user_config[AUTOTUNING]:
          if AUTOTUNING_EXPS_DIR in self.user_config[AUTOTUNING].keys():
              del self.user_config[AUTOTUNING][AUTOTUNING_EXPS_DIR]
          if (
              AUTOTUNING_RESULTS_DIR
              in self.user_config[AUTOTUNING].keys()
          ):
              del self.user_config[AUTOTUNING][AUTOTUNING_RESULTS_DIR]
  
      self.exps_dir = self.autotuning_config.exps_dir
      if self.autotuning_config.overwrite and os.path.exists(
          self.exps_dir
      ):
          shutil.rmtree(self.exps_dir, ignore_errors=True)
      if not os.path.exists(self.exps_dir):
          try:
              os.makedirs(self.exps_dir, exist_ok=True)
              logger.info(
                  f"Created autotuning experiments directory: {self.exps_dir}"
              )
          except:
              logger.error(
                  f"Failed to create {self.exps_dir}, please check `exps_dir` in the autotuning config file is accessible by all the nodes in the job."
              )
              exit(-1)
  
      self.results_dir = self.autotuning_config.results_dir
      if self.autotuning_config.overwrite and os.path.exists(
          self.results_dir
      ):
          shutil.rmtree(self.results_dir, ignore_errors=True)
      if not os.path.exists(self.results_dir):
          try:
              os.makedirs(self.results_dir, exist_ok=True)
              logger.info(
                  f"Created autotuning resutls directory: {self.exps_dir}"
              )
          except:
              logger.error(
                  f"Failed to create {self.results_dir}, please check `results_dir` in the autotuning config file is accessible by all the nodes in the job."
              )
              exit(-1)
  
      # set the active resource for the autotuner resource manager
      self.rm = self._get_resource_manager(active_resources)
  
      # get resource requirement for each autotuning experiment
      self.exp_num_nodes, self.exp_num_gpus = self._get_exp_resources(
          args
      )
  
      assert (
          self.exp_num_gpus <= self.rm.num_gpus_per_node
      ), "num_gpus in the autotuning configuration must not be less than the --num_gpus value in the train script if any"
      assert self.exp_num_nodes <= len(
          self.rm.nodes
      ), "num_nodes in the autotuning configuration must not be less than the --num_nodes value in the train script if any"
  
      self.records = {}
      self.optimal_cmd = None
      self.optmal_ds_config = None
  
imports: import shutil
  import subprocess
  import torch
  import time
  import datetime
  import math
  import hjson
  from ..runtime.config_utils import dict_raise_error_on_duplicate_keys
  from ..runtime.constants import *
  from ..runtime.zero.config import ZERO_OPTIMIZATION, ZeroStageEnum
  from ..utils import logger
  from .config import DeepSpeedAutotuningConfig
  from .constants import *
  from .scheduler import ResourceManager
  from .tuner import GridSearchTuner, RandomTuner, ModelBasedTuner
  from .utils import *
label_desc: Use image node_id 1 to create a new Autotuner object  with example: obj = Autotuner(args, active_resources)