class_docstr: FP16 Optimizer for training fp16 models. Handles loss scaling.
  
  For usage example please see, TODO:  DeepSpeed V2 Tutorial
   
class_docstr_tok: ["FP16","Optimizer","for","training","fp16","models",".","Handles","loss","scaling",".","For","usage","example","please","see",",","TODO",":","DeepSpeed","V2","Tutorial"]
code: def __init__(
      self,
      init_optimizer,
      deepspeed=None,
      static_loss_scale=1.0,
      dynamic_loss_scale=False,
      initial_dynamic_scale=2**32,
      dynamic_loss_args=None,
      verbose=True,
      mpu=None,
      clip_grad=0.0,
      fused_adam_legacy=False,
      has_moe_layers=False,
      timers=None,
  ):
      self.fused_adam_legacy = fused_adam_legacy
      self.timers = timers
      self.deepspeed = deepspeed
      self.has_moe_layers = has_moe_layers
      self.using_pipeline = self.deepspeed.pipeline_parallelism
      if not torch.cuda.is_available:
          raise SystemError("Cannot use fp16 without CUDA.")
      self.optimizer = init_optimizer
  
      # param flattened by groups
      self.fp16_groups = []
      self.fp16_groups_flat = []
      self.fp32_groups_flat = []
  
      self._global_grad_norm = 0.0
  
      # loop to deal with groups
      for i, param_group in enumerate(self.optimizer.param_groups):
          # push this group to list before modify
          self.fp16_groups.append(param_group["params"])
          # init fp16 weight buffer, flattened
          self.fp16_groups_flat.append(
              _flatten_dense_tensors(
                  [p.clone().detach() for p in self.fp16_groups[i]]
              )
          )
          # set model fp16 weight to slices of flattened buffer
          updated_params = _unflatten_dense_tensors(
              self.fp16_groups_flat[i], self.fp16_groups[i]
          )
          for p, q in zip(self.fp16_groups[i], updated_params):
              p.data = q.data
          # init master weight, flattened
          self.fp32_groups_flat.append(
              self.fp16_groups_flat[i].clone().float().detach()
          )
          # modify optimizer of have flat master weight
          self.fp32_groups_flat[
              i
          ].requires_grad = (
              True  # keep this in case internal optimizer uses it
          )
          param_group["params"] = [self.fp32_groups_flat[i]]
  
      # we may have a way of fusing dynamic scale. Do not support for now
      if dynamic_loss_scale:
          self.dynamic_loss_scale = True
          self.cur_iter = 0
          self.last_overflow_iter = -1
          self.scale_factor = 2
  
          if dynamic_loss_args is None:
              self.cur_scale = initial_dynamic_scale
              self.scale_window = 1000
              self.min_loss_scale = 1
          else:
              self.cur_scale = dynamic_loss_args[INITIAL_LOSS_SCALE]
              self.scale_window = dynamic_loss_args[SCALE_WINDOW]
              self.min_loss_scale = dynamic_loss_args[MIN_LOSS_SCALE]
      else:
          self.dynamic_loss_scale = False
          self.cur_iter = 0
          self.cur_scale = static_loss_scale
      self.verbose = verbose
  
      self.custom_loss_scaler = False
      self.external_loss_scale = None
  
      self.clip_grad = clip_grad
      self.norm_type = 2
      self.step_count = 0
  
      TORCH_MAJOR = int(torch.__version__.split(".")[0])
      TORCH_MINOR = int(torch.__version__.split(".")[1])
      if TORCH_MAJOR == 0 and TORCH_MINOR <= 4:
          self.clip_grad_norm = torch.nn.utils.clip_grad_norm
      else:
          self.clip_grad_norm = torch.nn.utils.clip_grad_norm_
  
      # model parallel object
      self.mpu = mpu
  
      self.overflow = False
      self.overflow_checker = CheckOverflow(
          self.fp16_groups, mpu=self.mpu, deepspeed=deepspeed
      )
      self.initialize_optimizer_states()
  
imports: import torch
  from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors
  from deepspeed.runtime import DeepSpeedOptimizer
  from deepspeed.runtime.utils import get_global_norm, get_grad_norm, CheckOverflow, get_weight_norm
  from deepspeed.runtime.fp16.loss_scaler import INITIAL_LOSS_SCALE, SCALE_WINDOW, MIN_LOSS_SCALE
  from deepspeed.utils import groups, logger, log_dist
  from deepspeed import comm
  from deepspeed.checkpoint.constants import OPTIMIZER_STATE_DICT, CLIP_GRAD
label_desc: Use image node_id 1 to create a new FP16_Optimizer object from inherited base classes: DeepSpeedOptimizer with example: obj = FP16_Optimizer(init_optimizer, deepspeed, static_loss_scale, dynamic_loss_scale, initial_dynamic_scale, dynamic_loss_args, verbose, mpu, clip_grad, fused_adam_legacy, has_moe_layers, timers)