class_docstr: FP16 Optimizer for training fp16 models. Handles loss scaling.
  
  For usage example please see, TODO:  DeepSpeed V2 Tutorial
   
class_docstr_tok: ["FP16","Optimizer","for","training","fp16","models",".","Handles","loss","scaling",".","For","usage","example","please","see",",","TODO",":","DeepSpeed","V2","Tutorial"]
docstr: Not supporting closure.
docstr_tok: ["Not","supporting","closure","."]
code: def step(self, closure=None):
      """
      Not supporting closure.
      """
  
      if self.fused_adam_legacy:
          return self.step_fused_adam()
  
      # First determine if there is overflow.
      self.timers(OVERFLOW_CHECK_TIMER).start()
      fp16_params = []
      for i, group in enumerate(self.fp16_groups):
          fp16_params.extend([p for p in group if p.grad is not None])
      self.overflow = self.overflow_checker.has_overflow(fp16_params)
      self.timers(OVERFLOW_CHECK_TIMER).stop()
      prev_scale = self.cur_scale
      self._update_scale(self.overflow)
      if self.overflow:
          if self.verbose:
              log_dist(
                  "Overflow detected. Skipping step. Attempted loss "
                  f"scale: {prev_scale}, reducing to {self.cur_scale}",
                  ranks=[0],
              )
          # Clear gradients
          for i, group in enumerate(self.fp16_groups):
              for p in group:
                  p.grad = None
  
          self.timers.log(OVERFLOW_TIMERS)
          return self.overflow
  
      grads_groups_flat = []
      for i, group in enumerate(self.fp16_groups):
          data_type = self.fp32_groups_flat[i].dtype
  
          grads_groups_flat.append(
              _flatten_dense_tensors(
                  [
                      torch.zeros(
                          p.size(), dtype=data_type, device=p.device
                      )
                      if p.grad is None
                      else p.grad.to(data_type)
                      for p in group
                  ]
              )
          )
  
          for p in group:
              p.grad = None
  
          self.fp32_groups_flat[i].grad = grads_groups_flat[i]
  
      self.timers(COMPUTE_NORM_TIMER).start()
  
      all_groups_norm = get_grad_norm(
          self.fp32_groups_flat, mpu=self.mpu
      )
  
      self.timers(COMPUTE_NORM_TIMER).stop()
  
      if self.has_moe_layers:
          all_groups_norm = self._get_norm_with_moe_layers(
              all_groups_norm
          )
  
      scaled_global_grad_norm = get_global_norm(
          norm_list=[all_groups_norm]
      )
  
      # Stash unscaled gradient norm
      self._global_grad_norm = scaled_global_grad_norm / self.cur_scale
  
      self.timers(UNSCALE_AND_CLIP_TIMER).start()
      self.unscale_and_clip_grads(
          grads_groups_flat, scaled_global_grad_norm
      )
      self.timers(UNSCALE_AND_CLIP_TIMER).stop()
  
      self.timers(BASIC_STEP_TIMER).start()
      self.optimizer.step()
      self.timers(BASIC_STEP_TIMER).stop()
  
      # get rid of the fp32 gradients. Not needed anymore
      for group in self.fp32_groups_flat:
          group.grad = None
  
      self.timers(UPDATE_FP16_TIMER).start()
  
      for i in range(len(self.fp16_groups)):
          updated_params = _unflatten_dense_tensors(
              self.fp32_groups_flat[i], self.fp16_groups[i]
          )
          for p, q in zip(self.fp16_groups[i], updated_params):
              p.data.copy_(q.data)
  
      self.timers(UPDATE_FP16_TIMER).stop()
  
      self.timers.log(STEP_TIMERS)
  
      return self.overflow
  
imports: import torch
  from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors
  from deepspeed.runtime import DeepSpeedOptimizer
  from deepspeed.runtime.utils import get_global_norm, get_grad_norm, CheckOverflow, get_weight_norm, required_torch_version
  from deepspeed.runtime.fp16.loss_scaler import INITIAL_LOSS_SCALE, SCALE_WINDOW, MIN_LOSS_SCALE
  from deepspeed.utils import groups, logger, log_dist
  from deepspeed import comm
  from deepspeed.checkpoint.constants import OPTIMIZER_STATE_DICT, CLIP_GRAD
  from deepspeed.accelerator import get_accelerator
label_desc: Use image node_id 8 for calling the FP16_Optimizer obj's underlying member method code with example usage: obj.step(closure) and returns: self, self, self