class_docstr: FP16 Optimizer for training fp16 models. Handles loss scaling.
  
  For usage example please see, TODO:  DeepSpeed V2 Tutorial
   
class_docstr_tok: ["FP16","Optimizer","for","training","fp16","models",".","Handles","loss","scaling",".","For","usage","example","please","see",",","TODO",":","DeepSpeed","V2","Tutorial"]
docstr: Not supporting closure.
docstr_tok: ["Not","supporting","closure","."]
code: def step_fused_adam(self, closure=None):
      """
      Not supporting closure.
      """
  
      # First compute norm for all group so we know if there is overflow
      grads_groups_flat = []
      norm_groups = []
      for i, group in enumerate(self.fp16_groups):
          grads_groups_flat.append(
              _flatten_dense_tensors(
                  [
                      torch.zeros(
                          p.size(), dtype=p.dtype, device=p.device
                      )
                      if p.grad is None
                      else p.grad
                      for p in group
                  ]
              )
          )
          norm_groups.append(
              get_weight_norm(grads_groups_flat[i], mpu=self.mpu)
          )
  
      self.overflow = self.overflow_checker.check_using_norm(
          norm_groups
      )
      prev_scale = self.cur_scale
      self._update_scale(self.overflow)
  
      if self.overflow:
          if self.verbose:
              logger.info(
                  "[deepspeed] fp16 dynamic loss scale overflow! Skipping step. Attempted loss "
                  "scale: {}, reducing to {}".format(
                      prev_scale, self.cur_scale
                  )
              )
          return self.overflow
  
      scaled_grad_norm = get_global_norm(norm_list=norm_groups)
  
      combined_scale = self.unscale_and_clip_grads(
          grads_groups_flat, scaled_grad_norm, apply_scale=False
      )
  
      # Stash unscaled gradient norm
      self._global_grad_norm = scaled_grad_norm / self.cur_scale
  
      # norm is in fact norm*cur_scale
      self.optimizer.step(
          grads=[[g] for g in grads_groups_flat],
          output_params=[[p] for p in self.fp16_groups_flat],
          scale=combined_scale,
          grad_norms=norm_groups,
      )
      # TODO: we probably don't need this? just to be safe
      for i in range(len(norm_groups)):
          updated_params = _unflatten_dense_tensors(
              self.fp16_groups_flat[i], self.fp16_groups[i]
          )
          for p, q in zip(self.fp16_groups[i], updated_params):
              p.data = q.data
      return self.overflow
  
imports: import torch
  from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors
  from deepspeed.runtime import DeepSpeedOptimizer
  from deepspeed.runtime.utils import get_global_norm, get_grad_norm, CheckOverflow, get_weight_norm, required_torch_version
  from deepspeed.runtime.fp16.loss_scaler import INITIAL_LOSS_SCALE, SCALE_WINDOW, MIN_LOSS_SCALE
  from deepspeed.utils import groups, logger, log_dist
  from deepspeed import comm
  from deepspeed.checkpoint.constants import OPTIMIZER_STATE_DICT, CLIP_GRAD
  from deepspeed.accelerator import get_accelerator
label_desc: Use image node_id 4 for calling the FP16_Optimizer obj's underlying member method code with example usage: obj.step_fused_adam(closure) and returns: self, self