class_docstr: FP16 Optimizer without weight fusion to support LAMB optimizer
  
  For usage example please see, TODO:  DeepSpeed V2 Tutorial
class_docstr_tok: ["FP16","Optimizer","without","weight","fusion","to","support","LAMB","optimizer","For","usage","example","please","see",",","TODO",":","DeepSpeed","V2","Tutorial"]
code: def __init__(
      self,
      init_optimizer,
      deepspeed=None,
      static_loss_scale=1.0,
      dynamic_loss_scale=False,
      dynamic_loss_args=None,
      verbose=True,
      mpu=None,
      clip_grad=0.0,
      fused_lamb_legacy=False,
  ):
      self.fused_lamb_legacy = fused_lamb_legacy
      self._global_grad_norm = 0.0
  
      if dist.get_rank() == 0:
          logger.info(f"Fused Lamb Legacy : {self.fused_lamb_legacy} ")
  
      if not torch.cuda.is_available:
          raise SystemError("Cannot use fp16 without CUDA.")
      self.optimizer = init_optimizer
  
      # param groups
      self.fp16_groups = []
      self.fp32_groups = []
  
      # loop to deal with groups
      for i, param_group in enumerate(self.optimizer.param_groups):
          # fp16 weights that represents the actual model weights
          self.fp16_groups.append(param_group["params"])
  
          # creating a fp32 copy of the weights that will be updated first then
          # copied to fp16 weights
          fp32_group = [
              p.clone().float().detach() for p in param_group["params"]
          ]
  
          # in case the internal optimizer needs it
          for p in fp32_group:
              p.requires_grad = True
  
          # setting the param groups in the optimizer to point to fp32
          # note these are not the weights used by the model
          # the model uses the fp16 version that we added to fp16_group
          self.fp32_groups.append(fp32_group)
          param_group["params"] = self.fp32_groups[i]
  
      # we may have a way of fusing dynamic scale. Do not support for now
      if dynamic_loss_scale:
          self.dynamic_loss_scale = True
          self.cur_iter = 0
          self.last_overflow_iter = -1
          self.scale_factor = 2.0
          if dynamic_loss_args is None:
              self.cur_scale = 1.0 * 2**16
              self.scale_window = 1000
              self.min_loss_scale = 0.25
          else:
              self.cur_scale = dynamic_loss_args[INITIAL_LOSS_SCALE]
              self.scale_window = dynamic_loss_args[SCALE_WINDOW]
              self.min_loss_scale = dynamic_loss_args[MIN_LOSS_SCALE]
      else:
          self.dynamic_loss_scale = False
          self.cur_iter = 0
          self.cur_scale = static_loss_scale
  
      self.custom_loss_scaler = False
      self.external_loss_scale = None
  
      self.verbose = verbose
  
      self.clip_grad = clip_grad
      self.norm_type = 2
  
      TORCH_MAJOR = int(torch.__version__.split(".")[0])
      TORCH_MINOR = int(torch.__version__.split(".")[1])
      if TORCH_MAJOR == 0 and TORCH_MINOR <= 4:
          self.clip_grad_norm = torch.nn.utils.clip_grad_norm
      else:
          self.clip_grad_norm = torch.nn.utils.clip_grad_norm_
  
      self.mpu = mpu
  
      self.overflow = False
      self.overflow_checker = CheckOverflow(
          self.fp16_groups, mpu=self.mpu, deepspeed=deepspeed
      )
  
      self.initialize_optimizer_states()
  
imports: from deepspeed.moe.utils import split_params_grads_into_shared_and_expert_params
  import torch
  from torch._utils import _flatten_dense_tensors
  from deepspeed.runtime import DeepSpeedOptimizer
  from deepspeed.runtime.utils import get_global_norm, CheckOverflow, get_weight_norm
  from deepspeed.runtime.fp16.loss_scaler import INITIAL_LOSS_SCALE, SCALE_WINDOW, MIN_LOSS_SCALE
  from deepspeed.utils import logger
  from deepspeed.checkpoint.constants import OPTIMIZER_STATE_DICT
  from deepspeed import comm
label_desc: Use image node_id 1 to create a new FP16_UnfusedOptimizer object from inherited base classes: DeepSpeedOptimizer with example: obj = FP16_UnfusedOptimizer(init_optimizer, deepspeed, static_loss_scale, dynamic_loss_scale, dynamic_loss_args, verbose, mpu, clip_grad, fused_lamb_legacy)