class_docstr: :class:`FP16_Optimizer` A cutdown version of apex.fp16_utils.FP16_Optimizer.
  Designed only to wrap apex.contrib.optimizers.FusedAdam, FusedSGD.
  Refer to apex.fp16_utils documents for more information.
  Example::
      model = torch.nn.Linear(D_in, D_out).cuda().half()
      optimizer = apex.contrib.optimizers.FusedSGD(model.parameters())
      optimizer = FP16_Optimizer(optimizer, static_loss_scale = 128.0)
      ...
      # loss.backward() becomes:
      optimizer.backward(loss)
      ...
  Example with dynamic loss scaling::
      ...
      optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)
                                 # optional arg to control dynamic loss scaling behavior
                                 # dynamic_loss_args={'scale_window' : 500})
                                 # Usually, dynamic_loss_args is not necessary.
class_docstr_tok: [":","class",":","`","FP16_Optimizer","`","A","cutdown","version","of","apex.fp16_utils.FP16_Optimizer",".","Designed","only","to","wrap","apex.contrib.optimizers.FusedAdam",",","FusedSGD",".","Refer","to","apex.fp16_utils","documents","for","more","information",".","Example",":",":","model","=","torch.nn.Linear","(","D_in",",","D_out",")",".cuda","(",")",".half","(",")","optimizer","=","apex.contrib.optimizers.FusedSGD","(","model.parameters","(",")",")","optimizer","=","FP16_Optimizer","(","optimizer",",","static_loss_scale","=","128.0",")","...","#","loss.backward","(",")","becomes",":","optimizer.backward","(","loss",")","...","Example","with","dynamic","loss","scaling",":",":","...","optimizer","=","FP16_Optimizer","(","optimizer",",","dynamic_loss_scale=True",")","#","optional","arg","to","control","dynamic","loss","scaling","behavior","#","dynamic_loss_args=","{","'scale_window","'",":","500","}",")","#","Usually",",","dynamic_loss_args","is","not","necessary","."]
code: def __init__(
      self,
      init_optimizer,
      static_loss_scale=1.0,
      dynamic_loss_scale=False,
      dynamic_loss_args=None,
      verbose=True,
  ):
      print(
          "\nThis fp16_optimizer is designed to only work with apex.contrib.optimizers.*"
      )
      print("To update, use updated optimizers with AMP.")
      # The fused optimizer does all the work. We need this layer for two reason:
      # 1. maintain same user API from apex.fp16_utils
      # 2. keep common stuff here in case we need to add new fused optimizer later
  
      if not torch.cuda.is_available:
          raise SystemError("Cannot use fp16 without CUDA.")
      self.optimizer = init_optimizer
  
      self.fp16_groups = []  # model params
      self.fp32_groups = []  # master weights
  
      # iterate over param_groups
      for param_group in self.optimizer.param_groups:
          fp16_group = []
          fp32_group = []
          for p in param_group["params"]:
              fp16_group.append(p)
              fp32_group.append(p.clone().float().detach())
          self.fp16_groups.append(fp16_group)
          self.fp32_groups.append(fp32_group)
          param_group["params"] = fp32_group
  
      if multi_tensor_applier.available:
          import amp_C
  
          self.overflow_buf = torch.cuda.IntTensor([0])
          self.multi_tensor_l2norm = amp_C.multi_tensor_l2norm
      else:
          raise RuntimeError("FP16_Optimizer requires cuda extensions")
  
      # we may have a way of fusing dynamic scale. Do not support for now
      if dynamic_loss_scale:
          if dynamic_loss_args is not None:
              raise SystemError(
                  "Do not support dynamic loss scale args for now."
              )
          self.dynamic_loss_scale = True
          self.cur_scale = 2**16
          self.cur_iter = 0
          self.last_overflow_iter = -1
          self.scale_factor = 2
          self.scale_window = 1000
      else:
          self.dynamic_loss_scale = False
          self.cur_iter = 0
          self.cur_scale = static_loss_scale
      self.verbose = verbose
  
file_path: apex/apex/contrib/optimizers/fp16_optimizer.py
imports: import torch
  from apex.multi_tensor_apply import multi_tensor_applier
label_desc: Use image node_id 1 to create a new FP16_Optimizer object from inherited base classes: object with example: obj = FP16_Optimizer(init_optimizer, static_loss_scale, dynamic_loss_scale, dynamic_loss_args, verbose)