class_docstr: Implements Adam algorithm. Currently GPU-only.
     Requires Apex to be installed via
     ``python setup.py install --cuda_ext --cpp_ext``.
  
  Arguments:
      params (iterable): iterable of parameters to optimize or dicts defining
          parameter groups.
      lr (float, optional): learning rate. (default: 1e-3)
      betas (Tuple[float, float], optional): coefficients used for computing
          running averages of gradient and its square.
          (default: (0.9, 0.999))
      eps (float, optional): term added to the denominator to improve
          numerical stability. (default: 1e-8)
      weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
      amsgrad (boolean, optional): whether to use the AMSGrad variant of this
          algorithm from the paper 'On the Convergence of Adam and Beyond'
          (default: False) NOT SUPPORTED in FusedAdam!
      eps_inside_sqrt (boolean, optional): in the 'update parameters' step,
          adds eps to the bias-corrected second moment estimate before
          evaluating square root instead of adding it to the square root of
          second moment estimate as in the original paper. (default: False)
class_docstr_tok: ["Implements","Adam","algorithm",".","Currently","GPU-only",".","Requires","Apex","to","be","installed","via","``","python","setup.py","install","--","cuda_ext","--","cpp_ext","``",".","Arguments",":","params","(","iterable",")",":","iterable","of","parameters","to","optimize","or","dicts","defining","parameter","groups",".","lr","(","float",",","optional",")",":","learning","rate",".","(","default",":","1e-3",")","betas","(","Tuple","[","float",",","float","]",",","optional",")",":","coefficients","used","for","computing","running","averages","of","gradient","and","its","square",".","(","default",":","(","0.9",",","0.999",")",")","eps","(","float",",","optional",")",":","term","added","to","the","denominator","to","improve","numerical","stability",".","(","default",":","1e-8",")","weight_decay","(","float",",","optional",")",":","weight","decay","(","L2","penalty",")","(","default",":","0",")","amsgrad","(","boolean",",","optional",")",":","whether","to","use","the","AMSGrad","variant","of","this","algorithm","from","the","paper","'On","the","Convergence","of","Adam","and","Beyond'","(","default",":","False",")","NOT","SUPPORTED","in","FusedAdam","!","eps_inside_sqrt","(","boolean",",","optional",")",":","in","the","'update","parameters","'","step",",","adds","eps","to","the","bias-corrected","second","moment","estimate","before","evaluating","square","root","instead","of","adding","it","to","the","square","root","of","second","moment","estimate","as","in","the","original","paper",".","(","default",":","False",")"]
code: def __init__(
      self,
      params,
      lr=1e-3,
      bias_correction=True,
      betas=(0.9, 0.999),
      eps=1e-8,
      eps_inside_sqrt=False,
      weight_decay=0.0,
      max_grad_norm=0.0,
      amsgrad=False,
  ):
      global fused_adam_cuda
      fused_adam_cuda = importlib.import_module("fused_adam_cuda")
  
      if amsgrad:
          raise RuntimeError("AMSGrad variant not supported.")
      defaults = dict(
          lr=lr,
          bias_correction=bias_correction,
          betas=betas,
          eps=eps,
          weight_decay=weight_decay,
          max_grad_norm=max_grad_norm,
      )
      super(FusedAdam, self).__init__(params, defaults)
      self.eps_mode = 0 if eps_inside_sqrt else 1
  
imports: import torch
  import torch.optim
  from torch.nn.utils import clip_grad_norm_
  import operator
  import functools
  from copy import copy
  from math import sqrt
  import types
  import os
  import importlib
  from onmt.utils.misc import fn_args
label_desc: Use image node_id 1 to create a new FusedAdam object from inherited base classes: torch.optim with example: obj = FusedAdam(params, lr, bias_correction, betas, eps, eps_inside_sqrt, weight_decay, max_grad_norm, amsgrad)