class_docstr: Implements AdaBound algorithm with Decoupled Weight Decay (arxiv.org/abs/1711.05101)
  It has been proposed in `Adaptive Gradient Methods with Dynamic Bound of Learning Rate`_.
  Arguments:
      params (iterable): iterable of parameters to optimize or dicts defining
          parameter groups
      lr (float, optional): Adam learning rate (default: 1e-3)
      betas (Tuple[float, float], optional): coefficients used for computing
          running averages of gradient and its square (default: (0.9, 0.999))
      final_lr (float, optional): final (SGD) learning rate (default: 0.1)
      gamma (float, optional): convergence speed of the bound functions (default: 1e-3)
      eps (float, optional): term added to the denominator to improve
          numerical stability (default: 1e-8)
      weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
      amsbound (boolean, optional): whether to use the AMSBound variant of this algorithm
  .. Adaptive Gradient Methods with Dynamic Bound of Learning Rate:
      https://openreview.net/forum?id=Bkg3g2R9FX
class_docstr_tok: ["Implements","AdaBound","algorithm","with","Decoupled","Weight","Decay","(","arxiv.org\/abs\/1711.05101",")","It","has","been","proposed","in","`","Adaptive","Gradient","Methods","with","Dynamic","Bound","of","Learning","Rate","`","_",".","Arguments",":","params","(","iterable",")",":","iterable","of","parameters","to","optimize","or","dicts","defining","parameter","groups","lr","(","float",",","optional",")",":","Adam","learning","rate","(","default",":","1e-3",")","betas","(","Tuple","[","float",",","float","]",",","optional",")",":","coefficients","used","for","computing","running","averages","of","gradient","and","its","square","(","default",":","(","0.9",",","0.999",")",")","final_lr","(","float",",","optional",")",":","final","(","SGD",")","learning","rate","(","default",":","0.1",")","gamma","(","float",",","optional",")",":","convergence","speed","of","the","bound","functions","(","default",":","1e-3",")","eps","(","float",",","optional",")",":","term","added","to","the","denominator","to","improve","numerical","stability","(","default",":","1e-8",")","weight_decay","(","float",",","optional",")",":","weight","decay","(","L2","penalty",")","(","default",":","0",")","amsbound","(","boolean",",","optional",")",":","whether","to","use","the","AMSBound","variant","of","this","algorithm","..","Adaptive","Gradient","Methods","with","Dynamic","Bound","of","Learning","Rate",":","https",":","\/\/openreview.net\/forum","?","id=Bkg3g2R9FX"]
code: def __init__(
      self,
      params,
      lr=1e-3,
      betas=(0.9, 0.999),
      final_lr=0.1,
      gamma=1e-3,
      eps=1e-8,
      weight_decay=0,
      amsbound=False,
  ):
      if not 0.0 <= lr:
          raise ValueError("Invalid learning rate: {}".format(lr))
      if not 0.0 <= eps:
          raise ValueError("Invalid epsilon value: {}".format(eps))
      if not 0.0 <= betas[0] < 1.0:
          raise ValueError(
              "Invalid beta parameter at index 0: {}".format(betas[0])
          )
      if not 0.0 <= betas[1] < 1.0:
          raise ValueError(
              "Invalid beta parameter at index 1: {}".format(betas[1])
          )
      if not 0.0 <= final_lr:
          raise ValueError(
              "Invalid final learning rate: {}".format(final_lr)
          )
      if not 0.0 <= gamma < 1.0:
          raise ValueError("Invalid gamma parameter: {}".format(gamma))
      defaults = dict(
          lr=lr,
          betas=betas,
          final_lr=final_lr,
          gamma=gamma,
          eps=eps,
          weight_decay=weight_decay,
          amsbound=amsbound,
      )
      super(AdaBoundW, self).__init__(params, defaults)
  
      self.base_lrs = list(
          map(lambda group: group["lr"], self.param_groups)
      )
  
file_path: AdaBound/adabound/adabound.py
imports: import math
  import torch
  from torch.optim import Optimizer
label_desc: Use image node_id 1 to create a new AdaBoundW object from inherited base classes: Optimizer with example: obj = AdaBoundW(params, lr, betas, final_lr, gamma, eps, weight_decay, amsbound)