class_docstr: Do various flavours of stochastic gradient descent, with first and
  second order momentum. Currently support 'vanilla' SGD, Adam, and RAdam.
class_docstr_tok: ["Do","various","flavours","of","stochastic","gradient","descent",",","with","first","and","second","order","momentum",".","Currently","support","'vanilla","'","SGD",",","Adam",",","and","RAdam","."]
docstr: Initialize an optimizer.
  
  learn_rate (float): The initial learning rate.
  L2 (float): The L2 regularization term.
  beta1 (float): First-order momentum.
  beta2 (float): Second-order momentum.
  eps (float): Epsilon term for Adam etc.
  grad_clip (float): Gradient clipping.
  use_averages (bool): Whether to track moving averages of the parameters.
  use_radam (bool): Whether to use the RAdam optimizer.
  L2_is_weight_decay (bool): Whether to interpret the L2 parameter as a
      weight decay term, in the style of the AdamW optimizer.
docstr_tok: ["Initialize","an","optimizer",".","learn_rate","(","float",")",":","The","initial","learning","rate",".","L2","(","float",")",":","The","L2","regularization","term",".","beta1","(","float",")",":","First-order","momentum",".","beta2","(","float",")",":","Second-order","momentum",".","eps","(","float",")",":","Epsilon","term","for","Adam","etc",".","grad_clip","(","float",")",":","Gradient","clipping",".","use_averages","(","bool",")",":","Whether","to","track","moving","averages","of","the","parameters",".","use_radam","(","bool",")",":","Whether","to","use","the","RAdam","optimizer",".","L2_is_weight_decay","(","bool",")",":","Whether","to","interpret","the","L2","parameter","as","a","weight","decay","term",",","in","the","style","of","the","AdamW","optimizer","."]
code: def __init__(
      self,
      learn_rate: FloatOrSeq,
      *,
      L2: FloatOrSeq = ADAM_DEFAULTS["L2"],
      beta1: FloatOrSeq = ADAM_DEFAULTS["beta1"],
      beta2: FloatOrSeq = ADAM_DEFAULTS["beta2"],
      eps: FloatOrSeq = ADAM_DEFAULTS["eps"],
      grad_clip: FloatOrSeq = ADAM_DEFAULTS["grad_clip"],
      use_averages: bool = True,
      use_radam: bool = False,
      L2_is_weight_decay: bool = True,
  ):
      """
      Initialize an optimizer.
  
      learn_rate (float): The initial learning rate.
      L2 (float): The L2 regularization term.
      beta1 (float): First-order momentum.
      beta2 (float): Second-order momentum.
      eps (float): Epsilon term for Adam etc.
      grad_clip (float): Gradient clipping.
      use_averages (bool): Whether to track moving averages of the parameters.
      use_radam (bool): Whether to use the RAdam optimizer.
      L2_is_weight_decay (bool): Whether to interpret the L2 parameter as a
          weight decay term, in the style of the AdamW optimizer.
      """
      self.mom1 = {}
      self.mom2 = {}
      if use_averages:
          self.averages = {}
      else:
          self.averages = None
      self.schedules = {}
      self.nr_update = defaultdict(int)
      self.last_seen = defaultdict(int)
      self._set_attr_or_schedule("grad_clip", grad_clip)
      self._set_attr_or_schedule("learn_rate", learn_rate)
      self._set_attr_or_schedule("b1", beta1)
      self._set_attr_or_schedule("b2", beta2)
      self._set_attr_or_schedule("eps", eps)
      self._set_attr_or_schedule("L2", L2)
      self.use_radam = use_radam
      self.L2_is_weight_decay = L2_is_weight_decay
      self._radam_buffer = [[None, None, None] for _ in range(10)]
  
file_path: thinc/thinc/optimizers.py
imports: import math
  from collections import defaultdict
  from typing import Dict, List, Optional, Tuple, Union, cast
  from .backends import get_array_ops
  from .config import registry
  from .types import FloatsXd, Generator
label_desc: Use image node_id 1 to create a new Optimizer object from inherited base classes: object with example: obj = Optimizer(learn_rate)