class_docstr: Optimizer that implements the Layer-wise Adaptive Moments (LAMB).
  
  See paper [Large Batch Optimization for Deep Learning: Training BERT
  in 76 minutes](https://arxiv.org/abs/1904.00962).
class_docstr_tok: ["Optimizer","that","implements","the","Layer-wise","Adaptive","Moments","(","LAMB",")",".","See","paper","[","Large","Batch","Optimization","for","Deep","Learning",":","Training","BERT","in","76","minutes","]","(","https",":","\/\/arxiv.org\/abs\/1904.00962",")","."]
docstr: Construct a new LAMB optimizer.
  
  Args:
      learning_rate: A `Tensor` or a floating point value. or a schedule
          that is a `tf.keras.optimizers.schedules.LearningRateSchedule`
          The learning rate.
      beta_1: A `float` value or a constant `float` tensor.
        The exponential decay rate for the 1st moment estimates.
      beta_2: A `float` value or a constant `float` tensor.
        The exponential decay rate for the 2nd moment estimates.
      epsilon: A small constant for numerical stability.
      weight_decay: weight decay.
      exclude_from_weight_decay: List of regex patterns of
        variables excluded from weight decay. Variables whose name
        contain a substring matching the pattern will be excluded.
      exclude_from_layer_adaptation: List of regex patterns of
        variables excluded from layer adaptation. Variables whose name
        contain a substring matching the pattern will be excluded.
      name: Optional name for the operations created when applying
        gradients. Defaults to "LAMB".
      **kwargs: keyword arguments. Allowed to be {`clipnorm`,
        `clipvalue`, `lr`, `decay`}. `clipnorm` is clip gradients by
        norm; `clipvalue` is clip gradients by value, `decay` is
        included for backward compatibility to allow time inverse
        decay of learning rate. `lr` is included for backward
        compatibility, recommended to use `learning_rate` instead.
docstr_tok: ["Construct","a","new","LAMB","optimizer",".","Args",":","learning_rate",":","A","`","Tensor","`","or","a","floating","point","value",".","or","a","schedule","that","is","a","`","tf.keras.optimizers.schedules.LearningRateSchedule","`","The","learning","rate",".","beta_1",":","A","`","float","`","value","or","a","constant","`","float","`","tensor",".","The","exponential","decay","rate","for","the","1st","moment","estimates",".","beta_2",":","A","`","float","`","value","or","a","constant","`","float","`","tensor",".","The","exponential","decay","rate","for","the","2nd","moment","estimates",".","epsilon",":","A","small","constant","for","numerical","stability",".","weight_decay",":","weight","decay",".","exclude_from_weight_decay",":","List","of","regex","patterns","of","variables","excluded","from","weight","decay",".","Variables","whose","name","contain","a","substring","matching","the","pattern","will","be","excluded",".","exclude_from_layer_adaptation",":","List","of","regex","patterns","of","variables","excluded","from","layer","adaptation",".","Variables","whose","name","contain","a","substring","matching","the","pattern","will","be","excluded",".","name",":","Optional","name","for","the","operations","created","when","applying","gradients",".","Defaults","to","``","LAMB","''",".","*","*","kwargs",":","keyword","arguments",".","Allowed","to","be","{","`","clipnorm","`",",","`","clipvalue","`",",","`","lr","`",",","`","decay","`","}",".","`","clipnorm","`","is","clip","gradients","by","norm",";","`","clipvalue","`","is","clip","gradients","by","value",",","`","decay","`","is","included","for","backward","compatibility","to","allow","time","inverse","decay","of","learning","rate",".","`","lr","`","is","included","for","backward","compatibility",",","recommended","to","use","`","learning_rate","`","instead","."]
code: def __init__(
      self,
      learning_rate: Union[FloatTensorLike, Callable] = 0.001,
      beta_1: FloatTensorLike = 0.9,
      beta_2: FloatTensorLike = 0.999,
      epsilon: FloatTensorLike = 1e-6,
      weight_decay: FloatTensorLike = 0.0,
      exclude_from_weight_decay: Optional[List[str]] = None,
      exclude_from_layer_adaptation: Optional[List[str]] = None,
      name: str = "LAMB",
      **kwargs,
  ):
      """Construct a new LAMB optimizer.
  
      Args:
          learning_rate: A `Tensor` or a floating point value. or a schedule
              that is a `tf.keras.optimizers.schedules.LearningRateSchedule`
              The learning rate.
          beta_1: A `float` value or a constant `float` tensor.
            The exponential decay rate for the 1st moment estimates.
          beta_2: A `float` value or a constant `float` tensor.
            The exponential decay rate for the 2nd moment estimates.
          epsilon: A small constant for numerical stability.
          weight_decay: weight decay.
          exclude_from_weight_decay: List of regex patterns of
            variables excluded from weight decay. Variables whose name
            contain a substring matching the pattern will be excluded.
          exclude_from_layer_adaptation: List of regex patterns of
            variables excluded from layer adaptation. Variables whose name
            contain a substring matching the pattern will be excluded.
          name: Optional name for the operations created when applying
            gradients. Defaults to "LAMB".
          **kwargs: keyword arguments. Allowed to be {`clipnorm`,
            `clipvalue`, `lr`, `decay`}. `clipnorm` is clip gradients by
            norm; `clipvalue` is clip gradients by value, `decay` is
            included for backward compatibility to allow time inverse
            decay of learning rate. `lr` is included for backward
            compatibility, recommended to use `learning_rate` instead.
      """
  
      if "weight_decay_rate" in kwargs:
          warnings.warn(
              "weight_decay_rate has been renamed to weight_decay,"
              "and will be deprecated in Addons 0.18.",
              DeprecationWarning,
          )
          weight_decay = kwargs["weight_decay_rate"]
          del kwargs["weight_decay_rate"]
  
      super().__init__(name, **kwargs)
  
      # Just adding the square of the weights to the loss function is *not*
      # the correct way of using L2 regularization/weight decay with Adam,
      # since that will interact with the m and v parameters in strange ways.
      #
      # Instead we want to decay the weights in a manner that doesn't interact
      # with the m/v parameters.
      self._set_hyper("weight_decay", weight_decay)
      self._set_hyper("learning_rate", kwargs.get("lr", learning_rate))
  
      # This is learning rate decay for using keras learning rate schedule.
      self._set_hyper("decay", self._initial_decay)
      self._set_hyper("beta_1", beta_1)
      self._set_hyper("beta_2", beta_2)
      self.epsilon = epsilon or tf.backend_config.epsilon()
      self.exclude_from_weight_decay = exclude_from_weight_decay
      # exclude_from_layer_adaptation is set to exclude_from_weight_decay if
      # the arg is None.
      if exclude_from_layer_adaptation:
          self.exclude_from_layer_adaptation = (
              exclude_from_layer_adaptation
          )
      else:
          self.exclude_from_layer_adaptation = exclude_from_weight_decay
  
file_path: addons/tensorflow_addons/optimizers/lamb.py
imports: import warnings
  from typing import Optional, Union, Callable, List
  from typeguard import typechecked
  import tensorflow
  from tensorflow_addons.optimizers import KerasLegacyOptimizer
  from tensorflow_addons.utils.types import FloatTensorLike
  from tensorflow_addons.optimizers.utils import is_variable_matched_by_regexes
label_desc: Use image node_id 1 to create a new LAMB object from inherited base classes: KerasLegacyOptimizer with example: obj = LAMB(learning_rate, beta_1, beta_2, epsilon, weight_decay, exclude_from_weight_decay, exclude_from_layer_adaptation, name)