class_docstr: Adaptive Moment Estimation (Adam) optimizer.
  
  Adam is an algorithm for first-order gradient-based optimization of stochastic
  objective functions, based on adaptive estimates of lower-order moments. See
  :cite:`kingma2014adam` for more details.
  
  Note: default parameter values have been taken from the paper.
  
  Attributes:
    learning_rate: Step size (``alpha`` in the paper).
    beta1: Exponential decay rate for first moment estimate.
    beta2: Exponential decay rate for second moment estimate.
    epsilon: Small value to avoid zero denominator.
    step: Step count.
    m: Biased first moment estimate (a list with one value per parameter).
    v: Biased second raw moment estimate (a list with one value per parameter).
class_docstr_tok: ["Adaptive","Moment","Estimation","(","Adam",")","optimizer",".","Adam","is","an","algorithm","for","first-order","gradient-based","optimization","of","stochastic","objective","functions",",","based","on","adaptive","estimates","of","lower-order","moments",".","See",":","cite",":","`","kingma2014adam","`","for","more","details",".","Note",":","default","parameter","values","have","been","taken","from","the","paper",".","Attributes",":","learning_rate",":","Step","size","(","``","alpha","``","in","the","paper",")",".","beta1",":","Exponential","decay","rate","for","first","moment","estimate",".","beta2",":","Exponential","decay","rate","for","second","moment","estimate",".","epsilon",":","Small","value","to","avoid","zero","denominator",".","step",":","Step","count",".","m",":","Biased","first","moment","estimate","(","a","list","with","one","value","per","parameter",")",".","v",":","Biased","second","raw","moment","estimate","(","a","list","with","one","value","per","parameter",")","."]
docstr: Applies updates to parameters.
  
  Applies the Adam update rule for each update, parameter pair:
  
  .. math::
  
     \begin{array}{ll}
     m_t = \beta_1 \cdot m_{t-1} + (1 - \beta_1) \cdot update \\
     v_t = \beta_2 \cdot v_{t-1} + (1 - \beta_2) \cdot update^2 \\
     \hat{m}_t = m_t / (1 - \beta_1^t) \\
     \hat{v}_t = v_t / (1 - \beta_2^t) \\
     delta = \alpha \cdot \hat{m}_t / (\sqrt{\hat{v}_t} + \epsilon) \\
     param_t = param_{t-1} - delta \\
     \end{array}
  
  Args:
    updates: A list of updates to apply to parameters. Updates are often
      gradients as returned by :tf:`GradientTape.gradient`.
    parameters: A list of parameters.
  
  Raises:
    ValueError: If `updates` and `parameters` are empty, have different
      lengths, or have inconsistent types.
docstr_tok: ["Applies","updates","to","parameters",".","Applies","the","Adam","update","rule","for","each","update",",","parameter","pair",":","..","math",":",":","\\begin","{","array","}","{","ll","}","m_t","=","\\beta_1","\\cdot","m_","{","t-1","}","+","(","1","-","\\beta_1",")","\\cdot","update","\\\\","v_t","=","\\beta_2","\\cdot","v_","{","t-1","}","+","(","1","-","\\beta_2",")","\\cdot","update^2","\\\\","\\hat","{","m","}","_t","=","m_t","\/","(","1","-","\\beta_1^t",")","\\\\","\\hat","{","v","}","_t","=","v_t","\/","(","1","-","\\beta_2^t",")","\\\\","delta","=","\\alpha","\\cdot","\\hat","{","m","}","_t","\/","(","\\sqrt","{","\\hat","{","v","}","_t","}","+","\\epsilon",")","\\\\","param_t","=","param_","{","t-1","}","-","delta","\\\\","\\end","{","array","}","Args",":","updates",":","A","list","of","updates","to","apply","to","parameters",".","Updates","are","often","gradients","as","returned","by",":","tf",":","`","GradientTape.gradient","`",".","parameters",":","A","list","of","parameters",".","Raises",":","ValueError",":","If","`","updates","`","and","`","parameters","`","are","empty",",","have","different","lengths",",","or","have","inconsistent","types","."]
code: def apply(
      self,
      updates: Sequence[types.ParameterUpdate],
      parameters: Sequence[tf.Variable],
  ):
      r"""Applies updates to parameters.
  
      Applies the Adam update rule for each update, parameter pair:
  
      .. math::
  
         \begin{array}{ll}
         m_t = \beta_1 \cdot m_{t-1} + (1 - \beta_1) \cdot update \\
         v_t = \beta_2 \cdot v_{t-1} + (1 - \beta_2) \cdot update^2 \\
         \hat{m}_t = m_t / (1 - \beta_1^t) \\
         \hat{v}_t = v_t / (1 - \beta_2^t) \\
         delta = \alpha \cdot \hat{m}_t / (\sqrt{\hat{v}_t} + \epsilon) \\
         param_t = param_{t-1} - delta \\
         \end{array}
  
      Args:
        updates: A list of updates to apply to parameters. Updates are often
          gradients as returned by :tf:`GradientTape.gradient`.
        parameters: A list of parameters.
  
      Raises:
        ValueError: If `updates` and `parameters` are empty, have different
          lengths, or have inconsistent types.
      """
      optimizer_utils.check_distribution_strategy()
      optimizer_utils.check_updates_parameters(updates, parameters)
      self._initialize(parameters)
      self.step.assign_add(1)
      for update, param, m_var, v_var in zip(
          updates, parameters, self.m, self.v
      ):
          if update is None:
              continue
  
          optimizer_utils.check_same_dtype(update, param)
          learning_rate = tf.cast(self.learning_rate, update.dtype)
          beta_1 = tf.cast(self.beta1, update.dtype)
          beta_2 = tf.cast(self.beta2, update.dtype)
          epsilon = tf.cast(self.epsilon, update.dtype)
          step = tf.cast(self.step, update.dtype)
  
          if isinstance(update, tf.IndexedSlices):
              # Sparse read our state.
              (
                  update,
                  indices,
              ) = optimizer_utils.deduplicate_indexed_slices(update)
              m = m_var.sparse_read(indices)
              v = v_var.sparse_read(indices)
  
              # Compute and apply a sparse update to our parameter and state.
              update, m, v = adam_update(
                  g=update,
                  alpha=learning_rate,
                  beta_1=beta_1,
                  beta_2=beta_2,
                  epsilon=epsilon,
                  t=step,
                  m=m,
                  v=v,
              )
              param.scatter_sub(tf.IndexedSlices(update, indices))
              m_var.scatter_update(tf.IndexedSlices(m, indices))
              v_var.scatter_update(tf.IndexedSlices(v, indices))
  
          else:
              # Compute and apply a dense update to our parameter and state.
              update, m, v = adam_update(
                  g=update,
                  alpha=learning_rate,
                  beta_1=beta_1,
                  beta_2=beta_2,
                  epsilon=epsilon,
                  t=step,
                  m=m_var,
                  v=v_var,
              )
              param.assign_sub(update)
              m_var.assign(m)
              v_var.assign(v)
  
file_path: sonnet/sonnet/src/optimizers/adam.py
imports: from typing import Optional, Sequence, Union
  from sonnet.src import base
  from sonnet.src import once
  from sonnet.src import types
  from sonnet.src import utils
  from sonnet.src.optimizers import optimizer_utils
  import tensorflow
label_desc: Use image node_id 3 for calling the Adam obj's underlying member method code with example usage: obj.apply(updates, parameters) without return types