class_docstr: ## Adam Optimizer
  
  We extend the class `GenericAdaptiveOptimizer` defined in [`__init__.py`](index.html)
  to implement the Adam optimizer.
docstr: ### Calculate $m_t$ and and $v_t$
  
  * `state` is the optimizer state of the parameter (tensor)
  * `group` stores optimizer attributes of the parameter group
  * `grad` is the current gradient tensor $g_t$ for the parameter $       heta_{t-1}$
docstr_tok: ["#","#","#","Calculate","$","m_t","$","and","and","$","v_t","$","*","`","state","`","is","the","optimizer","state","of","the","parameter","(","tensor",")","*","`","group","`","stores","optimizer","attributes","of","the","parameter","group","*","`","grad","`","is","the","current","gradient","tensor","$","g_t","$","for","the","parameter","$","heta_","{","t-1","}","$"]
code: def get_mv(
      self,
      state: Dict[str, Any],
      group: Dict[str, Any],
      grad: torch.Tensor,
  ):
      """
      ### Calculate $m_t$ and and $v_t$
  
      * `state` is the optimizer state of the parameter (tensor)
      * `group` stores optimizer attributes of the parameter group
      * `grad` is the current gradient tensor $g_t$ for the parameter $\theta_{t-1}$
      """
  
      # Get $\beta_1$ and $\beta_2$
      beta1, beta2 = group["betas"]
  
      # Get $m_{t-1}$ and $v_{t-1}$
      m, v = state["exp_avg"], state["exp_avg_sq"]
  
      # In-place calculation of $m_t$
      # $$m_t \leftarrow \beta_1 m_{t-1} + (1 - \beta_1) \cdot g_t$$
      m.mul_(beta1).add_(grad, alpha=1 - beta1)
      # In-place calculation of $v_t$
      # $$v_t \leftarrow \beta_2 v_{t-1} + (1 - \beta_2) \cdot g_t^2$$
      v.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
  
      return m, v
  
file_path: annotated_deep_learning_paper_implementations/labml_nn/optimizers/adam.py
imports: import math
  from typing import Dict, Any, Tuple, Optional
  import torch
  from labml import tracker
  from torch import nn
  from labml_nn.optimizers import GenericAdaptiveOptimizer, WeightDecay
label_desc: Use image node_id 3 for calling the Adam obj's underlying member method code with example usage: obj.get_mv(state, group, grad) and returns: m,  v