class_docstr: ## Rectified Adam Optimizer
  
  This class extends from AMSAdam optimizer defined in [`amsadam.py`](amsadam.html).
class_docstr_tok: ["#","#","Rectified","Adam","Optimizer","This","class","extends","from","AMSAdam","optimizer","defined","in","[","`","amsadam.py","`","]","(","amsadam.html",")","."]
docstr: ### Calculate rectification term $r_t$
docstr_tok: ["#","#","#","Calculate","rectification","term","$","r_t","$"]
code: def calc_rectification_term(
      beta2: float, step: int
  ) -> Optional[float]:
      """
      ### Calculate rectification term $r_t$
      """
  
      # $\beta_2^t$
      beta2_t = beta2**step
      # $$\rho_\infty = \frac{2}{1 - \beta_2} - 1$$
      rho_inf = 2 / (1 - beta2) - 1
      # $$\rho_t = \frac{2}{1-\beta_2} - 1 - \frac{2 t \beta_2^t}{1-\beta_2^t}$$
      rho = rho_inf - 2 * step * beta2_t / (1 - beta2_t)
  
      # $r_t$ is tractable when $\rho_t >= 4$.
      # We are being a little more conservative since it's an approximated value
      if rho >= 5:
          # $$r_t = \sqrt{\frac{(\rho_t-2)(\rho_t-4)\rho_\infty}{(\rho_\infty-2)(\rho_\infty-4)\rho_t}}$$
          r2 = (
              (rho - 4)
              / (rho_inf - 4)
              * (rho - 2)
              / rho
              * rho_inf
              / (rho_inf - 2)
          )
          return math.sqrt(r2)
      else:
          return None
  
file_path: annotated_deep_learning_paper_implementations/labml_nn/optimizers/radam.py
imports: import math
  from typing import Dict, Optional
  import torch
  from labml_nn.optimizers import WeightDecay
  from labml_nn.optimizers.amsgrad import AMSGrad
label_desc: Use image node_id 3 for calling the RAdam obj's underlying member method code with example usage: obj.calc_rectification_term(beta2, step) and returns: math, None