class_docstr: ## Noam Optimizer
  
  This class extends from Adam optimizer defined in [`adam.py`](adam.html).
docstr: ### Get learning-rate
  
  $$lpha rac{1}{\sqrt{d_{model}}} \min igg(rac{1}{\sqrt{t}}, rac{t}{w^{3/2}}igg)$$
  where $w$ is the number of warmup steps.
docstr_tok: ["#","#","#","Get","learning-rate","$","$","\u0007lpha","rac","{","1","}","{","\\sqrt","{","d_","{","model","}","}","}","\\min","\bigg","(","rac","{","1","}","{","\\sqrt","{","t","}","}",",","rac","{","t","}","{","w^","{","3\/2","}","}","\bigg",")","$","$","where","$","w","$","is","the","number","of","warmup","steps","."]
code: def get_lr(self, state: Dict[str, any], group: Dict[str, any]):
      """
      ### Get learning-rate
  
      $$\alpha \frac{1}{\sqrt{d_{model}}} \min \bigg(\frac{1}{\sqrt{t}}, \frac{t}{w^{3/2}}\bigg)$$
      where $w$ is the number of warmup steps.
      """
      # $$\min \bigg(\frac{1}{\sqrt{t}}, \frac{t}{w^{3/2}}\bigg)$$
      factor = min(
          state["step"] ** (-0.5),
          state["step"] * group["warmup"] ** (-1.5),
      )
      # $$\alpha \frac{1}{\sqrt{d_{model}}} \min \bigg(\frac{1}{\sqrt{t}}, \frac{t}{w^{3/2}}\bigg)$$
      return group["lr"] * self.d_model ** (-0.5) * factor
  
file_path: annotated_deep_learning_paper_implementations/labml_nn/optimizers/noam.py
imports: from typing import Dict
  from labml_nn.optimizers import WeightDecay
  from labml_nn.optimizers.amsgrad import AMSGrad
label_desc: Use image node_id 2 for calling the Noam obj's underlying member method code with example usage: obj.get_lr(state, group) and returns: unknown