class_docstr: <a id="EmbeddingsWithPositionalEncoding"></a>
  
  ## Adam Optimizer with Warmup and Cosine Decay
  
  This class extends from AMSGrad optimizer defined in [`amsgrad.py`](amsgrad.html).
docstr: ### Get learning-rate
  
  $$lpha \min igg(1, rac{t}{w}igg)$$
  where $w$ is the number of warmup steps.
code: def get_lr(self, state: Dict[str, any], group: Dict[str, any]):
      """
      ### Get learning-rate
  
      $$\alpha \min \bigg(1, \frac{t}{w}\bigg)$$
      where $w$ is the number of warmup steps.
      """
      # If we are in warmup stage
      if group["warmup"] > state["step"]:
          # A linearly increasing learning rate from $0$ to $\alpha$
          return 1e-8 + state["step"] * group["lr"] / group["warmup"]
      else:
          # Constant learning rate $\alpha$
          progress = (state["step"] - group["warmup"]) / max(
              1, group["total_steps"] - group["warmup"]
          )
          return group["lr"] * max(
              0.1, 0.5 * (1.0 + math.cos(math.pi * progress))
          )
  
file_path: annotated_deep_learning_paper_implementations/labml_nn/optimizers/adam_warmup_cosine_decay.py
imports: import math
  from typing import Dict
  from labml_nn.optimizers import WeightDecay
  from labml_nn.optimizers.amsgrad import AMSGrad
label_desc: Use image node_id 2 for calling the AdamWarmupCosineDecay obj's underlying member method code with example usage: obj.get_lr(state, group) and returns: unknown, unknown