class_docstr: ## Adam Optimizer for Half Precision Training
  
  We extend [Adam Optimizer](adam.html) but use FP32 to store gradients and moments.
class_docstr_tok: ["#","#","Adam","Optimizer","for","Half","Precision","Training","We","extend","[","Adam","Optimizer","]","(","adam.html",")","but","use","FP32","to","store","gradients","and","moments","."]
code: def __init__(
      self,
      params,
      lr: float = 1e-3,
      betas: Tuple[float, float] = (0.9, 0.999),
      eps: float = 1e-16,
      weight_decay: WeightDecay = WeightDecay(),
      optimized_update: bool = True,
      defaults: Optional[Dict[str, Any]] = None,
  ):
      # Parameter to store 32 bit gradients. This get populated by the `GradScaler` defined below.
      self.grad_fp32 = {}
      # Call the [Adam Optimizer](adam.html) initializer
      super().__init__(
          params,
          lr,
          betas,
          eps,
          weight_decay,
          optimized_update,
          defaults,
      )
  
file_path: annotated_deep_learning_paper_implementations/labml_nn/optimizers/adam_fp16.py
imports: from typing import Dict, Tuple, Optional, Any
  import torch
  from torch import nn
  from torch.optim import Optimizer
  from torch.cuda.amp import grad_scaler
  from collections import defaultdict, abc
  from labml_nn.optimizers import WeightDecay
  from labml_nn.optimizers.adam import Adam
label_desc: Use image node_id 1 to create a new AdamFP16 object from inherited base classes: Adam with example: obj = AdamFP16(params, lr, betas, eps, weight_decay, optimized_update, defaults)