class_docstr: Performs loss scaling during backward pass to prevent underflow of half
  precision gradients.
class_docstr_tok: ["Performs","loss","scaling","during","backward","pass","to","prevent","underflow","of","half","precision","gradients","."]
docstr: :param optimizer:
      The optimizer that holds the gradients that will be unscaled.
  :param gang:
      The associated gang.
  :param init_scale:
      The initial scale.
  :param scale_factor:
      The factor by which the scale is multiplied if no inf/NaN gradients
      occur for ``scale_window`` consecutive optimizer steps.
  :param scale_window:
      The number of consecutive optimizer steps without inf/NaN gradients
      that must occur for the scale to be multiplied by ``scale_factor``.
  :param min_scale:
      The minimum allowed scale.
  :param logger:
      The logger to output diagnostic messages.
  :param enabled:
      If ``False``, disables loss scaling.
docstr_tok: [":","param","optimizer",":","The","optimizer","that","holds","the","gradients","that","will","be","unscaled",".",":","param","gang",":","The","associated","gang",".",":","param","init_scale",":","The","initial","scale",".",":","param","scale_factor",":","The","factor","by","which","the","scale","is","multiplied","if","no","inf\/NaN","gradients","occur","for","``","scale_window","``","consecutive","optimizer","steps",".",":","param","scale_window",":","The","number","of","consecutive","optimizer","steps","without","inf\/NaN","gradients","that","must","occur","for","the","scale","to","be","multiplied","by","``","scale_factor","``",".",":","param","min_scale",":","The","minimum","allowed","scale",".",":","param","logger",":","The","logger","to","output","diagnostic","messages",".",":","param","enabled",":","If","``","False","``",",","disables","loss","scaling","."]
code: def __init__(
      self,
      optimizer: Optimizer,
      gang: Gang,
      *,
      init_scale: float = 2.0**15,
      scale_factor: float = 2.0,
      scale_window: int = 2000,
      min_scale: float = 0.0,
      logger: Optional[Logger] = None,
      enabled: bool = True,
  ) -> None:
      """
      :param optimizer:
          The optimizer that holds the gradients that will be unscaled.
      :param gang:
          The associated gang.
      :param init_scale:
          The initial scale.
      :param scale_factor:
          The factor by which the scale is multiplied if no inf/NaN gradients
          occur for ``scale_window`` consecutive optimizer steps.
      :param scale_window:
          The number of consecutive optimizer steps without inf/NaN gradients
          that must occur for the scale to be multiplied by ``scale_factor``.
      :param min_scale:
          The minimum allowed scale.
      :param logger:
          The logger to output diagnostic messages.
      :param enabled:
          If ``False``, disables loss scaling.
      """
      if enabled:
          for group in optimizer.param_groups:
              for param in group["params"]:
                  if param.dtype != torch.float16:
                      raise ValueError(
                          f"The parameters held by `optimizer` must be of type `torch.float16`, but at least one parameter is of type `{param.dtype}`."
                      )
  
      if gang.size == 1:
          self._grad_scaler = GradScaler(
              init_scale,
              scale_factor,
              1 / scale_factor,
              scale_window,
              enabled,
          )
      else:
          pg = gang.as_process_group()
  
          # Yes, `growth_factor` and `backoff_factor` parameters are swapped.
          self._grad_scaler = ShardedGradScaler(
              init_scale,
              1 / scale_factor,
              scale_factor,
              scale_window,
              enabled,
              pg,
          )
  
      self.optimizer = optimizer
      self.init_scale = init_scale
      self.scale_factor = scale_factor
      self.scale_window = scale_window
      self.min_scale = min_scale
      self.gang = gang
      self.logger = logger
      self.enabled = enabled
  
imports: from __future__ import annotations
  import logging
  import math
  from dataclasses import dataclass
  from logging import Logger
  from typing import Any, Callable, Dict, Mapping, Optional, Tuple, cast
  import torch
  from torch import Tensor
  from torch.cuda.amp.grad_scaler import GradScaler
  from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler
  from torch.optim import Optimizer
  from fairseq2.gang import Gang
label_desc: Use image node_id 1 to create a new DynamicLossScaler object  with example: obj = DynamicLossScaler(optimizer, gang)