class_docstr: Checks for overflow in gradient across parallel process
class_docstr_tok: ["Checks","for","overflow","in","gradient","across","parallel","process"]
code: def __init__(
      self,
      param_groups=None,
      mpu=None,
      zero_reduce_scatter=False,
      deepspeed=None,
  ):
      self.mpu = mpu
      self.params = [] if param_groups else None
      self.zero_reduce_scatter = zero_reduce_scatter
      self.deepspeed = deepspeed
      self.has_moe_params = False
      if param_groups:
          for group in param_groups:
              for param in group:
                  self.params.append(param)
                  if is_moe_param(param):
                      self.has_moe_params = True
  
imports: from collections.abc import Iterable
  from deepspeed.moe.utils import is_moe_param
  import os
  import psutil
  import gc
  from math import sqrt
  from math import floor
  from bisect import bisect_left
  import torch
  from torch import inf
  from deepspeed import comm
  from deepspeed.utils import groups, logger
  from deepspeed.runtime.constants import PIPE_REPLICATED
  from numpy import prod
label_desc: Use image node_id 1 to create a new CheckOverflow object from inherited base classes: object with example: obj = CheckOverflow(param_groups, mpu, zero_reduce_scatter, deepspeed)