class_docstr: Checks for overflow in gradient across parallel process
class_docstr_tok: ["Checks","for","overflow","in","gradient","across","parallel","process"]
code: def check_using_norm(self, norm_group, reduce_overflow=True):
      # TODO: I don't think reduce_overflow is needed if mpu is None
      overflow = -1 in norm_group
      overflow_gpu = torch.cuda.FloatTensor([overflow])
      if self.has_moe_params:
          # In this case, we need to do an all_reduce across
          # the expert_parallel_group, so that if there was
          # an overflow due to expert weights, we detect it
  
          # Only need to check groups.get_largest_expert_parallel_group()
          dist.all_reduce(
              overflow_gpu,
              op=dist.ReduceOp.MAX,
              group=groups._get_max_expert_parallel_group(),
          )
      if self.mpu is not None:
          dist.all_reduce(
              overflow_gpu,
              op=dist.ReduceOp.MAX,
              group=self.mpu.get_model_parallel_group(),
          )
      elif reduce_overflow:
          dist.all_reduce(overflow_gpu, op=dist.ReduceOp.MAX)
          dist.barrier()
      overflow = overflow_gpu[0].item()
      return bool(overflow)
  
imports: from collections.abc import Iterable
  from deepspeed.moe.utils import is_moe_param
  import os
  import psutil
  import gc
  from math import sqrt
  from math import floor
  from bisect import bisect_left
  import torch
  from torch import inf
  from deepspeed import comm
  from deepspeed.utils import groups, logger
  from deepspeed.runtime.constants import PIPE_REPLICATED
  from numpy import prod
label_desc: Use image node_id 2 for calling the CheckOverflow obj's underlying member method code with example usage: obj.check_using_norm(norm_group, reduce_overflow) and returns: bool