class_docstr: Checks for overflow in gradient across parallel process
class_docstr_tok: ["Checks","for","overflow","in","gradient","across","parallel","process"]
code: def has_overflow(self, params, has_moe_params=None):
      if has_moe_params is None:
          has_moe_params = self.has_moe_params
      overflow = self.has_overflow_serial(params)
      # Since each model parallel GPU carries only part of the model,
      # make sure overflow flag is synced across all the model parallel GPUs
      overflow_gpu = torch.cuda.ByteTensor([overflow])
      # deepspeeed.comm.all_reduce(overflow_gpu,
      #                             op=deepspeed.comm.ReduceOp.MAX,
      #                             group=mpu.get_model_parallel_group())
      if has_moe_params:
          # All reduce this across expert_parallel_group, so that if an expert
          # overflows, we detect it here
          dist.all_reduce(
              overflow_gpu,
              op=dist.ReduceOp.MAX,
              group=groups._get_max_expert_parallel_group(),
          )
      if self.zero_reduce_scatter:
          dist.all_reduce(
              overflow_gpu,
              op=dist.ReduceOp.MAX,
              group=dist.get_world_group(),
          )
      elif self.mpu is not None:
          if self.deepspeed is not None:
              using_pipeline = hasattr(
                  self.deepspeed, "pipeline_enable_backward_allreduce"
              )
              if (
                  using_pipeline
                  and self.deepspeed.pipeline_enable_backward_allreduce
                  is False
              ) or (
                  not using_pipeline
                  and self.deepspeed.enable_backward_allreduce is False
              ):
                  dist.all_reduce(
                      overflow_gpu,
                      op=dist.ReduceOp.MAX,
                      group=self.mpu.get_data_parallel_group(),
                  )
          dist.all_reduce(
              overflow_gpu,
              op=dist.ReduceOp.MAX,
              group=self.mpu.get_model_parallel_group(),
          )
      elif (
          self.deepspeed is not None
          and self.deepspeed.enable_backward_allreduce is False
      ):
          dist.all_reduce(
              overflow_gpu,
              op=dist.ReduceOp.MAX,
              group=dist.get_world_group(),
          )
  
      overflow = overflow_gpu[0].item()
      return bool(overflow)
  
imports: from collections.abc import Iterable
  from deepspeed.moe.utils import is_moe_param
  import os
  import psutil
  import gc
  from math import sqrt
  from math import floor
  from bisect import bisect_left
  import torch
  from torch import inf
  from deepspeed import comm
  from deepspeed.utils import groups, logger
  from deepspeed.runtime.constants import PIPE_REPLICATED
  from numpy import prod
label_desc: Use image node_id 5 for calling the CheckOverflow obj's underlying member method code with example usage: obj.has_overflow(params, has_moe_params) and returns: bool