class_docstr: FP16 Optimizer for training fp16 models. Handles loss scaling.
  
  For usage example please see, TODO:  DeepSpeed V2 Tutorial
   
class_docstr_tok: ["FP16","Optimizer","for","training","fp16","models",".","Handles","loss","scaling",".","For","usage","example","please","see",",","TODO",":","DeepSpeed","V2","Tutorial"]
code: def _get_norm_with_moe_layers(self, all_groups_norm):
      # all_groups_norm_old = all_groups_norm
      # Need to allreduce (avg) the norms across different ranks because moe params will not be synced during allreduce
      if self.using_pipeline:
          pg = self.deepspeed.mpu.get_data_parallel_group()
      else:
          pg = groups._get_data_parallel_group()
      scaled_norm = (
          all_groups_norm * 1.0 / float(dist.get_world_size(group=pg))
      )
      scaled_norm_tensor = torch.tensor(
          scaled_norm,
          device=self.fp32_groups_flat[0].device,
          dtype=torch.float,
      )
      dist.all_reduce(scaled_norm_tensor, group=pg)
      all_groups_norm = scaled_norm_tensor.item()
      # print(f"old = {all_groups_norm_old} and new = {all_groups_norm} at rank: {deepspeed.comm.get_rank()}")
      return all_groups_norm
  
imports: import torch
  from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors
  from deepspeed.runtime import DeepSpeedOptimizer
  from deepspeed.runtime.utils import get_global_norm, get_grad_norm, CheckOverflow, get_weight_norm
  from deepspeed.runtime.fp16.loss_scaler import INITIAL_LOSS_SCALE, SCALE_WINDOW, MIN_LOSS_SCALE
  from deepspeed.utils import groups, logger, log_dist
  from deepspeed import comm
  from deepspeed.checkpoint.constants import OPTIMIZER_STATE_DICT, CLIP_GRAD
label_desc: Use image node_id 12 for calling the FP16_Optimizer obj's underlying member method code with example usage: obj._get_norm_with_moe_layers(all_groups_norm) and returns: all_groups_norm