code: def get_weight_norm(parameters, norm_type=2, mpu=None):
      """Get norm of an iterable of parameters.
  
      This is adapted from torch.nn.utils.clip_grad.clip_grad_norm_ and
      added functionality to handle model parallel parameters. Note that
      the gradients are modified in place. Taken from Nvidia Megatron.
  
      Arguments:
          parameters (Iterable[Tensor] or Tensor): an iterable of Tensors or a
              single Tensor that will have gradients normalized
          max_norm (float or int): max norm of the gradients
          norm_type (float or int): type of the used p-norm. Can be ``'inf'`` for
              infinity norm.
  
      Returns:
          Total norm of the parameters (viewed as a single vector).
      """
      if isinstance(parameters, torch.Tensor):
          parameters = [parameters]
  
      norm_type = float(norm_type)
      if norm_type == inf:
          total_norm = max(p.data.abs().max() for p in parameters)
          total_norm_cuda = torch.cuda.FloatTensor([float(total_norm)])
          # Take max across all GPUs.
          if mpu is not None:
              dist.all_reduce(
                  total_norm_cuda,
                  op=dist.ReduceOp.MAX,
                  group=mpu.get_model_parallel_group(),
              )
          total_norm = total_norm_cuda[0].item()
      else:
          total_norm = 0.0
          tensor_mp_rank = bwc_tensor_model_parallel_rank(mpu=mpu)
          for p in parameters:
              # Pipeline parallelism may replicate parameters. Avoid multi-counting.
              if hasattr(p, PIPE_REPLICATED) and p.ds_pipe_replicated:
                  continue
  
              # Filter to avoid over-counting replicated tensors from tensor
              # model parallelism
              if (
                  tensor_mp_rank > 0
              ) and not is_model_parallel_parameter(p):
                  continue
  
              param_norm = p.data.float().norm(norm_type)
              total_norm += param_norm**norm_type
  
          # Sum across all model parallel GPUs.
          total_norm_cuda = torch.cuda.FloatTensor([float(total_norm)])
          if mpu is not None:
              dist.all_reduce(
                  total_norm_cuda,
                  op=dist.ReduceOp.SUM,
                  group=mpu.get_model_parallel_group(),
              )
          total_norm = total_norm_cuda[0].item() ** (1.0 / norm_type)
  
      if (
          total_norm == float("inf")
          or total_norm == -float("inf")
          or total_norm != total_norm
      ):
          total_norm = -1
  
      return total_norm
  
imports: from collections.abc import Iterable
  from deepspeed.moe.utils import is_moe_param
  import os
  import psutil
  import gc
  from math import sqrt
  from math import floor
  from bisect import bisect_left
  import torch
  from torch import inf
  from deepspeed import comm
  from deepspeed.utils import groups, logger
  from deepspeed.runtime.constants import PIPE_REPLICATED
  from numpy import prod
label_desc: Use image node_id 13 for calling a global function with example usage: get_weight_norm(parameters, norm_type, mpu) and returns: total_norm