code: def get_global_norm_of_tensors(input_tensors, norm_type=2, mpu=None):
      """Get norm of an iterable of tensors.
  
      This is adapted from torch.nn.utils.clip_grad.clip_grad_norm_ and
      added functionality to handle model parallel parameters. Taken from Nvidia Megatron.
  
      Arguments:
          input_tensors (Iterable[Tensor]): an iterable of Tensors will have norm computed
          norm_type (float or int): type of the used p-norm. Can be ``'inf'`` for
              infinity norm.
  
      Returns:
          Total norm of the tensors (viewed as a single vector).
      """
  
      assert isinstance(
          input_tensors, Iterable
      ), f"expected Iterable type not {type(input_tensors)}"
      assert all(
          [torch.is_tensor(t) for t in input_tensors]
      ), f"expected list of only tensors"
  
      norm_type = float(norm_type)
      if norm_type == inf:
          total_norm = max(t.data.abs().max() for t in input_tensors)
          total_norm_cuda = torch.cuda.FloatTensor([float(total_norm)])
          if mpu is not None:
              dist.all_reduce(
                  total_norm_cuda,
                  op=dist.ReduceOp.MAX,
                  group=mpu.get_model_parallel_group(),
              )
              total_norm = total_norm_cuda[0].item()
      else:
          total_norm = sum(
              [
                  t.data.float().norm(norm_type).item() ** norm_type
                  for t in input_tensors
              ]
          )
          total_norm_cuda = torch.cuda.FloatTensor([float(total_norm)])
          if mpu is not None:
              dist.all_reduce(
                  total_norm_cuda,
                  op=dist.ReduceOp.SUM,
                  group=mpu.get_model_parallel_group(),
              )
          total_norm = total_norm_cuda[0].item() ** (1.0 / norm_type)
  
      if (
          total_norm == float("inf")
          or total_norm == -float("inf")
          or total_norm != total_norm
      ):
          total_norm = -1
  
      return total_norm
  
imports: from collections.abc import Iterable
  from deepspeed.moe.utils import is_moe_param
  import os
  import psutil
  import gc
  from math import sqrt
  from math import floor
  from bisect import bisect_left
  import torch
  from torch import inf
  from deepspeed import comm
  from deepspeed.utils import groups, logger
  from deepspeed.runtime.constants import PIPE_REPLICATED
  from numpy import prod
label_desc: Use image node_id 25 for calling a global function with example usage: get_global_norm_of_tensors(input_tensors, norm_type, mpu) and returns: total_norm