code: def all_reduce_item(value, op="sum"):
      """
      All-reduces single scalar value if distributed is in use
      """
      if (
          torch.distributed.is_available()
          and torch.distributed.is_initialized()
      ):
          if op == "sum" or op == "mean":
              dop = torch.distributed.ReduceOp.SUM
          elif op == "min":
              dop = torch.distributed.ReduceOp.MIN
          elif op == "max":
              dop = torch.distributed.ReduceOp.MAX
          elif op == "product":
              dop = torch.distributed.ReduceOp.PRODUCT
          else:
              raise RuntimeError("Unsupported reduce op")
  
          backend = torch.distributed.get_backend()
          if backend == torch.distributed.Backend.NCCL:
              device = torch.device("cuda")
          elif backend == torch.distributed.Backend.GLOO:
              device = torch.device("cpu")
          else:
              raise RuntimeError("Unsupported distributed backend")
  
          tensor = torch.tensor(value, device=device)
          torch.distributed.all_reduce(tensor, dop)
          if op == "mean":
              tensor /= get_world_size()
          ret = tensor.item()
      else:
          ret = value
      return ret
  
file_path: zoo/src/utils/distributed.py
imports: import os
  from contextlib import contextmanager
  import torch
label_desc: Use image node_id 5 for calling a global function with example usage: all_reduce_item(value, op) and returns: ret