code: def evenly_divisible_all_gather(
      data: torch.Tensor, concat: bool = True
  ) -> torch.Tensor | list[torch.Tensor]:
      """
      Utility function for distributed data parallel to pad at first dim to make it evenly divisible and all_gather.
      The input data of every rank should have the same number of dimensions, only the first dim can be different.
  
      Note: If has ignite installed, will execute based on ignite distributed APIs, otherwise, if the native
      PyTorch distributed group initialized, will execute based on native PyTorch distributed APIs.
  
      Args:
          data: source tensor to pad and execute all_gather in distributed data parallel.
          concat: whether to concat the gathered list to be a Tensor, if False, return a list
              of Tensors, similar behavior as torch.distributed.all_gather(). default to True.
  
      Note:
          The input data on different ranks must have exactly same `dtype`.
  
      """
      if not isinstance(data, torch.Tensor):
          raise ValueError("input data must be PyTorch Tensor.")
      # data of all the ranks must have same number of dimensions
      ndims = data.ndimension()
      length: int = data.shape[0] if ndims > 0 else 1
  
      def _torch_all_gather(data: torch.Tensor) -> list[torch.Tensor]:
          """
          Implementation based on native PyTorch distributed data parallel APIs.
  
          """
          device = get_dist_device()
          orig_device = data.device
          data = data.to(device)
          data = data.unsqueeze(0) if ndims == 0 else data
  
          # make sure the data is evenly-divisible on multi-GPUs
          length_tensor = torch.as_tensor([length], device=device)
          all_lens = [
              torch.zeros_like(length_tensor)
              for _ in range(dist.get_world_size())
          ]
          dist.all_gather(all_lens, length_tensor)
          all_lens_: list[int] = [int(i.item()) for i in all_lens]
  
          max_len: int = max(all_lens_)
          if length < max_len:
              size = [max_len - length] + list(data.shape[1:])
              data = torch.cat([data, data.new_full(size, 0)], dim=0)
          # all gather across all processes
          output = [
              torch.zeros_like(data)
              for _ in range(dist.get_world_size())
          ]
          dist.all_gather(output, data)
          # remove the padding items, if all the input data doesn't have batch dim, squeeze the first dim
          return [
              (o.squeeze(0) if ndims == 0 else o[:l, ...]).to(
                  orig_device
              )
              for o, l in zip(output, all_lens_)
          ]
  
      def _ignite_all_gather(data: torch.Tensor) -> list[torch.Tensor]:
          """
          Implementation based on PyTorch ignite package, it can support more kinds of backends.
  
          """
          data = data.unsqueeze(0) if ndims == 0 else data
          # make sure the data is evenly-divisible on multi-GPUs
          all_lens: list[int] = idist.all_gather(length)
          max_len: int = max(all_lens)
          if length < max_len:
              size = [max_len - length] + list(data.shape[1:])
              data = torch.cat([data, data.new_full(size, 0)], dim=0)
          # all gather across all processes
          output = idist.all_gather(data)
          # delete the padding NaN items
          if ndims == 0:
              # if all the input data doesn't have batch dim, unbind to a list of 0-dim Tensors
              return list(torch.unbind(output, dim=0))
          return [
              output[i * max_len : i * max_len + l, ...]
              for i, l in enumerate(all_lens)
          ]
  
      output: list[torch.Tensor]
      if has_ignite:
          if idist.get_world_size() <= 1:
              return data
          output = _ignite_all_gather(data=data)
      elif dist.is_available() and dist.is_initialized():
          if dist.get_world_size() <= 1:
              return data
          output = _torch_all_gather(data=data)
      else:
          return data
  
      return torch.cat(output, dim=0) if concat else output
  
file_path: MONAI/monai/utils/dist.py
imports: from __future__ import annotations
  import sys
  import warnings
  from collections.abc import Callable
  from logging import Filter
  from typing import overload
  import torch
  import torch.distributed
  from monai.config import IgniteInfo
  from monai.utils.module import min_version, optional_import
label_desc: Use image node_id 5 for calling a global function with example usage: evenly_divisible_all_gather(data, concat) and returns: unknown, unknown, unknown, list, data, data, data