code: def _initialize_parameter_parallel_groups(
      parameter_parallel_size=None,
  ):
      data_parallel_size = int(dist.get_world_size())
      parameter_parallel_size = (
          parameter_parallel_size or data_parallel_size
      )
      logger.info(
          "data_parallel_size: %s, parameter_parallel_size: %s",
          data_parallel_size,
          parameter_parallel_size,
      )
      assert (
          data_parallel_size % parameter_parallel_size == 0
      ), "world size should be divisible by parameter parallel size"
      rank = dist.get_rank()
      my_group = None
      for i in range(data_parallel_size // parameter_parallel_size):
          ranks = range(
              i * parameter_parallel_size,
              (i + 1) * parameter_parallel_size,
          )
          group = dist.new_group(ranks)
          if rank in ranks:
              my_group = group
      return my_group
  
file_path: H2O/h2o_flexgen/benchmark/third_party/DeepSpeed/deepspeed/runtime/zero/utils.py
imports: import os
  from typing import List
  import torch
  from deepspeed import comm
  from deepspeed.utils import logger
  from deepspeed.ops.adam import DeepSpeedCPUAdam
  from deepspeed.ops.adam import FusedAdam
  from deepspeed.utils.nvtx import instrument_w_nvtx
label_desc: Use image node_id 1 for calling a global function with example usage: _initialize_parameter_parallel_groups(parameter_parallel_size) and returns: my_group