code: def _hierarchical_all_gather_params(
      self, params, params_buffers=None
  ):
      """"""
      params, params_buffers = self._pre_all_gather(
          params, params_buffers
      )
  
      mics_comm_groups: MiCS_CommGroups = params[0].comm
      local_rank = dist.get_rank(
          group=mics_comm_groups.param_intra_node_group
      )
      inter_node_comm_group = (
          mics_comm_groups.param_inter_node_shard_group
      )
      intra_node_comm_group = mics_comm_groups.param_intra_node_group
      param_shard_size = mics_comm_groups.param_shard_size
  
      inter_node_size = dist.get_world_size(group=inter_node_comm_group)
      intra_node_size = dist.get_world_size(group=intra_node_comm_group)
      param_tensors = []
      for i, p in enumerate(params):
          param_size = p.ds_tensor.ds_numel * param_shard_size
          if (
              params_buffers is not None
              and params_buffers[i] is not None
          ):
              assert (
                  params_buffers[i].numel() == param_size
              ), f"param_buffers[{i}] size {params_buffers[i].numel()} does not match with param_size {param_size}"
              param_tensor = params_buffers[i]
          else:
              param_tensor = torch.empty(
                  param_size,
                  dtype=p.dtype,
                  device=self.local_device,
                  requires_grad=False,
              ).view(-1)
          param_tensors.append(param_tensor)
  
      # inter node all-gather
      inter_outputs = []
      inter_inputs = []
      for i, p in enumerate(params):
          inter_size = p.ds_tensor.ds_numel * inter_node_size
          _out = param_tensors[i].narrow(
              0, local_rank * inter_size, inter_size
          )
          inter_outputs.append(_out)
          inter_inputs.append(
              p.ds_tensor.data.view(-1).to(self.local_device)
          )
      # sync enqueue
      dist.all_gather_coalesced(
          inter_outputs,
          inter_inputs,
          group=inter_node_comm_group,
          async_op=False,
      )
  
      # intra node all-gather
      intra_outputs = []
      intra_inputs = []
      for i, p in enumerate(params):
          # partition param into multiple chunks for allgather
          # because inter-node all-gather outputs are in a continues memory
          # while in param memory, those inter-node data are placed in different
          # location.
          # each chunk is an intra-node output
          param_chunk = (
              param_tensors[i]
              .view(
                  (
                      inter_node_size,
                      intra_node_size,
                      p.ds_tensor.ds_numel,
                  )
              )
              .narrow(1, local_rank, 1)
          )
          param_chunk.copy_(
              inter_outputs[i].detach().clone().view(param_chunk.size())
          )
          output_chunks = torch.chunk(param_tensors[i], inter_node_size)
          for j, _out in enumerate(output_chunks):
              intra_chunk_size = intra_node_size * p.ds_tensor.ds_numel
              local_offset = local_rank * p.ds_tensor.ds_numel
              _in = param_tensors[i].narrow(
                  0,
                  j * intra_chunk_size + local_offset,
                  p.ds_tensor.ds_numel,
              )
              intra_outputs.append(_out)
              intra_inputs.append(_in)
  
      all_gather_handle = dist.all_gather_coalesced(
          intra_outputs,
          intra_inputs,
          group=intra_node_comm_group,
          async_op=True,
      )
      for i, param in enumerate(params):
          param.data = (
              param_tensors[i]
              .narrow(0, 0, param.ds_numel)
              .view(param.ds_shape)
              .data
          )
  
      return MiCS_AllGatherCoalescedHandle(
          allgather_handle=all_gather_handle,
          params=params,
          partitions=[],
          world_size=param_shard_size,
      )
  
imports: import sys
  from typing import List
  import deepspeed
  import torch
  from deepspeed import comm
  from deepspeed.runtime.zero.mics_utils import MiCS_CommGroups, create_mics_comm_groups, scale_tensors
  from deepspeed.runtime.zero.parameter_offload import DeepSpeedZeRoOffload, is_zero_param
  from deepspeed.runtime.zero.partition_parameters import Init, AllGatherCoalescedHandle, ZeroParamStatus
  from deepspeed.runtime.zero.stage3 import DeepSpeedZeroOptimizer_Stage3
  from deepspeed.utils import instrument_w_nvtx, log_dist
  from deepspeed.accelerator import get_accelerator
  from torch import Tensor
  from torch.nn import Parameter
label_desc: Use image node_id 5 for calling the MiCS_Init obj's underlying member method code with example usage: obj._hierarchical_all_gather_params(params, params_buffers) and returns: MiCS_AllGatherCoalescedHandle