class_docstr: A ProcessGroupInitializer for sequence parallelism all-reduce.
  
  In Sequence Parallelism, each GPU holds the full copy of model weights,
  thus, gradient all-reduce occurs across all processes in the same pipeline stage
  
  Args:
      rank (int): The rank of current process
      world_size (int): Size of whole communication world
      config (Config): Running configuration
      data_parallel_size (int): Size of data parallel
      pipeline_parallel_size (int): Size of pipeline parallel
      tensor_parallel_size (int): Size of tensor parallel
class_docstr_tok: ["A","ProcessGroupInitializer","for","sequence","parallelism","all-reduce",".","In","Sequence","Parallelism",",","each","GPU","holds","the","full","copy","of","model","weights",",","thus",",","gradient","all-reduce","occurs","across","all","processes","in","the","same","pipeline","stage","Args",":","rank","(","int",")",":","The","rank","of","current","process","world_size","(","int",")",":","Size","of","whole","communication","world","config","(","Config",")",":","Running","configuration","data_parallel_size","(","int",")",":","Size","of","data","parallel","pipeline_parallel_size","(","int",")",":","Size","of","pipeline","parallel","tensor_parallel_size","(","int",")",":","Size","of","tensor","parallel"]
docstr: Initialize Sequence Parallel process groups used for gradient all-reduce.
  
  Returns:
      Tuple: A tuple (local_rank, group_world_size, process_group, ranks_in_group, mode).
docstr_tok: ["Initialize","Sequence","Parallel","process","groups","used","for","gradient","all-reduce",".","Returns",":","Tuple",":","A","tuple","(","local_rank",",","group_world_size",",","process_group",",","ranks_in_group",",","mode",")","."]
code: def init_dist_group(self):
      """Initialize Sequence Parallel process groups used for gradient all-reduce.
  
      Returns:
          Tuple: A tuple (local_rank, group_world_size, process_group, ranks_in_group, mode).
      """
      local_rank = None
      ranks_in_group = None
      process_group = None
      cpu_group = None
      group_world_size = None
      mode = ParallelMode.SEQUENCE_DP
  
      for i in range(self.num_group):
          ranks = [i * self.dp_size + j for j in range(self.dp_size)]
          group = dist.new_group(ranks)
          group_cpu = (
              dist.new_group(ranks, backend="gloo")
              if dist.get_backend() != "gloo"
              else group
          )
  
          if self.rank in ranks:
              local_rank = ranks.index(self.rank)
              group_world_size = len(ranks)
              process_group = group
              cpu_group = group_cpu
              ranks_in_group = ranks
  
      return (
          local_rank,
          group_world_size,
          process_group,
          cpu_group,
          ranks_in_group,
          mode,
      )
  
file_path: ColossalAI/colossalai/legacy/context/process_group_initializer/initializer_sequence.py
imports: import torch.distributed
  from colossalai.legacy.registry import DIST_GROUP_INITIALIZER
  from ..parallel_mode import ParallelMode
  from .initializer_tensor import Initializer_Tensor
  from .process_group_initializer import ProcessGroupInitializer
label_desc: Use image node_id 2 for calling the Initializer_Sequence_DP obj's underlying member method code with example usage: obj.init_dist_group() and returns: local_rank,  group_world_size,  process_group,  cpu_group,  ranks_in_group,  mode