class_docstr: A ProcessGroupInitializer for sequence parallelism all-reduce.
  
  In Sequence Parallelism, each GPU holds the full copy of model weights,
  thus, gradient all-reduce occurs across all processes in the same pipeline stage
  
  Args:
      rank (int): The rank of current process
      world_size (int): Size of whole communication world
      config (Config): Running configuration
      data_parallel_size (int): Size of data parallel
      pipeline_parallel_size (int): Size of pipeline parallel
      tensor_parallel_size (int): Size of tensor parallel
class_docstr_tok: ["A","ProcessGroupInitializer","for","sequence","parallelism","all-reduce",".","In","Sequence","Parallelism",",","each","GPU","holds","the","full","copy","of","model","weights",",","thus",",","gradient","all-reduce","occurs","across","all","processes","in","the","same","pipeline","stage","Args",":","rank","(","int",")",":","The","rank","of","current","process","world_size","(","int",")",":","Size","of","whole","communication","world","config","(","Config",")",":","Running","configuration","data_parallel_size","(","int",")",":","Size","of","data","parallel","pipeline_parallel_size","(","int",")",":","Size","of","pipeline","parallel","tensor_parallel_size","(","int",")",":","Size","of","tensor","parallel"]
code: def __init__(self, *args, **kwargs):
      super().__init__(*args, **kwargs)
      self.dp_size = self.world_size // self.pipeline_parallel_size
      self.num_group = self.pipeline_parallel_size
  
file_path: ColossalAI/colossalai/legacy/context/process_group_initializer/initializer_sequence.py
imports: import torch.distributed
  from colossalai.legacy.registry import DIST_GROUP_INITIALIZER
  from ..parallel_mode import ParallelMode
  from .initializer_tensor import Initializer_Tensor
  from .process_group_initializer import ProcessGroupInitializer
label_desc: Use image node_id 1 to create a new Initializer_Sequence_DP object from inherited base classes: ProcessGroupInitializer with example: obj = Initializer_Sequence_DP()