class_docstr: MoE manager. This class manages different
  parallel groups in MoE context and MoE loss in training.
docstr: Setup MoE distributed context.
  
  Args:
      seed (int): Random seed. Defaults to 42.
      use_kernel_optim (bool, optional): Use cuda kernel. Defaults to True.
      parallel (bool, optional): Parallel mode, should be EP, TP or None. Defaults to None.
      mode (str, optional): Should be "fixed" or "dynamic". Defaults to "dynamic".
          In fixed mode, the ep size and dp size is fixed.
          In dynamic mode, the ep size and dp size will be changed according to num experts.
      max_ep_size (int, optional): Max ep size in dynamic mode. Defaults to 8.
      fixed_dp_size (int, optional): Fixed dp size in fixed mode. Defaults to 0.
      fixed_ep_size (int, optional): Fixed ep size in fixed mode. Defaults to 0.
      fixed_pp_size (int, optional): Fixed pp size in fixed mode. Defaults to 0.
      use_ep_inside (bool, optional): Use ep inside dp if True, dp inside ep if Fasle. Defaults to True.
docstr_tok: ["Setup","MoE","distributed","context",".","Args",":","seed","(","int",")",":","Random","seed",".","Defaults","to","42.","use_kernel_optim","(","bool",",","optional",")",":","Use","cuda","kernel",".","Defaults","to","True",".","parallel","(","bool",",","optional",")",":","Parallel","mode",",","should","be","EP",",","TP","or","None",".","Defaults","to","None",".","mode","(","str",",","optional",")",":","Should","be","``","fixed","''","or","``","dynamic","''",".","Defaults","to","``","dynamic","''",".","In","fixed","mode",",","the","ep","size","and","dp","size","is","fixed",".","In","dynamic","mode",",","the","ep","size","and","dp","size","will","be","changed","according","to","num","experts",".","max_ep_size","(","int",",","optional",")",":","Max","ep","size","in","dynamic","mode",".","Defaults","to","8.","fixed_dp_size","(","int",",","optional",")",":","Fixed","dp","size","in","fixed","mode",".","Defaults","to","0.","fixed_ep_size","(","int",",","optional",")",":","Fixed","ep","size","in","fixed","mode",".","Defaults","to","0.","fixed_pp_size","(","int",",","optional",")",":","Fixed","pp","size","in","fixed","mode",".","Defaults","to","0.","use_ep_inside","(","bool",",","optional",")",":","Use","ep","inside","dp","if","True",",","dp","inside","ep","if","Fasle",".","Defaults","to","True","."]
code: def setup(
      self,
      parallel: str = None,
      mode: str = "dynamic",
      max_ep_size: int = 8,
      fixed_dp_size: int = 0,
      fixed_ep_size: int = 0,
      fixed_pp_size: int = 0,
      use_ep_inside: bool = True,
  ) -> None:
      """
      Setup MoE distributed context.
  
      Args:
          seed (int): Random seed. Defaults to 42.
          use_kernel_optim (bool, optional): Use cuda kernel. Defaults to True.
          parallel (bool, optional): Parallel mode, should be EP, TP or None. Defaults to None.
          mode (str, optional): Should be "fixed" or "dynamic". Defaults to "dynamic".
              In fixed mode, the ep size and dp size is fixed.
              In dynamic mode, the ep size and dp size will be changed according to num experts.
          max_ep_size (int, optional): Max ep size in dynamic mode. Defaults to 8.
          fixed_dp_size (int, optional): Fixed dp size in fixed mode. Defaults to 0.
          fixed_ep_size (int, optional): Fixed ep size in fixed mode. Defaults to 0.
          fixed_pp_size (int, optional): Fixed pp size in fixed mode. Defaults to 0.
          use_ep_inside (bool, optional): Use ep inside dp if True, dp inside ep if Fasle. Defaults to True.
      """
      assert (
          not self.is_initialized
      ), "MoE distributed context shouldn't be set up again"
      assert (
          torch.cuda.is_available()
      ), "MoE requires to enable CUDA first"
  
      self.parallel = parallel
      self.use_ep_inside = use_ep_inside
      self.world_size = dist.get_world_size()
  
      # init by mode
      self.mode = mode
      assert self.mode in [
          "fixed",
          "dynamic",
      ], "mode should be fixed or dynamic"
      if self.mode == "dynamic":
          self.max_ep_size = min(max_ep_size, self.world_size)
      else:
          assert (
              fixed_dp_size > 0
              and fixed_ep_size > 0
              and fixed_pp_size > 0
          ), "dp_size, ep_size and pp_size should be greater than 0"
          assert (
              isinstance(fixed_dp_size, int)
              and isinstance(fixed_ep_size, int)
              and isinstance(fixed_pp_size, int)
          ), "dp_size, ep_size and pp_size should be int"
          self.ep_size = fixed_ep_size
          self.dp_size = fixed_dp_size
          self.pp_size = fixed_pp_size
  
      self.has_setup = True
  
file_path: ColossalAI/colossalai/moe/manager.py
imports: from typing import Tuple
  import torch
  import torch.distributed
  from colossalai.context.singleton_meta import SingletonMeta
  from colossalai.tensor.moe_tensor.api import get_moe_info
  from colossalai.tensor.moe_tensor.moe_info import MoeParallelInfo
label_desc: Use image node_id 4 for calling the MoEManager obj's underlying member method code with example usage: obj.setup(parallel, mode, max_ep_size, fixed_dp_size, fixed_ep_size, fixed_pp_size, use_ep_inside) without return types