code: def __init__(self, **kwargs):
      # Call the init function of the parent class to initialize the tensors and configs from parent class
      super().__init__(**kwargs)
  
      self.num_experts = self.policy.get_num_experts()
      self.ep_world_size = dist.get_world_size()
      self.local_ep_size = (
          1
          if self.num_experts < self.ep_world_size
          else self.num_experts // self.ep_world_size
      )
  
      self.layer_norm_eps = (
          self.config.layer_norm_eps
          if hasattr(self.config, "layer_norm_eps")
          else 1e-12,
      )
  
      # MoE models will have a list of mlp related tensors
      self._h4h_w = []
      self._h4h_b = []
      self._4hh_w = []
      self._4hh_b = []
  
      # Residual MoE needs extra parameters
      self._res_h4h_w = None
      self._res_h4h_b = None
      self._res_4hh_w = None
      self._res_4hh_b = None
      self._res_coef = None
  
file_path: DeepSpeed/deepspeed/module_inject/containers/base_moe.py
imports: from .base import *
  from deepspeed import comm
  import deepspeed.ops.transformer
  from deepspeed.accelerator import get_accelerator
label_desc: Use image node_id 1 to create a new BaseTransformerMoEContainer object from inherited base classes: BaseTransformerContainer with example: obj = BaseTransformerMoEContainer()