code: def __init__(
      self,
      num_experts,
      num_experts_per_tok,
      d_model,
      d_ff,
      dropout,
      pos_ffn_activation_fn,
      add_ffnbias,
      parallel_residual,
      layer_norm,
      norm_eps,
      use_ckpting=[],
      parallel_gpu=1,
  ):
      super().__init__()
      self.experts = nn.ModuleList(
          [
              PositionwiseFeedForward(
                  d_model,
                  d_ff,
                  dropout,
                  pos_ffn_activation_fn,
                  add_ffnbias,
                  parallel_residual,
                  layer_norm,
                  norm_eps,
                  use_ckpting=use_ckpting,
                  parallel_gpu=parallel_gpu,
              )
              for i in range(num_experts)
          ]
      )
      self.gate = nn.Linear(d_model, num_experts, bias=False)
      self.num_experts_per_tok = num_experts_per_tok
      self.parallel_gpu = parallel_gpu
  
file_path: OpenNMT-py/onmt/modules/moe.py
imports: import torch
  import torch.nn
  from onmt.modules.position_ffn import PositionwiseFeedForward
  from torch.distributed import all_reduce
label_desc: Use image node_id 1 to create a new MoE object from inherited base classes: nn with example: obj = MoE(num_experts, num_experts_per_tok, d_model, d_ff, dropout, pos_ffn_activation_fn, add_ffnbias, parallel_residual, layer_norm, norm_eps, use_ckpting, parallel_gpu)