class_docstr: A complete MoE MLP module in a Transformer block.
  * `activation` is the activation function to be used in MLP in each expert.
  * `d_hidden` is the dimension of the MLP layer.
class_docstr_tok: ["A","complete","MoE","MLP","module","in","a","Transformer","block",".","*","`","activation","`","is","the","activation","function","to","be","used","in","MLP","in","each","expert",".","*","`","d_hidden","`","is","the","dimension","of","the","MLP","layer","."]
code: def __init__(
      self,
      num_expert=32,
      d_model=1024,
      d_hidden=4096,
      activation=torch.nn.GELU(),
      expert_dp_comm="none",
      expert_rank=0,
      **kwargs,
  ):
      def one_expert(d_model):
          return _Expert(1, d_model, d_hidden, activation, rank=0)
  
      expert = one_expert
      super().__init__(
          num_expert=num_expert,
          d_model=d_model,
          expert=expert,
          **kwargs,
      )
      self.mark_parallel_comm(expert_dp_comm)
  
file_path: fastmoe/fmoe/transformer.py
imports: import torch
  import torch.nn
  from .layers import FMoE
  from .linear import FMoELinear
  from .fastermoe.config import switch_from_env
label_desc: Use image node_id 1 to create a new FMoETransformerMLP object from inherited base classes: FMoE with example: obj = FMoETransformerMLP(num_expert, d_model, d_hidden, activation, expert_dp_comm, expert_rank)