docstr: Instantiates the MoE layer for the model. This sets the `self.moe` attribute.
docstr_tok: ["Instantiates","the","MoE","layer","for","the","model",".","This","sets","the","`","self.moe","`","attribute","."]
code: def make_moe_layer(self) -> None:
      """
      Instantiates the MoE layer for the model. This sets the `self.moe` attribute.
      """
      sharded_dim = sharded_intermediate_dim(
          self.intermediate_dim, self.tp_size, self.tp_rank
      )
  
      moe_config = DSMoEConfig(
          max_tokens=self._engine_config.state_manager.max_ragged_batch_size,
          model_dim=self.model_dim,
          intermediate_features=sharded_dim,
          activation=self.mlp_activation_fn,
          n_experts=self.n_experts,
          top_k=self.n_top_k,
          input_dtype=self.activation_dtype,
          output_dtype=self.activation_dtype,
          normalize_scores=self.normalize_expert_scores,
      )
  
      self.moe = heuristics.instantiate_moe(
          moe_config, self._engine_config
      )
  
imports: from abc import abstractmethod
  from typing import Optional
  import torch
  from deepspeed.accelerator import get_accelerator
  from ..config_v2 import RaggedInferenceEngineConfig
  from ..inference_utils import ActivationType, ceil_div, is_gated
  from ..model_implementations import *
  from ..model_implementations.sharding import *
  from ..modules.configs import DSEmbeddingsConfig, DSLinearConfig, DSMoEConfig, DSNormConfig, DSSelfAttentionConfig, DSUnembedConfig, NormTypeEnum, PositionalEmbeddingType, RotateHalfConfig
  from ..modules import heuristics
  from ..ragged import DSSequenceDescriptor, KVCacheConfig, RaggedBatchWrapper
  from .inference_model_base import DSInferenceModelBase, DSModelImplementationConfig, MPType
  from ..inference_parameter import InferenceParameter
label_desc: Use image node_id 4 for calling the DSMoETransformerModelBase obj's underlying member method code with example usage: obj.make_moe_layer() without return types