code: def __init__(self, prefix, config: MixtralConfig, weights):
      super().__init__()
      self.hidden_dim = config.hidden_size
      self.ffn_dim = (
          config.intermediate_size // weights.process_group.size()
      )
      self.num_experts = config.num_local_experts
      self.top_k = config.num_experts_per_tok
  
      act = config.hidden_act
      if "gelu" in act:
          self.act = lambda x: torch.nn.functional.gelu(
              x,
              approximate="tanh"
              if act in ["gelu_fast", "gelu_pytorch_tanh"]
              else "none",
          )
      elif "silu" in act:
          self.act = torch.nn.functional.silu
      else:
          self.act = ACT2FN[act]
  
      # gating
      self.gate = FastLinear.load(
          config, f"{prefix}.gate", weights, bias=False
      )
  
      self.w1 = [
          TensorParallelColumnLinear.load(
              config,
              prefix=f"{prefix}.experts.{i}.w1",
              weights=weights,
              bias=False,
          )
          for i in range(self.num_experts)
      ]
      self.w3 = [
          TensorParallelColumnLinear.load(
              config,
              prefix=f"{prefix}.experts.{i}.w3",
              weights=weights,
              bias=False,
          )
          for i in range(self.num_experts)
      ]
      self.w2 = [
          TensorParallelRowLinear.load(
              config,
              prefix=f"{prefix}.experts.{i}.w2",
              weights=weights,
              bias=False,
          )
          for i in range(self.num_experts)
      ]
  
      self.process_group = weights.process_group
  
imports: import torch
  import torch.distributed
  import numpy
  from torch import nn
  from transformers.activations import ACT2FN
  from transformers.configuration_utils import PretrainedConfig
  from typing import Optional, List, Tuple
  from loguru import logger
  from text_generation_server.utils import paged_attention, flash_attn
  from text_generation_server.utils.layers import FastLinear, FastRMSNorm, TensorParallelRowLinear, TensorParallelColumnLinear, TensorParallelEmbedding, PositionRotaryEmbedding, TensorParallelHead, get_linear
label_desc: Use image node_id 1 to create a new DenseMoE object from inherited base classes: nn with example: obj = DenseMoE(prefix, config, weights)