class_docstr: Mixture of Experts Layer.
class_docstr_tok: ["Mixture","of","Experts","Layer","."]
code: def __init__(
      self,
      num_experts=16,
      loss_coef=1e-2,
      hidden_size=4096,
      group_size=1024,
      capacity_factor_train=1.25,
      capacity_factor_eval=2.0,
      use_second_place_loss=False,
      second_policy_train="random",
      second_policy_eval="random",
      second_threshold_train=0.2,
      second_threshold_eval=0.2,
      dropout_rate=0.0,
      activation="relu",
      moe_gating="top_2",
      min_expert_capacity=4,
      switch_policy_train="input_jitter",
      switch_policy_eval="input_jitter",
      switch_dropout=0.1,
      switch_temperature=1.0,
      switch_jitter=1e-2,
      ntlb_top_k=4,
      output_dim=None,
      use_experts_attention=False,
      z_loss=None,
      word_embed_mode=None,
      use_second_place_expert_prob=None,
      use_second_place_expert_prob_temp=None,
      top_n_num_experts_per_token=3,
  ):
      self._hparams = HParams(
          moe_gating=moe_gating,
          moe_num_experts=num_experts,
          moe_loss_coef=loss_coef,
          moe_hidden_size=hidden_size,
          moe_group_size=group_size,
          moe_min_expert_capacity=min_expert_capacity,
          moe_capacity_factor_train=capacity_factor_train,
          moe_capacity_factor_eval=capacity_factor_eval,
          moe_use_second_place_loss=use_second_place_loss,
          moe_second_policy_train=second_policy_train,
          moe_second_policy_eval=second_policy_eval,
          moe_second_threshold_train=second_threshold_train,
          moe_second_threshold_eval=second_threshold_eval,
          moe_dropout_rate=dropout_rate,
          moe_switch_policy_train=switch_policy_train,
          moe_switch_policy_eval=switch_policy_eval,
          moe_switch_dropout=switch_dropout,
          moe_switch_temperature=switch_temperature,
          moe_switch_jitter=switch_jitter,
          moe_output_dim=output_dim,
          moe_ntlb_top_k=ntlb_top_k,
          moe_use_experts_attention=use_experts_attention,
          moe_z_loss=z_loss,
          moe_word_embed_mode=word_embed_mode,
          moe_use_second_place_expert_prob=(
              use_second_place_expert_prob
          ),
          moe_use_second_place_expert_prob_temp=(
              use_second_place_expert_prob_temp
          ),
          moe_top_n_num_experts_per_token=top_n_num_experts_per_token,
      )
      self._activation = activation
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import gin
  import mesh_tensorflow
  from mesh_tensorflow.transformer import transformer
  import tensorflow.compat.v1
label_desc: Use image node_id 1 to create a new MoE1D object from inherited base classes: transformer with example: obj = MoE1D(num_experts, loss_coef, hidden_size, group_size, capacity_factor_train, capacity_factor_eval, use_second_place_loss, second_policy_train, second_policy_eval, second_threshold_train, second_threshold_eval, dropout_rate, activation, moe_gating, min_expert_capacity, switch_policy_train, switch_policy_eval, switch_dropout, switch_temperature, switch_jitter, ntlb_top_k, output_dim, use_experts_attention, z_loss, word_embed_mode, use_second_place_expert_prob, use_second_place_expert_prob_temp, top_n_num_experts_per_token)