code: def set_default_moe_hparams(hparams):
      """Add necessary hyperparameters for mixture-of-experts."""
      hparams.moe_num_experts = 16
      hparams.moe_loss_coef = 1e-2
      hparams.add_hparam("moe_gating", "top_2")
      # Experts have fixed capacity per batch.  We need some extra capacity
      # in case gating is not perfectly balanced.
      # moe_capacity_factor_* should be set to a value >=1.
      hparams.add_hparam("moe_capacity_factor_train", 1.25)
      hparams.add_hparam("moe_capacity_factor_eval", 2.0)
      hparams.add_hparam("moe_capacity_factor_second_level", 1.0)
      # Each expert has a hidden layer with this size.
      hparams.add_hparam("moe_hidden_size", 4096)
      # For gating, divide inputs into groups of this size before gating.
      # Each group sends the same number of inputs to each expert.
      # Ideally, the group size would be the whole batch, but this is expensive
      # due to our use of matrix multiplication for reordering.
      hparams.add_hparam("moe_group_size", 1024)
      # For top_2 gating, whether to impose an additional loss in order to make
      # the experts equally used as the second-place expert.
      hparams.add_hparam("moe_use_second_place_loss", 0)
      # In top_2 gating, policy for whether to use a second-place expert.
      # Legal values are:
      #    "all": always
      #    "none": never
      #    "threshold": if gate value > the given threshold
      #    "random": if gate value > threshold*random_uniform(0,1)
      hparams.add_hparam("moe_second_policy_train", "random")
      hparams.add_hparam("moe_second_policy_eval", "random")
      hparams.add_hparam("moe_second_threshold_train", 0.2)
      hparams.add_hparam("moe_second_threshold_eval", 0.2)
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import gin
  import mesh_tensorflow
  from mesh_tensorflow.transformer import transformer
  import tensorflow.compat.v1
label_desc: Use image node_id 12 for calling a global function with example usage: set_default_moe_hparams(hparams) without return types