code: def xmoe_dense_4k():
      """Series of architectural experiments on cheap language models.
  
      For all of these architectures, we run on languagemodel_lm1b8k_packed
      for 32000 steps.
  
      All log-perplexities are per-token - multiply by 1.298 for per-word
  
      Results:
      model             params(M)  einsum  alltoall  mxu-util  log-ppl
      xmoe_dense_4k     30         3.0e12  0         45%        3.31
      xmoe_dense_8k     46         4.7e12  0         49%        3.24
      xmoe_dense_64k    282        2.8e13  0                    3.06
      xmoe_top_2        282        4.0e12  3.4e8     36%        3.07
      xmoe_top_2_c15    282        4.5e12  4.0e8     38%        3.07
      xmoe_2d           282        5.3e12  7.6e8     34%        3.06
  
      Trained at 4x the batch size:
      xmoe_2d_88        1090       2.1e13  3.0e9     24%        3.07
  
      Note: configurations and code are likely to change without notice.
  
      Returns:
        a hparams
      """
      hparams = mtf_transformer.mtf_transformer_base_lm()
      hparams.attention_dropout = 0.0
      hparams.relu_dropout = 0.0
      hparams.layer_prepostprocess_dropout = 0.0
  
      # The following hparams are constant across all these experiments.
      hparams.batch_size = 128
      hparams.d_model = 512
      hparams.d_kv = 128
      hparams.num_heads = 4
      hparams.decoder_layers = ["att", "drd"] * 4
      hparams.shared_embedding_and_softmax_weights = False
      hparams.learning_rate_schedule = "rsqrt_decay"
  
      # We will vary the following parameters related to the ffn/moe layers.
      hparams.d_ff = 4096
      hparams.layout = "batch:batch;vocab:model;d_ff:model;heads:model"
      hparams.mesh_shape = "batch:8"
      return hparams
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from tensor2tensor.models import mtf_transformer
  from tensor2tensor.models import mtf_transformer2
  from tensor2tensor.models.research import moe
  from tensor2tensor.utils import registry
label_desc: Use image node_id 5 for calling a global function with example usage: xmoe_dense_4k() and returns: hparams