code: def attention_lm_moe_large():
      """Large model for distributed training.
  
      Over 1B parameters, so requires multi-gpu training due to memory
       requirements.
  
      on lm1b_32k:
         After 45K steps on 8 GPUs (synchronous):
            eval_log_ppl_per_token = 3.18
            eval_ppl_per_word = exp(1.107893 * eval_log_ppl_per_token) = 33.9
  
      Returns:
        an hparams object.
      """
      hparams = attention_lm_moe_base()
      hparams.num_hidden_layers = 5
      hparams.moe_layers = "3"
      hparams.hidden_size = 1024
      hparams.num_heads = 16
      hparams.filter_size = 4096
      hparams.moe_hidden_sizes = "4096"
      hparams.moe_num_experts = 128
      hparams.layer_prepostprocess_dropout = 0.2
      return hparams
  
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import functools
  from six.moves import range
  from tensor2tensor.layers import common_attention
  from tensor2tensor.layers import common_hparams
  from tensor2tensor.layers import common_layers
  from tensor2tensor.utils import expert_utils
  from tensor2tensor.utils import registry
  from tensor2tensor.utils import t2t_model
  import tensorflow.compat.v1
  from tensorflow.compat.v1 import estimator
label_desc: Use image node_id 21 for calling a global function with example usage: attention_lm_moe_large() and returns: hparams