code: def _layer_stack(
      mp,
      inputs,
      self_attention_bias,
      layers,
      hparams,
      encoder_output=None,
      encoder_decoder_attention_bias=None,
  ):
      """A stack of layers.
  
      Args:
        mp: a Parallelism object
        inputs: a list of Tensors
        self_attention_bias: list of bias Tensor for self-attention
          (see common_attention.attention_bias())
        layers: a string
        hparams: hyperparameters for model
        encoder_output: optional list of tensors
        encoder_decoder_attention_bias: optional list of tensors
  
      Returns:
        y: a list of Tensors
      """
      layers = layers.strip(",").split(",")
  
      # scaled_dot_product_attention_with_projections uses a 3d attention bias
      # (no heads), where multihead_attention uses 4d attention bias.
      self_attention_bias_3d = mp(tf.squeeze, self_attention_bias, 1)
      if encoder_decoder_attention_bias is not None:
          encoder_decoder_attention_bias_3d = mp(
              tf.squeeze, encoder_decoder_attention_bias, 1
          )
      relu_dropout_broadcast_dims = (
          common_layers.comma_separated_string_to_integer_list(
              getattr(hparams, "relu_dropout_broadcast_dims", "")
          )
      )
      mix_size = int(hparams.mix_fraction * hparams.hidden_size)
      accumulator = inputs
      x = inputs
      for layer_num, layer_type in enumerate(layers):
          with tf.variable_scope("%s_%d" % (layer_type, layer_num)):
              tf.logging.info("%s_%d" % (layer_type, layer_num))
              if layer_type == "a":
                  # accumulate
                  accumulator = mp(tf.add, x, accumulator)
                  x = accumulator
              elif layer_type == "n":
                  # normalize
                  x = mp(
                      common_layers.apply_norm,
                      x,
                      hparams.norm_type,
                      hparams.hidden_size,
                      hparams.norm_epsilon,
                  )
              elif layer_type == "d":
                  # dropout
                  x = mp(
                      tf.nn.dropout,
                      x,
                      1.0 - hparams.layer_prepostprocess_dropout,
                  )
              elif layer_type == "m":
                  if mix_size > 0:
                      # mix across shards
                      def _split(t):
                          return tuple(
                              tf.split(
                                  t,
                                  [
                                      mix_size,
                                      hparams.hidden_size - mix_size,
                                  ],
                                  2,
                              )
                          )
  
                      to_mix, to_keep = mp(_split, x)
                      mixed = expert_utils.all_reduce_ring(to_mix, mp)
                      mixed = mp(tf.multiply, mixed, mp.n**-0.5)
                      x = mp(
                          lambda a, b: tf.concat([a, b], 2),
                          mixed,
                          to_keep,
                      )
              elif layer_type == "att":
                  # single-head attention
                  q = mp(
                      tf.layers.dense,
                      x,
                      hparams.hidden_size,
                      use_bias=False,
                      name="q_transform",
                  )
                  x = mp(
                      common_attention.scaled_dot_product_attention_simple,
                      q,
                      x,
                      x,
                      self_attention_bias_3d,
                  )
                  x = mp(
                      tf.layers.dense,
                      x,
                      hparams.hidden_size,
                      use_bias=False,
                      name="o_transform",
                  )
              elif layer_type == "enc-att":
                  # single-head attention over encoder
                  q = mp(
                      tf.layers.dense,
                      x,
                      hparams.hidden_size,
                      use_bias=False,
                      name="q_transform",
                  )
                  assert encoder_output is not None
                  x = mp(
                      common_attention.scaled_dot_product_attention_simple,
                      q,
                      encoder_output,
                      encoder_output,
                      encoder_decoder_attention_bias_3d,
                  )
                  x = mp(
                      tf.layers.dense,
                      x,
                      hparams.hidden_size,
                      use_bias=False,
                      name="o_transform",
                  )
              elif layer_type == "multihead-att":
                  # multi-head attention
                  x = mp(
                      common_attention.multihead_attention,
                      x,
                      None,
                      self_attention_bias,  # bias
                      hparams.multihead_attention_key_channels
                      or hparams.hidden_size,
                      hparams.multihead_attention_value_channels
                      or hparams.hidden_size,
                      hparams.hidden_size,
                      hparams.multihead_attention_num_heads,
                      hparams.attention_dropout,
                  )
              elif layer_type == "enc-multihead-att":
                  # multi-head attention
                  x = mp(
                      common_attention.multihead_attention,
                      x,
                      encoder_output,
                      encoder_decoder_attention_bias,  # bias
                      hparams.multihead_attention_key_channels
                      or hparams.hidden_size,
                      hparams.multihead_attention_value_channels
                      or hparams.hidden_size,
                      hparams.hidden_size,
                      hparams.multihead_attention_num_heads,
                      hparams.attention_dropout,
                  )
              elif layer_type == "ffn":
                  x = mp(
                      common_layers.dense_relu_dense,
                      x,
                      hparams.filter_size,
                      hparams.hidden_size,
                      dropout=hparams.relu_dropout,
                      dropout_broadcast_dims=[
                          relu_dropout_broadcast_dims
                      ]
                      * mp.n,
                  )
              else:
                  assert False, "unknown sublayer %s" % layer_type
      return x
  
file_path: tensor2tensor/tensor2tensor/models/research/transformer_symshard.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from six.moves import range
  from tensor2tensor.layers import common_attention
  from tensor2tensor.layers import common_hparams
  from tensor2tensor.layers import common_layers
  from tensor2tensor.layers import modalities
  from tensor2tensor.utils import expert_utils
  from tensor2tensor.utils import registry
  from tensor2tensor.utils import t2t_model
  import tensorflow.compat.v1
label_desc: Use image node_id 1 for calling a global function with example usage: _layer_stack(mp, inputs, _attention_bias, layers, hparams, encoder_output, encoder_decoder_attention_bias) and returns: x, tuple