import tensorflow as tf

    p.transformer_tpl.tr_atten_tpl.num_attention_heads = 8
    p.transformer_tpl.tr_fflayer_tpl.hidden_dim = 8192
    return p

  @base_layer.initializer
  def __init__(self, params):
    super(TransformerStack, self).__init__(params)
    p = self.params

    with tf.variable_scope(p.name):
      # Add transformer layers.
      transformer_layer_params = []
      for i in range(p.num_transformer_layers):
        params = p.transformer_tpl.Copy()
        params.name = 'trans_%d' % (i)
        params.source_dim = p.model_dim
        params.packed_input = p.packed_input
        params.has_aux_atten = p.has_aux_attention
