class_docstr: Transformer layer.
  
  This layer implements the Transformer from "Attention Is All You Need".
  (https://arxiv.org/abs/1706.03762).
  
  Attributes:
    num_attention_heads: Number of attention heads.
    intermediate_size: Size of the intermediate layer.
    intermediate_activation: Activation for the intermediate layer.
    dropout_rate: Dropout probability for the post-attention and output dropout.
    attention_dropout_rate: Dropout probability for within the attention layer.
    kernel_initializer: Initializer for dense layer kernels.
    bias_initializer: Initializer for dense layer biases.
    kernel_regularizer: Regularizer for dense layer kernels.
    bias_regularizer: Regularizer for dense layer biases.
    activity_regularizer: Regularizer for dense layer activity.
    kernel_constraint: Constraint for dense layer kernels.
    bias_constraint: Constraint for dense layer kernels.
class_docstr_tok: ["Transformer","layer",".","This","layer","implements","the","Transformer","from","``","Attention","Is","All","You","Need","''",".","(","https",":","\/\/arxiv.org\/abs\/1706.03762",")",".","Attributes",":","num_attention_heads",":","Number","of","attention","heads",".","intermediate_size",":","Size","of","the","intermediate","layer",".","intermediate_activation",":","Activation","for","the","intermediate","layer",".","dropout_rate",":","Dropout","probability","for","the","post-attention","and","output","dropout",".","attention_dropout_rate",":","Dropout","probability","for","within","the","attention","layer",".","kernel_initializer",":","Initializer","for","dense","layer","kernels",".","bias_initializer",":","Initializer","for","dense","layer","biases",".","kernel_regularizer",":","Regularizer","for","dense","layer","kernels",".","bias_regularizer",":","Regularizer","for","dense","layer","biases",".","activity_regularizer",":","Regularizer","for","dense","layer","activity",".","kernel_constraint",":","Constraint","for","dense","layer","kernels",".","bias_constraint",":","Constraint","for","dense","layer","kernels","."]
code: def build(self, input_shape):
      input_tensor = (
          input_shape[0] if len(input_shape) == 2 else input_shape
      )
      input_tensor_shape = tf.TensorShape(input_tensor)
      if len(input_tensor_shape) != 3:
          raise ValueError(
              "TransformerLayer expects a three-dimensional input of "
              "shape [batch, sequence, width]."
          )
      batch_size, sequence_length, hidden_size = input_tensor_shape
  
      if len(input_shape) == 2:
          mask_tensor_shape = tf.TensorShape(input_shape[1])
          expected_mask_tensor_shape = tf.TensorShape(
              [batch_size, sequence_length, sequence_length]
          )
          if not expected_mask_tensor_shape.is_compatible_with(
              mask_tensor_shape
          ):
              raise ValueError(
                  "When passing a mask tensor to TransformerLayer, the "
                  "mask tensor must be of shape [batch, "
                  "sequence_length, sequence_length] (here %s). Got a "
                  "mask tensor of shape %s."
                  % (expected_mask_tensor_shape, mask_tensor_shape)
              )
      if hidden_size % self._num_heads != 0:
          raise ValueError(
              "The input size (%d) is not a multiple of the number of attention "
              "heads (%d)" % (hidden_size, self._num_heads)
          )
      self._attention_head_size = int(hidden_size // self._num_heads)
  
      self._attention_layer = attention.Attention(
          num_heads=self._num_heads,
          head_size=self._attention_head_size,
          dropout_rate=self._attention_dropout_rate,
          kernel_initializer=self._kernel_initializer,
          bias_initializer=self._bias_initializer,
          kernel_regularizer=self._kernel_regularizer,
          bias_regularizer=self._bias_regularizer,
          activity_regularizer=self._activity_regularizer,
          kernel_constraint=self._kernel_constraint,
          bias_constraint=self._bias_constraint,
          name="self_attention",
      )
      self._attention_output_dense = dense_einsum.DenseEinsum(
          output_shape=hidden_size,
          num_summed_dimensions=2,
          kernel_initializer=self._kernel_initializer,
          bias_initializer=self._bias_initializer,
          kernel_regularizer=self._kernel_regularizer,
          bias_regularizer=self._bias_regularizer,
          activity_regularizer=self._activity_regularizer,
          kernel_constraint=self._kernel_constraint,
          bias_constraint=self._bias_constraint,
          name="self_attention_output",
      )
      self._attention_dropout = tf.keras.layers.Dropout(
          rate=self._dropout_rate
      )
      self._attention_layer_norm = tf.keras.layers.LayerNormalization(
          name="self_attention_layer_norm",
          axis=-1,
          epsilon=1e-12,
          dtype=tf.float32,
      )
      self._intermediate_dense = dense_einsum.DenseEinsum(
          output_shape=self._intermediate_size,
          activation=None,
          kernel_initializer=self._kernel_initializer,
          bias_initializer=self._bias_initializer,
          kernel_regularizer=self._kernel_regularizer,
          bias_regularizer=self._bias_regularizer,
          activity_regularizer=self._activity_regularizer,
          kernel_constraint=self._kernel_constraint,
          bias_constraint=self._bias_constraint,
          name="intermediate",
      )
      self._intermediate_activation_layer = tf.keras.layers.Activation(
          self._intermediate_activation
      )
      self._output_dense = dense_einsum.DenseEinsum(
          output_shape=hidden_size,
          kernel_initializer=self._kernel_initializer,
          bias_initializer=self._bias_initializer,
          kernel_regularizer=self._kernel_regularizer,
          bias_regularizer=self._bias_regularizer,
          activity_regularizer=self._activity_regularizer,
          kernel_constraint=self._kernel_constraint,
          bias_constraint=self._bias_constraint,
          name="output",
      )
      self._output_dropout = tf.keras.layers.Dropout(
          rate=self._dropout_rate
      )
      self._output_layer_norm = tf.keras.layers.LayerNormalization(
          name="output_layer_norm",
          axis=-1,
          epsilon=1e-12,
          dtype=tf.float32,
      )
  
      super(Transformer, self).build(input_shape)
  
file_path: autodist/examples/benchmark/utils/modeling/layers/transformer.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import tensorflow
  from utils.modeling.layers import attention
  from utils.modeling.layers import dense_einsum
label_desc: Use image node_id 2 for calling the Transformer obj's underlying member method code with example usage: obj.build(input_shape) without return types