class_docstr: Transformer layer.
  
  This layer implements the Transformer from "Attention Is All You Need".
  (https://arxiv.org/abs/1706.03762).
  
  Attributes:
    num_attention_heads: Number of attention heads.
    intermediate_size: Size of the intermediate layer.
    intermediate_activation: Activation for the intermediate layer.
    dropout_rate: Dropout probability for the post-attention and output dropout.
    attention_dropout_rate: Dropout probability for within the attention layer.
    kernel_initializer: Initializer for dense layer kernels.
    bias_initializer: Initializer for dense layer biases.
    kernel_regularizer: Regularizer for dense layer kernels.
    bias_regularizer: Regularizer for dense layer biases.
    activity_regularizer: Regularizer for dense layer activity.
    kernel_constraint: Constraint for dense layer kernels.
    bias_constraint: Constraint for dense layer kernels.
class_docstr_tok: ["Transformer","layer",".","This","layer","implements","the","Transformer","from","``","Attention","Is","All","You","Need","''",".","(","https",":","\/\/arxiv.org\/abs\/1706.03762",")",".","Attributes",":","num_attention_heads",":","Number","of","attention","heads",".","intermediate_size",":","Size","of","the","intermediate","layer",".","intermediate_activation",":","Activation","for","the","intermediate","layer",".","dropout_rate",":","Dropout","probability","for","the","post-attention","and","output","dropout",".","attention_dropout_rate",":","Dropout","probability","for","within","the","attention","layer",".","kernel_initializer",":","Initializer","for","dense","layer","kernels",".","bias_initializer",":","Initializer","for","dense","layer","biases",".","kernel_regularizer",":","Regularizer","for","dense","layer","kernels",".","bias_regularizer",":","Regularizer","for","dense","layer","biases",".","activity_regularizer",":","Regularizer","for","dense","layer","activity",".","kernel_constraint",":","Constraint","for","dense","layer","kernels",".","bias_constraint",":","Constraint","for","dense","layer","kernels","."]
code: def call(self, inputs):
      if isinstance(inputs, (list, tuple)) and len(inputs) == 2:
          input_tensor, attention_mask = inputs
      else:
          input_tensor, attention_mask = (inputs, None)
  
      attention_inputs = [input_tensor, input_tensor]
  
      if attention_mask is not None:
          attention_inputs.append(attention_mask)
  
      attention_output = self._attention_layer(attention_inputs)
      attention_output = self._attention_output_dense(attention_output)
      attention_output = self._attention_dropout(attention_output)
      # Use float32 in keras layer norm and the gelu activation in the
      # intermediate dense layer for numeric stability
      if self.dtype == tf.float16:
          input_tensor = tf.cast(input_tensor, tf.float32)
          attention_output = tf.cast(attention_output, tf.float32)
      attention_output = self._attention_layer_norm(
          input_tensor + attention_output
      )
      intermediate_output = self._intermediate_dense(attention_output)
      if self.dtype == tf.float16:
          # Casts to float32 so that activation is done in float32.
          intermediate_output = tf.cast(intermediate_output, tf.float32)
          intermediate_output = self._intermediate_activation_layer(
              intermediate_output
          )
          intermediate_output = tf.cast(intermediate_output, tf.float16)
      else:
          intermediate_output = self._intermediate_activation_layer(
              intermediate_output
          )
      layer_output = self._output_dense(intermediate_output)
      layer_output = self._output_dropout(layer_output)
      # Use float32 in keras layer norm for numeric stability
      if self.dtype == tf.float16:
          layer_output = tf.cast(layer_output, tf.float32)
      layer_output = self._output_layer_norm(
          layer_output + attention_output
      )
      if self.dtype == tf.float16:
          layer_output = tf.cast(layer_output, tf.float16)
  
      return layer_output
  
file_path: autodist/examples/benchmark/utils/modeling/layers/transformer.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import tensorflow
  from utils.modeling.layers import attention
  from utils.modeling.layers import dense_einsum
label_desc: Use image node_id 5 for calling the Transformer obj's underlying member method code with example usage: obj.call(inputs) and returns: layer_output