class_docstr: Transformer layer.
  
  This layer implements the Transformer from "Attention Is All You Need".
  (https://arxiv.org/abs/1706.03762).
  
  Attributes:
    num_attention_heads: Number of attention heads.
    intermediate_size: Size of the intermediate layer.
    intermediate_activation: Activation for the intermediate layer.
    dropout_rate: Dropout probability for the post-attention and output dropout.
    attention_dropout_rate: Dropout probability for within the attention layer.
    kernel_initializer: Initializer for dense layer kernels.
    bias_initializer: Initializer for dense layer biases.
    kernel_regularizer: Regularizer for dense layer kernels.
    bias_regularizer: Regularizer for dense layer biases.
    activity_regularizer: Regularizer for dense layer activity.
    kernel_constraint: Constraint for dense layer kernels.
    bias_constraint: Constraint for dense layer kernels.
class_docstr_tok: ["Transformer","layer",".","This","layer","implements","the","Transformer","from","``","Attention","Is","All","You","Need","''",".","(","https",":","\/\/arxiv.org\/abs\/1706.03762",")",".","Attributes",":","num_attention_heads",":","Number","of","attention","heads",".","intermediate_size",":","Size","of","the","intermediate","layer",".","intermediate_activation",":","Activation","for","the","intermediate","layer",".","dropout_rate",":","Dropout","probability","for","the","post-attention","and","output","dropout",".","attention_dropout_rate",":","Dropout","probability","for","within","the","attention","layer",".","kernel_initializer",":","Initializer","for","dense","layer","kernels",".","bias_initializer",":","Initializer","for","dense","layer","biases",".","kernel_regularizer",":","Regularizer","for","dense","layer","kernels",".","bias_regularizer",":","Regularizer","for","dense","layer","biases",".","activity_regularizer",":","Regularizer","for","dense","layer","activity",".","kernel_constraint",":","Constraint","for","dense","layer","kernels",".","bias_constraint",":","Constraint","for","dense","layer","kernels","."]
code: def __init__(
      self,
      num_attention_heads,
      intermediate_size,
      intermediate_activation,
      dropout_rate=0.0,
      attention_dropout_rate=0.0,
      kernel_initializer="glorot_uniform",
      bias_initializer="zeros",
      kernel_regularizer=None,
      bias_regularizer=None,
      activity_regularizer=None,
      kernel_constraint=None,
      bias_constraint=None,
      **kwargs,
  ):
      super(Transformer, self).__init__(**kwargs)
  
      self._num_heads = num_attention_heads
      self._intermediate_size = intermediate_size
      self._intermediate_activation = intermediate_activation
      self._attention_dropout_rate = attention_dropout_rate
      self._dropout_rate = dropout_rate
      self._kernel_initializer = tf.keras.initializers.get(
          kernel_initializer
      )
      self._bias_initializer = tf.keras.initializers.get(
          bias_initializer
      )
      self._kernel_regularizer = tf.keras.regularizers.get(
          kernel_regularizer
      )
      self._bias_regularizer = tf.keras.regularizers.get(
          bias_regularizer
      )
      self._kernel_constraint = tf.keras.constraints.get(
          kernel_constraint
      )
      self._bias_constraint = tf.keras.constraints.get(bias_constraint)
  
file_path: autodist/examples/benchmark/utils/modeling/layers/transformer.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import tensorflow
  from utils.modeling.layers import attention
  from utils.modeling.layers import dense_einsum
label_desc: Use image node_id 1 to create a new Transformer object from inherited base classes: tf.keras.layers with example: obj = Transformer(num_attention_heads, intermediate_size, intermediate_activation, dropout_rate, attention_dropout_rate, kernel_initializer, bias_initializer, kernel_regularizer, bias_regularizer, activity_regularizer, kernel_constraint, bias_constraint)