class_docstr: Attention layer.
  
  This is an implementation of multi-headed attention based on "Attention
  is all you Need". If `from_tensor` and `to_tensor` are the same, then
  this is self-attention. Each timestep in `from_tensor` attends to the
  corresponding sequence in `to_tensor`, and returns a fixed-width vector.
  
  This function first projects `from_tensor` into a "query" tensor and
  `to_tensor` into "key" and "value" tensors. These are (effectively) a list
  of tensors of length `num_attention_heads`, where each tensor is of shape
  [batch_size, seq_length, size_per_head].
  
  Then, the query and key tensors are dot-producted and scaled. These are
  softmaxed to obtain attention probabilities. The value tensors are then
  interpolated by these probabilities, then concatenated back to a single
  tensor and returned.
  
  Attributes:
    num_heads: Number of attention heads.
    head_size: Size of each attention head.
    dropout: Dropout probability.
    kernel_initializer: Initializer for dense layer kernels.
    bias_initializer: Initializer for dense layer biases.
    kernel_regularizer: Regularizer for dense layer kernels.
    bias_regularizer: Regularizer for dense layer biases.
    activity_regularizer: Regularizer for dense layer activity.
    kernel_constraint: Constraint for dense layer kernels.
    bias_constraint: Constraint for dense layer kernels.
class_docstr_tok: ["Attention","layer",".","This","is","an","implementation","of","multi-headed","attention","based","on","``","Attention","is","all","you","Need","''",".","If","`","from_tensor","`","and","`","to_tensor","`","are","the","same",",","then","this","is","self-attention",".","Each","timestep","in","`","from_tensor","`","attends","to","the","corresponding","sequence","in","`","to_tensor","`",",","and","returns","a","fixed-width","vector",".","This","function","first","projects","`","from_tensor","`","into","a","``","query","''","tensor","and","`","to_tensor","`","into","``","key","''","and","``","value","''","tensors",".","These","are","(","effectively",")","a","list","of","tensors","of","length","`","num_attention_heads","`",",","where","each","tensor","is","of","shape","[","batch_size",",","seq_length",",","size_per_head","]",".","Then",",","the","query","and","key","tensors","are","dot-producted","and","scaled",".","These","are","softmaxed","to","obtain","attention","probabilities",".","The","value","tensors","are","then","interpolated","by","these","probabilities",",","then","concatenated","back","to","a","single","tensor","and","returned",".","Attributes",":","num_heads",":","Number","of","attention","heads",".","head_size",":","Size","of","each","attention","head",".","dropout",":","Dropout","probability",".","kernel_initializer",":","Initializer","for","dense","layer","kernels",".","bias_initializer",":","Initializer","for","dense","layer","biases",".","kernel_regularizer",":","Regularizer","for","dense","layer","kernels",".","bias_regularizer",":","Regularizer","for","dense","layer","biases",".","activity_regularizer",":","Regularizer","for","dense","layer","activity",".","kernel_constraint",":","Constraint","for","dense","layer","kernels",".","bias_constraint",":","Constraint","for","dense","layer","kernels","."]
code: def get_config(self):
      config = {
          "num_heads": self._num_heads,
          "head_size": self._head_size,
          "dropout_rate": self._dropout_rate,
          "kernel_initializer": tf.keras.initializers.serialize(
              self._kernel_initializer
          ),
          "bias_initializer": tf.keras.initializers.serialize(
              self._bias_initializer
          ),
          "kernel_regularizer": tf.keras.regularizers.serialize(
              self._kernel_regularizer
          ),
          "bias_regularizer": tf.keras.regularizers.serialize(
              self._bias_regularizer
          ),
          "activity_regularizer": tf.keras.regularizers.serialize(
              self._activity_regularizer
          ),
          "kernel_constraint": tf.keras.constraints.serialize(
              self._kernel_constraint
          ),
          "bias_constraint": tf.keras.constraints.serialize(
              self._bias_constraint
          ),
      }
      base_config = super(Attention, self).get_config()
      return dict(list(base_config.items()) + list(config.items()))
  
file_path: autodist/examples/benchmark/utils/modeling/layers/attention.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import math
  import tensorflow
  from utils.modeling.layers import dense_einsum
  from utils.modeling.layers import masked_softmax
label_desc: Use image node_id 3 for calling the Attention obj's underlying member method code with example usage: obj.get_config() and returns: dict