class_docstr: Multi-headed attention layer.
  
  Arguments:
      units: Positive integer, output dim of hidden layer.
      num_heads: Positive integer, number of heads
          to repeat the same attention structure.
      attention_dropout_rate: Float, dropout rate inside attention for training.
      density: Approximate fraction of trainable weights (in
          `RandomlyConnectedDense` layers).
      unidirectional: Boolean, use a unidirectional or bidirectional encoder.
      use_key_relative_position: Boolean, if 'True' use key
          relative embeddings in attention.
      use_value_relative_position: Boolean, if 'True' use value
          relative embeddings in attention.
      max_relative_position: Positive integer, max position for relative embeddings.
      heads_share_relative_embedding: Boolean, if 'True'
          heads will share relative embeddings.
class_docstr_tok: ["Multi-headed","attention","layer",".","Arguments",":","units",":","Positive","integer",",","output","dim","of","hidden","layer",".","num_heads",":","Positive","integer",",","number","of","heads","to","repeat","the","same","attention","structure",".","attention_dropout_rate",":","Float",",","dropout","rate","inside","attention","for","training",".","density",":","Approximate","fraction","of","trainable","weights","(","in","`","RandomlyConnectedDense","`","layers",")",".","unidirectional",":","Boolean",",","use","a","unidirectional","or","bidirectional","encoder",".","use_key_relative_position",":","Boolean",",","if","'True","'","use","key","relative","embeddings","in","attention",".","use_value_relative_position",":","Boolean",",","if","'True","'","use","value","relative","embeddings","in","attention",".","max_relative_position",":","Positive","integer",",","max","position","for","relative","embeddings",".","heads_share_relative_embedding",":","Boolean",",","if","'True'","heads","will","share","relative","embeddings","."]
docstr: Calculate the attention weights.
  
  query, key, value must have matching leading dimensions.
  key, value must have matching penultimate dimension,
  i.e.: seq_len_k = seq_len_v.
  The mask has different shapes depending on its type (padding or look ahead)
  but it must be broadcastable for addition.
  
  Arguments:
      query: A tensor with shape (..., length, depth).
      key: A tensor with shape (..., length, depth).
      value: A tensor with shape (..., length, depth).
      pad_mask: Float tensor with shape broadcastable
          to (..., length, length). Defaults to None.
  
  Returns:
      output: A tensor with shape (..., length, depth).
      attention_weights: A tensor with shape (..., length, length).
docstr_tok: ["Calculate","the","attention","weights",".","query",",","key",",","value","must","have","matching","leading","dimensions",".","key",",","value","must","have","matching","penultimate","dimension",",","i.e",".",":","seq_len_k","=","seq_len_v",".","The","mask","has","different","shapes","depending","on","its","type","(","padding","or","look","ahead",")","but","it","must","be","broadcastable","for","addition",".","Arguments",":","query",":","A","tensor","with","shape","(","...",",","length",",","depth",")",".","key",":","A","tensor","with","shape","(","...",",","length",",","depth",")",".","value",":","A","tensor","with","shape","(","...",",","length",",","depth",")",".","pad_mask",":","Float","tensor","with","shape","broadcastable","to","(","...",",","length",",","length",")",".","Defaults","to","None",".","Returns",":","output",":","A","tensor","with","shape","(","...",",","length",",","depth",")",".","attention_weights",":","A","tensor","with","shape","(","...",",","length",",","length",")","."]
code: def _scaled_dot_product_attention(
      self,
      query: tf.Tensor,
      key: tf.Tensor,
      value: tf.Tensor,
      pad_mask: tf.Tensor,
      training: tf.Tensor,
  ) -> Tuple[tf.Tensor, tf.Tensor]:
      """Calculate the attention weights.
  
      query, key, value must have matching leading dimensions.
      key, value must have matching penultimate dimension,
      i.e.: seq_len_k = seq_len_v.
      The mask has different shapes depending on its type (padding or look ahead)
      but it must be broadcastable for addition.
  
      Arguments:
          query: A tensor with shape (..., length, depth).
          key: A tensor with shape (..., length, depth).
          value: A tensor with shape (..., length, depth).
          pad_mask: Float tensor with shape broadcastable
              to (..., length, length). Defaults to None.
  
      Returns:
          output: A tensor with shape (..., length, depth).
          attention_weights: A tensor with shape (..., length, length).
      """
      matmul_qk = tf.matmul(
          query, key, transpose_b=True
      )  # (..., length, length)
  
      if self.use_key_relative_position:
          matmul_qk += self._matmul_with_relative_keys(query)
  
      # scale matmul_qk
      dk = tf.cast(tf.shape(key)[-1], tf.float32)
      logits = matmul_qk / tf.math.sqrt(dk)
  
      # add the mask to the scaled tensor.
      if pad_mask is not None:
          logits += pad_mask * -1e9
  
      # apply attention dropout before softmax to maintain attention_weights norm as 1
      if self.attention_dropout_rate > 0:
          logits = self._drop_attention_logits(
              logits, pad_mask, training
          )
  
      # softmax is normalized on the last axis (length) so that the scores
      # add up to 1.
      attention_weights = tf.nn.softmax(
          logits, axis=-1
      )  # (..., length, length)
  
      output = tf.matmul(
          attention_weights, value
      )  # (..., length, depth)
      if self.use_value_relative_position:
          output += self._matmul_with_relative_values(attention_weights)
  
      return output, attention_weights
  
imports: from typing import Optional, Text, Tuple, Union
  import numpy
  import tensorflow
  from keras.utils.control_flow_util import smart_cond
  from tensorflow.keras import backend
  import rasa.shared.utils.cli
  from rasa.utils.tensorflow.layers import RandomlyConnectedDense
label_desc: Use image node_id 11 for calling the MultiHeadAttention obj's underlying member method code with example usage: obj._scaled_dot_product_attention(query, key, value, pad_mask, training) and returns: output,  attention_weights