class_docstr: BERT self-attention layer from Huggingface transformers.
  
  Compared to the BertSelfAttention of Huggingface, only add the clamp.
  
  Args:
      config (:class:`~transformers.BertConfig`):
          The configuration object that
          contains various parameters for the model.
      clamp_min_for_underflow (bool, optional):
          Whether to clamp the minimum value of the hidden states
           to prevent underflow. Defaults to `False`.
      clamp_max_for_overflow (bool, optional):
          Whether to clamp the maximum value of the hidden states
          to prevent overflow. Defaults to `False`.
class_docstr_tok: ["BERT","self-attention","layer","from","Huggingface","transformers",".","Compared","to","the","BertSelfAttention","of","Huggingface",",","only","add","the","clamp",".","Args",":","config","(",":","class",":","`","~transformers.BertConfig","`",")",":","The","configuration","object","that","contains","various","parameters","for","the","model",".","clamp_min_for_underflow","(","bool",",","optional",")",":","Whether","to","clamp","the","minimum","value","of","the","hidden","states","to","prevent","underflow",".","Defaults","to","`","False","`",".","clamp_max_for_overflow","(","bool",",","optional",")",":","Whether","to","clamp","the","maximum","value","of","the","hidden","states","to","prevent","overflow",".","Defaults","to","`","False","`","."]
docstr: Perform a forward pass through the BERT self-attention layer.
docstr_tok: ["Perform","a","forward","pass","through","the","BERT","self-attention","layer","."]
code: def forward(
      self,
      hidden_states: Tensor,
      attention_mask: Optional[Tensor] = None,
      head_mask: Optional[Tensor] = None,
      encoder_hidden_states: Optional[Tensor] = None,
      encoder_attention_mask: Optional[Tensor] = None,
      past_key_value: Optional[Tuple[Tensor, Tensor]] = None,
      output_attentions: bool = False,
  ) -> Tuple[Tensor, ...]:
      """Perform a forward pass through the BERT self-attention layer."""
  
      mixed_query_layer = self.query(hidden_states)
  
      # If this is instantiated as a cross-attention module, the keys
      # and values come from an encoder; the attention mask needs to be
      # such that the encoder's padding tokens are not attended to.
      is_cross_attention = encoder_hidden_states is not None
  
      if is_cross_attention and past_key_value is not None:
          # reuse k,v, cross_attentions
          key_layer = past_key_value[0]
          value_layer = past_key_value[1]
          attention_mask = encoder_attention_mask
      elif is_cross_attention:
          key_layer = self.transpose_for_scores(
              self.key(encoder_hidden_states)
          )
          value_layer = self.transpose_for_scores(
              self.value(encoder_hidden_states)
          )
          attention_mask = encoder_attention_mask
      elif past_key_value is not None:
          key_layer = self.transpose_for_scores(self.key(hidden_states))
          value_layer = self.transpose_for_scores(
              self.value(hidden_states)
          )
          key_layer = torch.cat([past_key_value[0], key_layer], dim=2)
          value_layer = torch.cat(
              [past_key_value[1], value_layer], dim=2
          )
      else:
          key_layer = self.transpose_for_scores(self.key(hidden_states))
          value_layer = self.transpose_for_scores(
              self.value(hidden_states)
          )
  
      query_layer = self.transpose_for_scores(mixed_query_layer)
  
      if self.is_decoder:
          past_key_value = (key_layer, value_layer)
  
      # Take the dot product between "query" and "key"
      # to get the raw attention scores.
      attention_scores = torch.matmul(
          query_layer, key_layer.transpose(-1, -2)
      )
  
      if (
          self.position_embedding_type == "relative_key"
          or self.position_embedding_type == "relative_key_query"
      ):
          seq_length = hidden_states.size()[1]
          position_ids_l = torch.arange(
              seq_length, dtype=torch.long, device=hidden_states.device
          ).view(-1, 1)
          position_ids_r = torch.arange(
              seq_length, dtype=torch.long, device=hidden_states.device
          ).view(1, -1)
          distance = position_ids_l - position_ids_r
          positional_embedding = self.distance_embedding(
              distance + self.max_position_embeddings - 1
          )
          positional_embedding = positional_embedding.to(
              dtype=query_layer.dtype
          )  # fp16 compatibility
  
          if self.position_embedding_type == "relative_key":
              relative_position_scores = torch.einsum(
                  "bhld,lrd->bhlr", query_layer, positional_embedding
              )
              attention_scores = (
                  attention_scores + relative_position_scores
              )
          elif self.position_embedding_type == "relative_key_query":
              relative_position_scores_query = torch.einsum(
                  "bhld,lrd->bhlr", query_layer, positional_embedding
              )
              relative_position_scores_key = torch.einsum(
                  "bhrd,lrd->bhlr", key_layer, positional_embedding
              )
              attention_scores = (
                  attention_scores
                  + relative_position_scores_query
                  + relative_position_scores_key
              )
  
      attention_scores = attention_scores / math.sqrt(
          self.attention_head_size
      )
  
      if self.clamp_min_for_underflow:
          attention_scores = torch.clamp(
              attention_scores, min=-MAX_CLAMP_VALUE
          )  # Do not increase -50000, data type half has quite limited range
      if self.clamp_max_for_overflow:
          attention_scores = torch.clamp(
              attention_scores, max=MAX_CLAMP_VALUE
          )  # Do not increase 50000, data type half has quite limited range
  
      if attention_mask is not None:
          # Apply the attention mask is
          # (precomputed for all layers in BertModel forward() function)
          attention_scores = attention_scores + attention_mask
  
      # Normalize the attention scores to probabilities.
      attention_probs = nn.Softmax(dim=-1)(attention_scores)
  
      # This is actually dropping out entire tokens to attend to, which might
      # seem a bit unusual, but is taken from the original Transformer paper.
      attention_probs = self.dropout(attention_probs)
  
      # Mask heads if we want to
      if head_mask is not None:
          attention_probs = attention_probs * head_mask
  
      context_layer = torch.matmul(attention_probs, value_layer)
  
      context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
      new_context_layer_shape = context_layer.size()[:-2] + (
          self.all_head_size,
      )
      context_layer = context_layer.view(*new_context_layer_shape)
  
      outputs = (
          (context_layer, attention_probs)
          if output_attentions
          else (context_layer,)
      )
  
      if self.is_decoder:
          outputs = outputs + (past_key_value,)
      return outputs
  
imports: import math
  from typing import Dict, Optional, Tuple
  import torch
  import torch.nn
  import torch.nn.functional
  import torch.utils.checkpoint
  from mmcv.cnn.bricks import DropPath
  from torch import Tensor
label_desc: Use image node_id 3 for calling the BertSelfAttention obj's underlying member method code with example usage: obj.forward(hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions) and returns: outputs