nlpmodels/models/transformer_blocks/attention.py
def compute_attention(self, query: torch.Tensor, key: torch.Tensor,
                          value: torch.Tensor, mask: torch.Tensor):
        """
        Attention function to compute scaled dot-product attention between query,
        key and determine which values to "pay attention to".


        Args:
            query (torch.Tensor):
                query 4D matrix of (batch_size, num_heads,max_seq_length, dim_keys) size
            key (torch.Tensor):
                key 4D matrix of (batch_size, num_heads,max_seq_length, dim_keys) size
            value (torch.Tensor):
                values 4D matrix of (batch_size, num_heads,max_seq_length, dim_keys) size
            mask (torch.Tensor):
                values 4D matrix of (batch_size,1,1,max_seq_length) size
        Returns:
            returns the attention values, attention probabilities (softmax output)
        """
        # confirm that dim 4 is correct
        assert query.size(-1) == self._dim_head

        # Scaled dot-product calculation (Q,K) -> scores
        # NOTE: matmul considers last 2 dimensions in batch matrix multiplication.
        # (batch_size, num_heads,max_seq_length, dim_keys) x
        # (batch_size, num_heads, dim_keys, max_seq_length)  ->
        # scores is (batch_size, num_heads, max_seq_length, max_seq_length)
        # NOTE: matmul, @ symbol do same operation
        scores = (query @ key.transpose(2, 3)) / math.sqrt(self._dim_head)
        # Wherever mask == False is where padding is, so set scores to -inf for softmax calculation
        # Note: would use float('-inf'), but softmax issue
        scores = scores.masked_fill(mask == 0, -1.e9)
        # Calculate soft-max of scores -> attention_probas (probabilities for each value)
        attention_probas = self._dropout(F.softmax(scores, dim=-1))
        # attention is expected values = attention_probas * values
        # (batch_size, num_heads, max_seq_length, max_seq_length) x
        # (batch_size, num_heads, max_seq_length, dim_keys) ->
        # attn_values is (batch_size, num_heads, max_seq_length, dim_keys)
        attention_values = attention_probas @ value
        return attention_values, attention_probas

        Attention function to compute scaled dot-product attention between query,
        key and determine which values to "pay attention to".


        Args:
            query (torch.Tensor):
                query 4D matrix of (batch_size, num_heads,max_seq_length, dim_keys) size
            key (torch.Tensor):
                key 4D matrix of (batch_size, num_heads,max_seq_length, dim_keys) size
            value (torch.Tensor):
                values 4D matrix of (batch_size, num_heads,max_seq_length, dim_keys) size
            mask (torch.Tensor):
                values 4D matrix of (batch_size,1,1,max_seq_length) size
        Returns:
            returns the attention values, attention probabilities (softmax output)
        
Attention function to compute scaled dot-product attention between query,
key and determine which values to "pay attention to".
Attention function to compute scaled dot - product attention between query key and determine which values to " pay attention to " .
def compute_attention(self, query: torch.Tensor, key: torch.Tensor,
                          value: torch.Tensor, mask: torch.Tensor):
        assert query.size(-1) == self._dim_head
        scores = (query @ key.transpose(2, 3)) / math.sqrt(self._dim_head)
        scores = scores.masked_fill(mask == 0, -1.e9)
        attention_probas = self._dropout(F.softmax(scores, dim=-1))
        attention_values = attention_probas @ value
        return attention_values, attention_probas
def compute_attention ( self , query : torch . Tensor , key : torch . Tensor , value : torch . Tensor , mask : torch . Tensor ) : assert query . size ( - 1 ) == self . _dim_head scores = ( query @ key . transpose ( 2 , 3 ) ) / math . sqrt ( self . _dim_head ) scores = scores . masked_fill ( mask == 0 , - 1.e9 ) attention_probas = self . _dropout ( F . softmax ( scores , dim = - 1 ) ) attention_values = attention_probas @ value return attention_values , attention_probas
Attention function to compute scaled dot-product attention between query,
key and determine which values to "pay attention to".
Attention function to compute scaled dot - product attention between query key and determine which values to " pay attention to " .
"""
        Attention function to compute scaled dot-product attention between query,
        key and determine which values to "pay attention to".


        Args:
            query (torch.Tensor):
                query 4D matrix of (batch_size, num_heads,max_seq_length, dim_keys) size
            key (torch.Tensor):
                key 4D matrix of (batch_size, num_heads,max_seq_length, dim_keys) size
            value (torch.Tensor):
                values 4D matrix of (batch_size, num_heads,max_seq_length, dim_keys) size
            mask (torch.Tensor):
                values 4D matrix of (batch_size,1,1,max_seq_length) size
        Returns:
            returns the attention values, attention probabilities (softmax output)
        """ # confirm that dim 4 is correct # Scaled dot-product calculation (Q,K) -> scores # NOTE: matmul considers last 2 dimensions in batch matrix multiplication. # (batch_size, num_heads,max_seq_length, dim_keys) x # (batch_size, num_heads, dim_keys, max_seq_length)  -> # scores is (batch_size, num_heads, max_seq_length, max_seq_length) # NOTE: matmul, @ symbol do same operation # Wherever mask == False is where padding is, so set scores to -inf for softmax calculation # Note: would use float('-inf'), but softmax issue # Calculate soft-max of scores -> attention_probas (probabilities for each value) # attention is expected values = attention_probas * values # (batch_size, num_heads, max_seq_length, max_seq_length) x # (batch_size, num_heads, max_seq_length, dim_keys) -> # attn_values is (batch_size, num_heads, max_seq_length, dim_keys)
returns the attention values, attention probabilities (softmax output)
returns the attention values attention probabilities ( softmax output )
self
query
torch.Tensor
query 4D matrix of (batch_size, num_heads,max_seq_length, dim_keys) size
query 4D matrix of ( batch_size num_heads max_seq_length dim_keys ) size
key
torch.Tensor
key 4D matrix of (batch_size, num_heads,max_seq_length, dim_keys) size
key 4D matrix of ( batch_size num_heads max_seq_length dim_keys ) size
value
torch.Tensor
values 4D matrix of (batch_size, num_heads,max_seq_length, dim_keys) size
values 4D matrix of ( batch_size num_heads max_seq_length dim_keys ) size
mask
torch.Tensor
values 4D matrix of (batch_size,1,1,max_seq_length) size
values 4D matrix of ( batch_size 1 1 max_seq_length ) size