src/transformers/models/longformer/modeling_tf_longformer.py
def _sliding_chunks_query_key_matmul(self, query, key, window_overlap):
        """
        Matrix multiplication of query and key tensors using with a sliding window attention pattern. This
        implementation splits the input into overlapping chunks of size 2w (e.g. 512 for pretrained Longformer) with an
        overlap of size window_overlap
        """
        batch_size, seq_len, num_heads, head_dim = shape_list(query)

        tf.debugging.assert_equal(
            seq_len % (window_overlap * 2),
            0,
            message=f"Sequence length should be multiple of {window_overlap * 2}. Given {seq_len}",
        )
        tf.debugging.assert_equal(
            shape_list(query),
            shape_list(key),
            message=f"Shape of query and key should be equal, but got query: {shape_list(query)} and key: {shape_list(key)}",
        )

        chunks_count = seq_len // window_overlap - 1

        # group batch_size and num_heads dimensions into one, then chunk seq_len into chunks of size window_overlap * 2
        query = tf.reshape(
            tf.transpose(query, (0, 2, 1, 3)),
            (batch_size * num_heads, seq_len, head_dim),
        )
        key = tf.reshape(tf.transpose(key, (0, 2, 1, 3)), (batch_size * num_heads, seq_len, head_dim))
        chunked_query = self._chunk(query, window_overlap)
        chunked_key = self._chunk(key, window_overlap)

        # matrix multiplication
        # bcxd: batch_size * num_heads x chunks x 2window_overlap x head_dim
        # bcyd: batch_size * num_heads x chunks x 2window_overlap x head_dim
        # bcxy: batch_size * num_heads x chunks x 2window_overlap x 2window_overlap
        chunked_attention_scores = tf.einsum("bcxd,bcyd->bcxy", chunked_query, chunked_key)  # multiply

        # convert diagonals into columns
        paddings = tf.convert_to_tensor([[0, 0], [0, 0], [0, 1], [0, 0]], dtype=tf.dtypes.int32)
        diagonal_chunked_attention_scores = self._pad_and_transpose_last_two_dims(chunked_attention_scores, paddings)

        # allocate space for the overall attention matrix where the chunks are combined. The last dimension
        # has (window_overlap * 2 + 1) columns. The first (window_overlap) columns are the window_overlap lower triangles (attention from a word to
        # window_overlap previous words). The following column is attention score from each word to itself, then
        # followed by window_overlap columns for the upper triangle.

        # copy parts from diagonal_chunked_attention_scores into the combined matrix of attentions
        # - copying the main diagonal and the upper triangle
        # TODO: This code is most likely not very efficient and should be improved
        diagonal_attn_scores_up_triang = tf.concat(
            [
                diagonal_chunked_attention_scores[:, :, :window_overlap, : window_overlap + 1],
                diagonal_chunked_attention_scores[:, -1:, window_overlap:, : window_overlap + 1],
            ],
            axis=1,
        )

        # - copying the lower triangle
        diagonal_attn_scores_low_triang = tf.concat(
            [
                tf.zeros((batch_size * num_heads, 1, window_overlap, window_overlap)),
                diagonal_chunked_attention_scores[:, :, -(window_overlap + 1) : -1, window_overlap + 1 :],
            ],
            axis=1,
        )
        diagonal_attn_scores_first_chunk = tf.concat(
            [
                tf.roll(
                    diagonal_chunked_attention_scores,
                    shift=[1, window_overlap],
                    axis=[2, 3],
                )[:, :, :window_overlap, :window_overlap],
                tf.zeros((batch_size * num_heads, 1, window_overlap, window_overlap)),
            ],
            axis=1,
        )
        first_chunk_mask = (
            tf.tile(
                tf.range(chunks_count + 1)[None, :, None, None],
                (batch_size * num_heads, 1, window_overlap, window_overlap),
            )
            < 1
        )
        diagonal_attn_scores_low_triang = tf.where(
            first_chunk_mask,
            diagonal_attn_scores_first_chunk,
            diagonal_attn_scores_low_triang,
        )

        # merging upper and lower triangle
        diagonal_attention_scores = tf.concat(
            [diagonal_attn_scores_low_triang, diagonal_attn_scores_up_triang], axis=-1
        )

        # separate batch_size and num_heads dimensions again
        diagonal_attention_scores = tf.transpose(
            tf.reshape(
                diagonal_attention_scores,
                (batch_size, num_heads, seq_len, 2 * window_overlap + 1),
            ),
            (0, 2, 1, 3),
        )

        diagonal_attention_scores = self._mask_invalid_locations(diagonal_attention_scores, window_overlap)

        return diagonal_attention_scores

        Matrix multiplication of query and key tensors using with a sliding window attention pattern. This
        implementation splits the input into overlapping chunks of size 2w (e.g. 512 for pretrained Longformer) with an
        overlap of size window_overlap
        
Matrix multiplication of query and key tensors using with a sliding window attention pattern. This
implementation splits the input into overlapping chunks of size 2w  with an
overlap of size window_overlap
Matrix multiplication of query and key tensors using with a sliding window attention pattern . This implementation splits the input into overlapping chunks of size 2w with an overlap of size window_overlap
def _sliding_chunks_query_key_matmul(self, query, key, window_overlap):
        batch_size, seq_len, num_heads, head_dim = shape_list(query)
        tf.debugging.assert_equal(
            seq_len % (window_overlap * 2),
            0,
            message=f"Sequence length should be multiple of {window_overlap * 2}. Given {seq_len}",
        )
        tf.debugging.assert_equal(
            shape_list(query),
            shape_list(key),
            message=f"Shape of query and key should be equal, but got query: {shape_list(query)} and key: {shape_list(key)}",
        )
        chunks_count = seq_len // window_overlap - 1
        query = tf.reshape(
            tf.transpose(query, (0, 2, 1, 3)),
            (batch_size * num_heads, seq_len, head_dim),
        )
        key = tf.reshape(tf.transpose(key, (0, 2, 1, 3)), (batch_size * num_heads, seq_len, head_dim))
        chunked_query = self._chunk(query, window_overlap)
        chunked_key = self._chunk(key, window_overlap)
        chunked_attention_scores = tf.einsum("bcxd,bcyd->bcxy", chunked_query, chunked_key)  
        paddings = tf.convert_to_tensor([[0, 0], [0, 0], [0, 1], [0, 0]], dtype=tf.dtypes.int32)
        diagonal_chunked_attention_scores = self._pad_and_transpose_last_two_dims(chunked_attention_scores, paddings)
        diagonal_attn_scores_up_triang = tf.concat(
            [
                diagonal_chunked_attention_scores[:, :, :window_overlap, : window_overlap + 1],
                diagonal_chunked_attention_scores[:, -1:, window_overlap:, : window_overlap + 1],
            ],
            axis=1,
        )
        diagonal_attn_scores_low_triang = tf.concat(
            [
                tf.zeros((batch_size * num_heads, 1, window_overlap, window_overlap)),
                diagonal_chunked_attention_scores[:, :, -(window_overlap + 1) : -1, window_overlap + 1 :],
            ],
            axis=1,
        )
        diagonal_attn_scores_first_chunk = tf.concat(
            [
                tf.roll(
                    diagonal_chunked_attention_scores,
                    shift=[1, window_overlap],
                    axis=[2, 3],
                )[:, :, :window_overlap, :window_overlap],
                tf.zeros((batch_size * num_heads, 1, window_overlap, window_overlap)),
            ],
            axis=1,
        )
        first_chunk_mask = (
            tf.tile(
                tf.range(chunks_count + 1)[None, :, None, None],
                (batch_size * num_heads, 1, window_overlap, window_overlap),
            )
            < 1
        )
        diagonal_attn_scores_low_triang = tf.where(
            first_chunk_mask,
            diagonal_attn_scores_first_chunk,
            diagonal_attn_scores_low_triang,
        )
        diagonal_attention_scores = tf.concat(
            [diagonal_attn_scores_low_triang, diagonal_attn_scores_up_triang], axis=-1
        )
        diagonal_attention_scores = tf.transpose(
            tf.reshape(
                diagonal_attention_scores,
                (batch_size, num_heads, seq_len, 2 * window_overlap + 1),
            ),
            (0, 2, 1, 3),
        )
        diagonal_attention_scores = self._mask_invalid_locations(diagonal_attention_scores, window_overlap)
        return diagonal_attention_scores
def _sliding_chunks_query_key_matmul ( self , query , key , window_overlap ) : batch_size , seq_len , num_heads , head_dim = shape_list ( query ) tf . debugging . assert_equal ( seq_len % ( window_overlap * 2 ) , 0 , message = f"Sequence length should be multiple of {window_overlap * 2}. Given {seq_len}" , ) tf . debugging . assert_equal ( shape_list ( query ) , shape_list ( key ) , message = f"Shape of query and key should be equal, but got query: {shape_list(query)} and key: {shape_list(key)}" , ) chunks_count = seq_len // window_overlap - 1 query = tf . reshape ( tf . transpose ( query , ( 0 , 2 , 1 , 3 ) ) , ( batch_size * num_heads , seq_len , head_dim ) , ) key = tf . reshape ( tf . transpose ( key , ( 0 , 2 , 1 , 3 ) ) , ( batch_size * num_heads , seq_len , head_dim ) ) chunked_query = self . _chunk ( query , window_overlap ) chunked_key = self . _chunk ( key , window_overlap ) chunked_attention_scores = tf . einsum ( "bcxd,bcyd->bcxy" , chunked_query , chunked_key ) paddings = tf . convert_to_tensor ( [ [ 0 , 0 ] , [ 0 , 0 ] , [ 0 , 1 ] , [ 0 , 0 ] ] , dtype = tf . dtypes . int32 ) diagonal_chunked_attention_scores = self . _pad_and_transpose_last_two_dims ( chunked_attention_scores , paddings ) diagonal_attn_scores_up_triang = tf . concat ( [ diagonal_chunked_attention_scores [ : , : , : window_overlap , : window_overlap + 1 ] , diagonal_chunked_attention_scores [ : , - 1 : , window_overlap : , : window_overlap + 1 ] , ] , axis = 1 , ) diagonal_attn_scores_low_triang = tf . concat ( [ tf . zeros ( ( batch_size * num_heads , 1 , window_overlap , window_overlap ) ) , diagonal_chunked_attention_scores [ : , : , - ( window_overlap + 1 ) : - 1 , window_overlap + 1 : ] , ] , axis = 1 , ) diagonal_attn_scores_first_chunk = tf . concat ( [ tf . roll ( diagonal_chunked_attention_scores , shift = [ 1 , window_overlap ] , axis = [ 2 , 3 ] , ) [ : , : , : window_overlap , : window_overlap ] , tf . zeros ( ( batch_size * num_heads , 1 , window_overlap , window_overlap ) ) , ] , axis = 1 , ) first_chunk_mask = ( tf . tile ( tf . range ( chunks_count + 1 ) [ None , : , None , None ] , ( batch_size * num_heads , 1 , window_overlap , window_overlap ) , ) < 1 ) diagonal_attn_scores_low_triang = tf . where ( first_chunk_mask , diagonal_attn_scores_first_chunk , diagonal_attn_scores_low_triang , ) diagonal_attention_scores = tf . concat ( [ diagonal_attn_scores_low_triang , diagonal_attn_scores_up_triang ] , axis = - 1 ) diagonal_attention_scores = tf . transpose ( tf . reshape ( diagonal_attention_scores , ( batch_size , num_heads , seq_len , 2 * window_overlap + 1 ) , ) , ( 0 , 2 , 1 , 3 ) , ) diagonal_attention_scores = self . _mask_invalid_locations ( diagonal_attention_scores , window_overlap ) return diagonal_attention_scores
Matrix multiplication of query and key tensors using with a sliding window attention pattern.
Matrix multiplication of query and key tensors using with a sliding window attention pattern .
"""
        Matrix multiplication of query and key tensors using with a sliding window attention pattern. This
        implementation splits the input into overlapping chunks of size 2w (e.g. 512 for pretrained Longformer) with an
        overlap of size window_overlap
        """ # group batch_size and num_heads dimensions into one, then chunk seq_len into chunks of size window_overlap * 2 # matrix multiplication # bcxd: batch_size * num_heads x chunks x 2window_overlap x head_dim # bcyd: batch_size * num_heads x chunks x 2window_overlap x head_dim # bcxy: batch_size * num_heads x chunks x 2window_overlap x 2window_overlap # multiply # convert diagonals into columns # allocate space for the overall attention matrix where the chunks are combined. The last dimension # has (window_overlap * 2 + 1) columns. The first (window_overlap) columns are the window_overlap lower triangles (attention from a word to # window_overlap previous words). The following column is attention score from each word to itself, then # followed by window_overlap columns for the upper triangle. # copy parts from diagonal_chunked_attention_scores into the combined matrix of attentions # - copying the main diagonal and the upper triangle # TODO: This code is most likely not very efficient and should be improved # - copying the lower triangle # merging upper and lower triangle # separate batch_size and num_heads dimensions again
self
query
key
window_overlap