class_docstr: Multi-Head Attention layer of Transformer-XL with sliding window local+global attention from Longformer.
  Partially adapted from allenai (https://github.com/allenai/longformer/blob/master/longformer/sliding_chunks.py)
  and huggingface (https://github.com/huggingface/transformers/blob/main/src/transformers/models/longformer/modeling_longformer.py) 
  Paper: https://arxiv.org/abs/1901.02860 (Transformer-XL),
         https://arxiv.org/abs/2004.05150 (Longformer)
  Args:
      n_head (int): number of heads
      n_feat (int): size of the features
      dropout_rate (float): dropout rate
      pos_bias_u (Tensor): the positional bias matrix U
      pos_bias_v (Tensor): the positional bias matrix V
      att_context_size (List[int]): List of 2 ints corresponding to left and right attention context sizes.
      max_cache_len (int): the maximum size of cache
      global_tokens (int): number of tokens to be used for global attention
      global_tokens_spacing (int): how far apart the global tokens are
      global_attn_separate (bool): whether the q, k, v layers used for global tokens should be separate
class_docstr_tok: ["Multi-Head","Attention","layer","of","Transformer-XL","with","sliding","window","local+global","attention","from","Longformer",".","Partially","adapted","from","allenai","(","https",":","\/\/github.com\/allenai\/longformer\/blob\/master\/longformer\/sliding_chunks.py",")","and","huggingface","(","https",":","\/\/github.com\/huggingface\/transformers\/blob\/main\/src\/transformers\/models\/longformer\/modeling_longformer.py",")","Paper",":","https",":","\/\/arxiv.org\/abs\/1901.02860","(","Transformer-XL",")",",","https",":","\/\/arxiv.org\/abs\/2004.05150","(","Longformer",")","Args",":","n_head","(","int",")",":","number","of","heads","n_feat","(","int",")",":","size","of","the","features","dropout_rate","(","float",")",":","dropout","rate","pos_bias_u","(","Tensor",")",":","the","positional","bias","matrix","U","pos_bias_v","(","Tensor",")",":","the","positional","bias","matrix","V","att_context_size","(","List","[","int","]",")",":","List","of","2","ints","corresponding","to","left","and","right","attention","context","sizes",".","max_cache_len","(","int",")",":","the","maximum","size","of","cache","global_tokens","(","int",")",":","number","of","tokens","to","be","used","for","global","attention","global_tokens_spacing","(","int",")",":","how","far","apart","the","global","tokens","are","global_attn_separate","(","bool",")",":","whether","the","q",",","k",",","v","layers","used","for","global","tokens","should","be","separate"]
docstr: Compute the attention probabilities using only global key vectors.
  
  Args:
      key (torch.Tensor): (batch, time, head, head_dim) The key vectors.
      query (torch.Tensor): (batch, time, head, head_dim) The query vectors.
      max_num_global_attn_indices (int): Maximum number of global attention indices in the batch.
      is_index_global_attn_nonzero (tuple): Indices of global attention (non-zero elements).
      is_local_index_global_attn_nonzero (tuple): Non-padding values within global attention indices.
      is_local_index_no_global_attn_nonzero (tuple): Padding values within global attention indices.
  
  Returns:
      attn_probs_from_global_key (torch.Tensor): (batch, time, head, max_num_global_attn_indices) The computed attention probabilities using only global key vectors.
docstr_tok: ["Compute","the","attention","probabilities","using","only","global","key","vectors",".","Args",":","key","(","torch.Tensor",")",":","(","batch",",","time",",","head",",","head_dim",")","The","key","vectors",".","query","(","torch.Tensor",")",":","(","batch",",","time",",","head",",","head_dim",")","The","query","vectors",".","max_num_global_attn_indices","(","int",")",":","Maximum","number","of","global","attention","indices","in","the","batch",".","is_index_global_attn_nonzero","(","tuple",")",":","Indices","of","global","attention","(","non-zero","elements",")",".","is_local_index_global_attn_nonzero","(","tuple",")",":","Non-padding","values","within","global","attention","indices",".","is_local_index_no_global_attn_nonzero","(","tuple",")",":","Padding","values","within","global","attention","indices",".","Returns",":","attn_probs_from_global_key","(","torch.Tensor",")",":","(","batch",",","time",",","head",",","max_num_global_attn_indices",")","The","computed","attention","probabilities","using","only","global","key","vectors","."]
code: def _compute_global_key_attn(
      self,
      key: torch.Tensor,
      query: torch.Tensor,
      max_num_global_attn_indices: int,
      is_index_global_attn_nonzero: tuple,
      is_local_index_global_attn_nonzero: tuple,
      is_local_index_no_global_attn_nonzero: tuple,
  ) -> torch.Tensor:
      """
      Compute the attention probabilities using only global key vectors.
  
      Args:
          key (torch.Tensor): (batch, time, head, head_dim) The key vectors.
          query (torch.Tensor): (batch, time, head, head_dim) The query vectors.
          max_num_global_attn_indices (int): Maximum number of global attention indices in the batch.
          is_index_global_attn_nonzero (tuple): Indices of global attention (non-zero elements).
          is_local_index_global_attn_nonzero (tuple): Non-padding values within global attention indices.
          is_local_index_no_global_attn_nonzero (tuple): Padding values within global attention indices.
  
      Returns:
          attn_probs_from_global_key (torch.Tensor): (batch, time, head, max_num_global_attn_indices) The computed attention probabilities using only global key vectors.
      """
      batch_size = key.shape[0]
  
      # create only global key vectors
      key_only_global = key.new_zeros(
          batch_size, max_num_global_attn_indices, self.h, self.d_k
      )
  
      key_only_global[is_local_index_global_attn_nonzero] = key[
          is_index_global_attn_nonzero
      ]
  
      # (batch_size, seq_len, head, max_num_global_attn_indices)
      attn_probs_from_global_key = torch.einsum(
          "blhd,bshd->blhs", (query, key_only_global)
      )
  
      # need to transpose since ONNX export only supports consecutive indexing: https://pytorch.org/docs/stable/onnx.html#writes-sets
      attn_probs_from_global_key = attn_probs_from_global_key.transpose(
          1, 3
      )
      attn_probs_from_global_key[
          is_local_index_no_global_attn_nonzero[0],
          is_local_index_no_global_attn_nonzero[1],
          :,
          :,
      ] = torch.finfo(attn_probs_from_global_key.dtype).min
      attn_probs_from_global_key = attn_probs_from_global_key.transpose(
          1, 3
      )
  
      return attn_probs_from_global_key
  
imports: import math
  from functools import lru_cache
  from typing import List, Tuple
  import torch
  import torch.nn
  import torch.nn.functional
  from nemo.utils import avoid_float16_autocast_context
label_desc: Use image node_id 4 for calling the RelPositionMultiHeadAttentionLongformer obj's underlying member method code with example usage: obj._compute_global_key_attn(key, query, max_num_global_attn_indices, is_index_global_attn_nonzero, is_local_index_global_attn_nonzero, is_local_index_no_global_attn_nonzero) and returns: attn_probs_from_global_key