class_docstr: Multi-Head Attention layer of Transformer-XL with sliding window local+global attention from Longformer.
  Partially adapted from allenai (https://github.com/allenai/longformer/blob/master/longformer/sliding_chunks.py)
  and huggingface (https://github.com/huggingface/transformers/blob/main/src/transformers/models/longformer/modeling_longformer.py) 
  Paper: https://arxiv.org/abs/1901.02860 (Transformer-XL),
         https://arxiv.org/abs/2004.05150 (Longformer)
  Args:
      n_head (int): number of heads
      n_feat (int): size of the features
      dropout_rate (float): dropout rate
      pos_bias_u (Tensor): the positional bias matrix U
      pos_bias_v (Tensor): the positional bias matrix V
      att_context_size (List[int]): List of 2 ints corresponding to left and right attention context sizes.
      max_cache_len (int): the maximum size of cache
      global_tokens (int): number of tokens to be used for global attention
      global_tokens_spacing (int): how far apart the global tokens are
      global_attn_separate (bool): whether the q, k, v layers used for global tokens should be separate
class_docstr_tok: ["Multi-Head","Attention","layer","of","Transformer-XL","with","sliding","window","local+global","attention","from","Longformer",".","Partially","adapted","from","allenai","(","https",":","\/\/github.com\/allenai\/longformer\/blob\/master\/longformer\/sliding_chunks.py",")","and","huggingface","(","https",":","\/\/github.com\/huggingface\/transformers\/blob\/main\/src\/transformers\/models\/longformer\/modeling_longformer.py",")","Paper",":","https",":","\/\/arxiv.org\/abs\/1901.02860","(","Transformer-XL",")",",","https",":","\/\/arxiv.org\/abs\/2004.05150","(","Longformer",")","Args",":","n_head","(","int",")",":","number","of","heads","n_feat","(","int",")",":","size","of","the","features","dropout_rate","(","float",")",":","dropout","rate","pos_bias_u","(","Tensor",")",":","the","positional","bias","matrix","U","pos_bias_v","(","Tensor",")",":","the","positional","bias","matrix","V","att_context_size","(","List","[","int","]",")",":","List","of","2","ints","corresponding","to","left","and","right","attention","context","sizes",".","max_cache_len","(","int",")",":","the","maximum","size","of","cache","global_tokens","(","int",")",":","number","of","tokens","to","be","used","for","global","attention","global_tokens_spacing","(","int",")",":","how","far","apart","the","global","tokens","are","global_attn_separate","(","bool",")",":","whether","the","q",",","k",",","v","layers","used","for","global","tokens","should","be","separate"]
docstr: Same as sliding_chunks_matmul_qk but for prob and value tensors.
  
  Args:
      prob (torch.Tensor): (batch, head, time, size)
      v (torch.Tensor): (batch, head, time, size)
      w (int): Chunk overlap size
  
  Returns:
      output (torch.Tensor): (batch, time, head, size)
docstr_tok: ["Same","as","sliding_chunks_matmul_qk","but","for","prob","and","value","tensors",".","Args",":","prob","(","torch.Tensor",")",":","(","batch",",","head",",","time",",","size",")","v","(","torch.Tensor",")",":","(","batch",",","head",",","time",",","size",")","w","(","int",")",":","Chunk","overlap","size","Returns",":","output","(","torch.Tensor",")",":","(","batch",",","time",",","head",",","size",")"]
code: def sliding_chunks_matmul_pv(
      self, prob: torch.Tensor, v: torch.Tensor, w: int
  ):
      """Same as sliding_chunks_matmul_qk but for prob and value tensors.
  
      Args:
          prob (torch.Tensor): (batch, head, time, size)
          v (torch.Tensor): (batch, head, time, size)
          w (int): Chunk overlap size
  
      Returns:
          output (torch.Tensor): (batch, time, head, size)
      """
      bsz, num_heads, seqlen, head_dim = v.size()
      chunks_count = seqlen // w - 1
      # group bsz and num_heads dimensions into one, then chunk seqlen into chunks of size 2w
      chunk_prob = prob.reshape(
          bsz * num_heads, seqlen // w, w, 2 * w + 1
      )
      # (batch x head, chunks_count + 1, w, 2w + 1)
  
      # group bsz and num_heads dimensions into one
      v = v.reshape(bsz * num_heads, seqlen, head_dim)
      # (batch x head, time, size)
  
      # pad seqlen with w at the beginning of the sequence and another w at the end
      padded_v = F.pad(v, (0, 0, w, w), value=-1)
      # (batch x head, time + 2w, size)
  
      # chunk padded_v into chunks of size 3w and an overlap of size w
      chunk_v_size = (
          bsz * num_heads,
          chunks_count + 1,
          3 * w,
          head_dim,
      )
      chunk_v_stride = padded_v.stride()
      chunk_v_stride = (
          chunk_v_stride[0],
          w * chunk_v_stride[1],
          chunk_v_stride[1],
          chunk_v_stride[2],
      )
      chunk_v = padded_v.as_strided(
          size=chunk_v_size, stride=chunk_v_stride
      )
      # (batch x head, chunks_count + 1, 3w, size)
  
      skewed_prob = self._skew2(chunk_prob, padding_value=0)
      # (batch x head, chunks_count + 1, w, 3w)
  
      context = torch.einsum("bcwd,bcdh->bcwh", (skewed_prob, chunk_v))
      # (batch x head, chunks_count + 1, w, size)
  
      return context.view(bsz, num_heads, seqlen, head_dim).transpose(
          1, 2
      )
  
imports: import math
  from functools import lru_cache
  from typing import List, Tuple
  import torch
  import torch.nn
  import torch.nn.functional
  from nemo.utils import avoid_float16_autocast_context
label_desc: Use image node_id 13 for calling the RelPositionMultiHeadAttentionLongformer obj's underlying member method code with example usage: obj.sliding_chunks_matmul_pv(prob, v, w) and returns: context