class_docstr: Sparse Multi-Headed Attention.
  
  "Generating Long Sequences with Sparse Transformers". Implements
  fixed factorized self attention, where l=stride and c=expressivity.
  A(1) includes all words in the stride window and A(2) takes a summary of c
  words from the end of each stride window.
  If is_bidirectional=False, we do not include any words past the current word,
  as in the paper.
class_docstr_tok: ["Sparse","Multi-Headed","Attention",".","``","Generating","Long","Sequences","with","Sparse","Transformers","''",".","Implements","fixed","factorized","self","attention",",","where","l=stride","and","c=expressivity",".","A","(","1",")","includes","all","words","in","the","stride","window","and","A","(","2",")","takes","a","summary","of","c","words","from","the","end","of","each","stride","window",".","If","is_bidirectional=False",",","we","do","not","include","any","words","past","the","current","word",",","as","in","the","paper","."]
code: def __init__(
      self,
      embed_dim,
      num_heads,
      kdim=None,
      vdim=None,
      dropout=0.0,
      bias=True,
      add_bias_kv=False,
      add_zero_attn=False,
      self_attention=False,
      encoder_decoder_attention=False,
      stride=32,
      expressivity=8,
      is_bidirectional=True,
  ):
      super().__init__(
          embed_dim,
          num_heads,
          kdim,
          vdim,
          dropout,
          bias,
          add_bias_kv,
          add_zero_attn,
          self_attention,
          encoder_decoder_attention,
      )
  
      self.is_bidirectional = is_bidirectional
      self.stride = stride
      self.expressivity = expressivity
      assert self.stride > 0 and self.stride >= self.expressivity
  
file_path: fairseq/fairseq/modules/sparse_multihead_attention.py
imports: import math
  import torch
  from .multihead_attention import MultiheadAttention
label_desc: Use image node_id 1 to create a new SparseMultiheadAttention object from inherited base classes: MultiheadAttention with example: obj = SparseMultiheadAttention(embed_dim, num_heads, kdim, vdim, dropout, bias, add_bias_kv, add_zero_attn, _attention, encoder_decoder_attention, stride, expressivity, is_bidirectional)