docstr: "Random" attention, as proposed for instance in BigBird_.
  Random means in that case that each query can attend to a random set of keys.
  This implementation is sparse-aware, meaning that the empty attention parts will not be represented in memory.
  
  Args:
      r (float): the ratio in [0,1] of keys that the query can attend to
      constant_masking (bool): if true, keep the same random set for all queries.
  
  .. _BigBird: https://arxiv.org/pdf/2007.14062.pdf
docstr_tok: ["``","Random","''","attention",",","as","proposed","for","instance","in","BigBird_",".","Random","means","in","that","case","that","each","query","can","attend","to","a","random","set","of","keys",".","This","implementation","is","sparse-aware",",","meaning","that","the","empty","attention","parts","will","not","be","represented","in","memory",".","Args",":","r","(","float",")",":","the","ratio","in","[","0,1","]","of","keys","that","the","query","can","attend","to","constant_masking","(","bool",")",":","if","true",",","keep","the","same","random","set","for","all","queries",".","..","_BigBird",":","https",":","\/\/arxiv.org\/pdf\/2007.14062.pdf"]
code: def __init__(
      self,
      dropout: float,
      causal: bool = False,
      r: float = 0.01,
      constant_masking: bool = True,
      force_sparsity: bool = False,
      *args,
      **kwargs,
  ):
      """
      "Random" attention, as proposed for instance in BigBird_.
      Random means in that case that each query can attend to a random set of keys.
      This implementation is sparse-aware, meaning that the empty attention parts will not be represented in memory.
  
      Args:
          r (float): the ratio in [0,1] of keys that the query can attend to
          constant_masking (bool): if true, keep the same random set for all queries.
  
      .. _BigBird: https://arxiv.org/pdf/2007.14062.pdf
  
      """
      super().__init__()
  
      self.attn_drop = nn.Dropout(dropout, inplace=False)
      self.causal = causal
      self.r = r
      self.rand_attention_mask: Optional[torch.Tensor] = None
      self.constant_masking = constant_masking
      self.force_sparsity = force_sparsity
  
      # Properties specific to this attention mechanism
      self.supports_attention_mask = True
      self.supports_key_padding_mask = False
  
      self.requires_same_k_q_dimensions = True
  
file_path: xformers/xformers/components/attention/random.py
imports: from dataclasses import dataclass
  from typing import Optional, Union
  import torch
  import torch.nn
  from xformers.components.attention import Attention, AttentionConfig, AttentionMask, maybe_sparsify, register_attention, sparsify
  from xformers.components.attention.attention_patterns import causal_1d_pattern, random_pattern
  from xformers.components.attention.core import scaled_dot_product_attention
label_desc: Use image node_id 1 to create a new RandomAttention object from inherited base classes: Attention with example: obj = RandomAttention(dropout, causal, r, constant_masking, force_sparsity)