docstr: Attention approximation using Lambda layers, from
  "Lambda networks: modeling long-range interactions without attention.", Bello, I. (2021).
code: def __init__(
      self, dropout: float, seq_len: int, dim_head: int, *_, **__
  ):
      """
      Attention approximation using Lambda layers, from
      "Lambda networks: modeling long-range interactions without attention.", Bello, I. (2021).
      """
      super().__init__()
  
      # Possible extensions:
      # - support different dimensions for key and queries
      # - support varying dimensions in between inputs and outputs
      # - support u hyperparam
  
      self.rel_pos_emb = torch.nn.Parameter(
          torch.randn(2 * seq_len - 1, int(dim_head))
      )
      self.rel_pos = calc_rel_pos(seq_len)
      self.attn_drop = torch.nn.Dropout(dropout, inplace=True)
  
      # Properties specific to this attention mechanism
      self.requires_same_k_q_dimensions = True
      self.supports_attention_mask = False
      self.supports_key_padding_mask = False
  
file_path: xformers/xformers/components/attention/lambda_layer.py
imports: from dataclasses import dataclass
  import torch
  from xformers.components.attention import Attention, AttentionConfig, register_attention
label_desc: Use image node_id 1 to create a new LambdaLayer object from inherited base classes: Attention with example: obj = LambdaLayer(dropout, seq_len, dim_head)