class_docstr: Lunar Multi-headed attention.
  See "Linformer: Self-Attention with Linear Complexity" for more details.
code: def __init__(
      self,
      embed_dim,
      num_heads,
      num_pheads,
      dropout=0.0,
      bias=True,
      self_attention=False,
      encoder_decoder_attention=False,
      tie_kv=True,
  ):
      super().__init__()
      self.embed_dim = embed_dim
      self.num_heads = num_heads
      self.num_pheads = num_pheads
      self.dropout_module = FairseqDropout(
          dropout, module_name=self.__class__.__name__
      )
  
      self.head_dim = embed_dim // num_heads
      self.phead_dim = embed_dim // num_pheads
      assert (
          self.head_dim * num_heads == self.embed_dim
      ), "embed_dim must be divisible by num_heads"
      assert (
          self.phead_dim * num_pheads == self.embed_dim
      ), "projected embed_dim must be divisible by num_pheads"
      self.scaling = self.head_dim**-0.5
      self.pscaling = self.phead_dim**-0.5
  
      self.self_attention = self_attention
      self.encoder_decoder_attention = encoder_decoder_attention
  
      self.pq_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
      self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
      if tie_kv:
          self.pc_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
          self.c_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
          self.pk_proj = self.k_proj = None
          self.pv_proj = self.v_proj = None
      else:
          self.pk_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
          self.pv_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
          self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
          self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
          self.pc_proj = self.c_proj = None
  
      self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
      self.reset_parameters()
  
      self.onnx_trace = False
      self.tpu = False
  
imports: import math
  from typing import Dict, Optional, Tuple, Union
  import torch
  import torch.nn.functional
  from fairseq import utils
  from fairseq.incremental_decoding_utils import with_incremental_state
  from fairseq.modules.fairseq_dropout import FairseqDropout
  from torch import nn, Tensor
label_desc: Use image node_id 1 to create a new LunarMultiheadAttention object from inherited base classes: nn with example: obj = LunarMultiheadAttention(embed_dim, num_heads, num_pheads, dropout, bias, _attention, encoder_decoder_attention, tie_kv)