class_docstr: Multi-headed attention.
  
  See "Attention Is All You Need" for more details.
code: def __init__(
      self,
      embed_dim,
      num_heads,
      kdim=None,
      vdim=None,
      dropout=0.0,
      bias=True,
      add_bias_kv=False,
      add_zero_attn=False,
      self_attention=False,
      encoder_decoder_attention=False,
      q_noise=0.0,
      qn_block_size=8,
      has_relative_attention_bias=False,
      num_buckets=32,
      max_distance=128,
      gru_rel_pos=False,
      rescale_init=False,
  ):
      super().__init__()
      self.embed_dim = embed_dim
      self.kdim = kdim if kdim is not None else embed_dim
      self.vdim = vdim if vdim is not None else embed_dim
      self.qkv_same_dim = (
          self.kdim == embed_dim and self.vdim == embed_dim
      )
  
      self.num_heads = num_heads
      self.dropout_module = nn.Dropout(dropout)
  
      self.has_relative_attention_bias = has_relative_attention_bias
      self.num_buckets = num_buckets
      self.max_distance = max_distance
      if self.has_relative_attention_bias:
          self.relative_attention_bias = nn.Embedding(
              num_buckets, num_heads
          )
  
      self.head_dim = embed_dim // num_heads
      self.q_head_dim = self.head_dim
      self.k_head_dim = self.head_dim
      assert (
          self.head_dim * num_heads == self.embed_dim
      ), "embed_dim must be divisible by num_heads"
      self.scaling = self.head_dim**-0.5
  
      self.self_attention = self_attention
      self.encoder_decoder_attention = encoder_decoder_attention
  
      assert not self.self_attention or self.qkv_same_dim, (
          "Self-attention requires query, key and "
          "value to be of the same size"
      )
  
      k_bias = True
      if rescale_init:
          k_bias = False
  
      k_embed_dim = embed_dim
      q_embed_dim = embed_dim
  
      self.k_proj = quant_noise(
          nn.Linear(self.kdim, k_embed_dim, bias=k_bias),
          q_noise,
          qn_block_size,
      )
      self.v_proj = quant_noise(
          nn.Linear(self.vdim, embed_dim, bias=bias),
          q_noise,
          qn_block_size,
      )
      self.q_proj = quant_noise(
          nn.Linear(embed_dim, q_embed_dim, bias=bias),
          q_noise,
          qn_block_size,
      )
  
      self.out_proj = quant_noise(
          nn.Linear(embed_dim, embed_dim, bias=bias),
          q_noise,
          qn_block_size,
      )
  
      if add_bias_kv:
          self.bias_k = Parameter(torch.Tensor(1, 1, embed_dim))
          self.bias_v = Parameter(torch.Tensor(1, 1, embed_dim))
      else:
          self.bias_k = self.bias_v = None
  
      self.add_zero_attn = add_zero_attn
  
      self.gru_rel_pos = gru_rel_pos
      if self.gru_rel_pos:
          self.grep_linear = nn.Linear(self.q_head_dim, 8)
          self.grep_a = nn.Parameter(torch.ones(1, num_heads, 1, 1))
  
      self.reset_parameters()
  
imports: import math
  import numpy
  from typing import Dict, Optional, Tuple
  import torch
  from torch import Tensor, nn
  import torch.nn.functional
  from torch.nn import LayerNorm, Parameter
  from lavis.models.beats.modules import GradMultiply, SamePad, get_activation_fn, GLU_Linear, quant_noise
label_desc: Use image node_id 1 to create a new MultiheadAttention object from inherited base classes: nn with example: obj = MultiheadAttention(embed_dim, num_heads, kdim, vdim, dropout, bias, add_bias_kv, add_zero_attn, _attention, encoder_decoder_attention, q_noise, qn_block_size, has_relative_attention_bias, num_buckets, max_distance, gru_rel_pos, rescale_init)