class_docstr: Multi-headed attention.
  
  See "Attention Is All You Need" for more details.
class_docstr_tok: ["Multi-headed","attention",".","See","``","Attention","Is","All","You","Need","''","for","more","details","."]
code: def __init__(
      self,
      embed_dim,
      num_heads,
      kdim=None,
      vdim=None,
      dropout=0.0,
      bias=True,
      add_bias_kv=False,
      add_zero_attn=False,
      self_attention=False,
      encoder_decoder_attention=False,
      is_decoder=False,
      has_causal_mask=False,
  ):
      super().__init__()
      self.embed_dim = embed_dim
      self.kdim = kdim if kdim is not None else embed_dim
      self.vdim = vdim if vdim is not None else embed_dim
      self.qkv_same_dim = (
          self.kdim == embed_dim and self.vdim == embed_dim
      )
      self.has_causal_mask = has_causal_mask
  
      self.num_heads = num_heads
      self.dropout_module = Dropout(dropout)
  
      self.head_dim = embed_dim // num_heads
      assert (
          self.head_dim * num_heads == self.embed_dim
      ), "embed_dim must be divisible by num_heads"
      self.scaling = self.head_dim**-0.5
  
      self.self_attention = self_attention
      self.encoder_decoder_attention = encoder_decoder_attention
      self.is_decoder = is_decoder
  
      max_positions = 1024
      self.register_buffer(
          "bias",
          torch.tril(
              torch.ones(
                  (max_positions, max_positions), dtype=torch.uint8
              )
          ).view(1, max_positions, max_positions),
      )
      self.register_buffer("masked_bias", torch.tensor(-1e4))
  
      assert (
          not self.self_attention or self.qkv_same_dim
      ), "Self-attention requires query, key and value to be of the same size"
  
      self.attention_quant = None
      if self.self_attention:
          # self.qkv_proj = Linear(embed_dim, 3*embed_dim, bias=bias)
          self.qkv_proj = QuantLinear(
              embed_dim, 3 * embed_dim, bias=bias
          )
  
          self.attention_quant = (
              TensorQuantizer(out_quant_config)
              if self.is_decoder
              else None
          )
      elif self.encoder_decoder_attention and self.is_decoder:
          self.k_proj = QuantLinear(
              self.kdim,
              embed_dim,
              pre_activation="encoder_out",
              bias=bias,
          )
          self.v_proj = QuantLinear(
              self.vdim,
              embed_dim,
              pre_activation="encoder_out",
              bias=bias,
          )
          self.q_proj = QuantLinear(embed_dim, embed_dim, bias=bias)
  
      self.out_proj = QuantLinear(embed_dim, embed_dim, bias=bias)
  
      if add_bias_kv:
          self.bias_k = Parameter(torch.Tensor(1, 1, embed_dim))
          self.bias_v = Parameter(torch.Tensor(1, 1, embed_dim))
      else:
          self.bias_k = self.bias_v = None
  
      self.add_zero_attn = add_zero_attn
  
      self.reset_parameters()
  
      self.onnx_trace = False
      self.tpu = False
      self.init_incremental_state()
  
imports: import math
  import uuid
  from typing import Dict, Optional, List
  import torch
  from torch import Tensor, nn
  from torch.nn import Parameter, LayerNorm, Dropout
  from lightseq.training.ops.pytorch import util
  from lightseq.training.ops.pytorch.layer_base import TransformerEmbeddingLayerBase, TransformerEncoderLayerBase, TransformerDecoderLayerBase
  from .quantization import QuantLinear, TensorQuantizer, act_quant_config, weight_quant_config, out_quant_config, emb_quant_config
label_desc: Use image node_id 1 to create a new MultiheadAttention object from inherited base classes: nn with example: obj = MultiheadAttention(embed_dim, num_heads, kdim, vdim, dropout, bias, add_bias_kv, add_zero_attn, _attention, encoder_decoder_attention, is_decoder, has_causal_mask)