class_docstr: Implements MultiHeadAttention; this is the core workhorse of the Transformer.
  
  See Vaswani (2017) for an extensive description.
class_docstr_tok: ["Implements","MultiHeadAttention",";","this","is","the","core","workhorse","of","the","Transformer",".","See","Vaswani","(","2017",")","for","an","extensive","description","."]
code: def __init__(
      self,
      opt: Opt,
      n_heads: int = None,
      dim: int = None,
      dropout: float = 0,
  ):
      super(MultiHeadAttention, self).__init__()
  
      n_heads = default(n_heads, opt["n_heads"])
      dim = default(dim, opt["embedding_size"])
  
      self.n_heads = n_heads
      self.dim = dim
  
      self.attn_dropout = nn.Dropout(p=dropout)  # --attention-dropout
      self.q_lin = nn.Linear(dim, dim)
      self.k_lin = nn.Linear(dim, dim)
      self.v_lin = nn.Linear(dim, dim)
      # TODO: merge for the initialization step
      nn.init.xavier_normal_(self.q_lin.weight)
      nn.init.xavier_normal_(self.k_lin.weight)
      nn.init.xavier_normal_(self.v_lin.weight)
      # and set biases to 0
      self.out_lin = nn.Linear(dim, dim)
  
      nn.init.xavier_normal_(self.out_lin.weight)
  
file_path: ParlAI/parlai/agents/transformer/modules/attention.py
imports: import math
  from typing import Dict, Tuple, Optional, Union
  import torch
  import torch.nn
  import torch.nn.functional
  from parlai.core.opt import Opt
  from parlai.core.params import default
  from parlai.utils.torch import neginf
label_desc: Use image node_id 1 to create a new MultiHeadAttention object from inherited base classes: nn with example: obj = MultiHeadAttention(opt, n_heads, dim, dropout)