class_docstr: Multi-head attention module
class_docstr_tok: ["Multi-head","attention","module"]
code: def __init__(
      self,
      n_head,
      d_model,
      d_k,
      d_v,
      residual_dropout=0.1,
      attention_dropout=0.1,
      d_positional=None,
  ):
      super(MultiHeadAttention, self).__init__()
  
      self.n_head = n_head
      self.d_k = d_k
      self.d_v = d_v
  
      if not d_positional:
          self.partitioned = False
      else:
          self.partitioned = True
  
      if self.partitioned:
          self.d_content = d_model - d_positional
          self.d_positional = d_positional
  
          self.w_qs1 = nn.Parameter(
              torch.FloatTensor(n_head, self.d_content, d_k // 2)
          )
          self.w_ks1 = nn.Parameter(
              torch.FloatTensor(n_head, self.d_content, d_k // 2)
          )
          self.w_vs1 = nn.Parameter(
              torch.FloatTensor(n_head, self.d_content, d_v // 2)
          )
  
          self.w_qs2 = nn.Parameter(
              torch.FloatTensor(n_head, self.d_positional, d_k // 2)
          )
          self.w_ks2 = nn.Parameter(
              torch.FloatTensor(n_head, self.d_positional, d_k // 2)
          )
          self.w_vs2 = nn.Parameter(
              torch.FloatTensor(n_head, self.d_positional, d_v // 2)
          )
  
          init.xavier_normal_(self.w_qs1)
          init.xavier_normal_(self.w_ks1)
          init.xavier_normal_(self.w_vs1)
  
          init.xavier_normal_(self.w_qs2)
          init.xavier_normal_(self.w_ks2)
          init.xavier_normal_(self.w_vs2)
      else:
          self.w_qs = nn.Parameter(
              torch.FloatTensor(n_head, d_model, d_k)
          )
          self.w_ks = nn.Parameter(
              torch.FloatTensor(n_head, d_model, d_k)
          )
          self.w_vs = nn.Parameter(
              torch.FloatTensor(n_head, d_model, d_v)
          )
  
          init.xavier_normal_(self.w_qs)
          init.xavier_normal_(self.w_ks)
          init.xavier_normal_(self.w_vs)
  
      self.attention = ScaledDotProductAttention(
          d_model, attention_dropout=attention_dropout
      )
      self.layer_norm = LayerNormalization(d_model)
  
      if not self.partitioned:
          # The lack of a bias term here is consistent with the t2t code, though
          # in my experiments I have never observed this making a difference.
          self.proj = nn.Linear(n_head * d_v, d_model, bias=False)
      else:
          self.proj1 = nn.Linear(
              n_head * (d_v // 2), self.d_content, bias=False
          )
          self.proj2 = nn.Linear(
              n_head * (d_v // 2), self.d_positional, bias=False
          )
  
      self.residual_dropout = FeatureDropout(residual_dropout)
  
imports: import numpy
  import functools
  import sys
  import torch
  from torch.autograd import Variable
  import torch.nn
  import torch.nn.init
label_desc: Use image node_id 1 to create a new MultiHeadAttention object from inherited base classes: nn with example: obj = MultiHeadAttention(n_head, d_model, d_k, d_v, residual_dropout, attention_dropout, d_positional)