class_docstr: Single-head Attention layer for label-specific representations
class_docstr_tok: ["Single-head","Attention","layer","for","label-specific","representations"]
code: def __init__(
      self,
      d_model,
      d_k,
      d_v,
      d_l,
      d_proj,
      combine_as_self,
      use_resdrop=True,
      q_as_matrix=False,
      residual_dropout=0.1,
      attention_dropout=0.1,
      d_positional=None,
  ):
      super(LabelAttention, self).__init__()
      self.d_k = d_k
      self.d_v = d_v
      self.d_l = d_l  # Number of Labels
      self.d_model = d_model  # Model Dimensionality
      self.d_proj = d_proj  # Projection dimension of each label output
      self.use_resdrop = use_resdrop  # Using Residual Dropout?
      self.q_as_matrix = q_as_matrix  # Using a Matrix of Q to be multiplied with input instead of learned q vectors
      self.combine_as_self = combine_as_self  # Using the Combination Method of Self-Attention
  
      if not d_positional:
          self.partitioned = False
      else:
          self.partitioned = True
  
      if self.partitioned:
          if d_model <= d_positional:
              raise ValueError(
                  "Unable to build LabelAttention.  d_model %d <= d_positional %d"
                  % (d_model, d_positional)
              )
          self.d_content = d_model - d_positional
          self.d_positional = d_positional
  
          if self.q_as_matrix:
              self.w_qs1 = nn.Parameter(
                  torch.FloatTensor(self.d_l, self.d_content, d_k // 2),
                  requires_grad=True,
              )
          else:
              self.w_qs1 = nn.Parameter(
                  torch.FloatTensor(self.d_l, d_k // 2),
                  requires_grad=True,
              )
          self.w_ks1 = nn.Parameter(
              torch.FloatTensor(self.d_l, self.d_content, d_k // 2),
              requires_grad=True,
          )
          self.w_vs1 = nn.Parameter(
              torch.FloatTensor(self.d_l, self.d_content, d_v // 2),
              requires_grad=True,
          )
  
          if self.q_as_matrix:
              self.w_qs2 = nn.Parameter(
                  torch.FloatTensor(
                      self.d_l, self.d_positional, d_k // 2
                  ),
                  requires_grad=True,
              )
          else:
              self.w_qs2 = nn.Parameter(
                  torch.FloatTensor(self.d_l, d_k // 2),
                  requires_grad=True,
              )
          self.w_ks2 = nn.Parameter(
              torch.FloatTensor(self.d_l, self.d_positional, d_k // 2),
              requires_grad=True,
          )
          self.w_vs2 = nn.Parameter(
              torch.FloatTensor(self.d_l, self.d_positional, d_v // 2),
              requires_grad=True,
          )
  
          init.xavier_normal_(self.w_qs1)
          init.xavier_normal_(self.w_ks1)
          init.xavier_normal_(self.w_vs1)
  
          init.xavier_normal_(self.w_qs2)
          init.xavier_normal_(self.w_ks2)
          init.xavier_normal_(self.w_vs2)
      else:
          if self.q_as_matrix:
              self.w_qs = nn.Parameter(
                  torch.FloatTensor(self.d_l, d_model, d_k),
                  requires_grad=True,
              )
          else:
              self.w_qs = nn.Parameter(
                  torch.FloatTensor(self.d_l, d_k), requires_grad=True
              )
          self.w_ks = nn.Parameter(
              torch.FloatTensor(self.d_l, d_model, d_k),
              requires_grad=True,
          )
          self.w_vs = nn.Parameter(
              torch.FloatTensor(self.d_l, d_model, d_v),
              requires_grad=True,
          )
  
          init.xavier_normal_(self.w_qs)
          init.xavier_normal_(self.w_ks)
          init.xavier_normal_(self.w_vs)
  
      self.attention = ScaledDotProductAttention(
          d_model, attention_dropout=attention_dropout
      )
      if self.combine_as_self:
          self.layer_norm = LayerNormalization(d_model)
      else:
          self.layer_norm = LayerNormalization(self.d_proj)
  
      if not self.partitioned:
          # The lack of a bias term here is consistent with the t2t code, though
          # in my experiments I have never observed this making a difference.
          if self.combine_as_self:
              self.proj = nn.Linear(self.d_l * d_v, d_model, bias=False)
          else:
              self.proj = nn.Linear(
                  d_v, d_model, bias=False
              )  # input dimension does not match, should be d_l * d_v
      else:
          if self.combine_as_self:
              self.proj1 = nn.Linear(
                  self.d_l * (d_v // 2), self.d_content, bias=False
              )
              self.proj2 = nn.Linear(
                  self.d_l * (d_v // 2), self.d_positional, bias=False
              )
          else:
              self.proj1 = nn.Linear(
                  d_v // 2, self.d_content, bias=False
              )
              self.proj2 = nn.Linear(
                  d_v // 2, self.d_positional, bias=False
              )
      if not self.combine_as_self:
          self.reduce_proj = nn.Linear(d_model, self.d_proj, bias=False)
  
      self.residual_dropout = FeatureDropout(residual_dropout)
  
imports: import numpy
  import functools
  import sys
  import torch
  from torch.autograd import Variable
  import torch.nn
  import torch.nn.init
label_desc: Use image node_id 1 to create a new LabelAttention object from inherited base classes: nn with example: obj = LabelAttention(d_model, d_k, d_v, d_l, d_proj, combine_as_use_resdrop, q_as_matrix, residual_dropout, attention_dropout, d_positional)