class_docstr: Computes attention between hidden and encoder states.
  
  See arxiv.org/abs/1508.04025 for more info on each attention type.
docstr: Compute attention over attn_params given input and hidden states.
  
  :param xes:         input state. will be combined with applied
                      attention.
  :param hidden:      hidden state from model. will be used to select
                      states to attend to in from the attn_params.
  :param attn_params: tuple of encoder output states and a mask showing
                      which input indices are nonzero.
  
  :returns: output, attn_weights
            output is a new state of same size as input state `xes`.
            attn_weights are the weights given to each state in the
            encoder outputs.
docstr_tok: ["Compute","attention","over","attn_params","given","input","and","hidden","states",".",":","param","xes",":","input","state",".","will","be","combined","with","applied","attention",".",":","param","hidden",":","hidden","state","from","model",".","will","be","used","to","select","states","to","attend","to","in","from","the","attn_params",".",":","param","attn_params",":","tuple","of","encoder","output","states","and","a","mask","showing","which","input","indices","are","nonzero",".",":","returns",":","output",",","attn_weights","output","is","a","new","state","of","same","size","as","input","state","`","xes","`",".","attn_weights","are","the","weights","given","to","each","state","in","the","encoder","outputs","."]
code: def forward(self, xes, hidden, attn_params):
      """
      Compute attention over attn_params given input and hidden states.
  
      :param xes:         input state. will be combined with applied
                          attention.
      :param hidden:      hidden state from model. will be used to select
                          states to attend to in from the attn_params.
      :param attn_params: tuple of encoder output states and a mask showing
                          which input indices are nonzero.
  
      :returns: output, attn_weights
                output is a new state of same size as input state `xes`.
                attn_weights are the weights given to each state in the
                encoder outputs.
      """
      if self.attention == "none":
          # do nothing, no attention
          return xes, None
  
      if type(hidden) == tuple:
          # for lstms use the "hidden" state not the cell state
          hidden = hidden[0]
      last_hidden = hidden[
          -1
      ]  # select hidden state from last RNN layer
  
      enc_out, attn_mask = attn_params
      bsz, seqlen, hszXnumdir = enc_out.size()
      numlayersXnumdir = last_hidden.size(1)
  
      if self.attention == "local":
          # local attention weights aren't based on encoder states
          h_merged = torch.cat((xes.squeeze(1), last_hidden), 1)
          attn_weights = F.softmax(self.attn(h_merged), dim=1)
  
          # adjust state sizes to the fixed window size
          if seqlen > self.max_length:
              offset = seqlen - self.max_length
              enc_out = enc_out.narrow(1, offset, self.max_length)
              seqlen = self.max_length
          if attn_weights.size(1) > seqlen:
              attn_weights = attn_weights.narrow(1, 0, seqlen)
      else:
          hid = last_hidden.unsqueeze(1)
          if self.attention == "concat":
              # concat hidden state and encoder outputs
              hid = hid.expand(bsz, seqlen, numlayersXnumdir)
              h_merged = torch.cat((enc_out, hid), 2)
              # then do linear combination of them with activation
              active = F.tanh(self.attn(h_merged))
              attn_w_premask = self.attn_v(active).squeeze(2)
          elif self.attention == "dot":
              # dot product between hidden and encoder outputs
              if numlayersXnumdir != hszXnumdir:
                  # enc_out has two directions, so double hid
                  hid = torch.cat([hid, hid], 2)
              enc_t = enc_out.transpose(1, 2)
              attn_w_premask = torch.bmm(hid, enc_t).squeeze(1)
          elif self.attention == "general":
              # before doing dot product, transform hidden state with linear
              # same as dot if linear is identity
              hid = self.attn(hid)
              enc_t = enc_out.transpose(1, 2)
              attn_w_premask = torch.bmm(hid, enc_t).squeeze(1)
  
          # calculate activation scores, apply mask if needed
          if attn_mask is not None:
              # remove activation from NULL symbols
              attn_w_premask.masked_fill_(
                  (~attn_mask), neginf(attn_w_premask.dtype)
              )
          attn_weights = F.softmax(attn_w_premask, dim=1)
  
      # apply the attention weights to the encoder states
      attn_applied = torch.bmm(attn_weights.unsqueeze(1), enc_out)
      # concatenate the input and encoder states
      merged = torch.cat((xes.squeeze(1), attn_applied.squeeze(1)), 1)
      # combine them with a linear layer and tanh activation
      output = torch.tanh(self.attn_combine(merged).unsqueeze(1))
  
      return output, attn_weights
  
imports: import math
  import torch
  import torch.nn
  from torch.nn.parameter import Parameter
  from torch.nn.utils.rnn import pad_packed_sequence, pack_padded_sequence
  import torch.nn.functional
  from parlai.utils.torch import neginf
  from parlai.core.torch_generator_agent import TorchGeneratorModel
label_desc: Use image node_id 2 for calling the AttentionLayer obj's underlying member method code with example usage: obj.forward(xes, hidden, attn_params) and returns: output,  attn_weights, xes,  None