class_docstr: Implements a stacked self-attention encoder similar to the Transformer
  architecture in [Attention is all you Need]
  (https://www.semanticscholar.org/paper/Attention-Is-All-You-Need-Vaswani-Shazeer/0737da0767d77606169cbf4187b83e1ab62f6077).
  
  This class adapts the Transformer from torch.nn for use in AllenNLP. Optionally, it adds positional encodings.
  
  Registered as a `Seq2SeqEncoder` with name "pytorch_transformer".
  
  # Parameters
  
  input_dim : `int`, required.
      The input dimension of the encoder.
  num_layers : `int`, required.
      The number of stacked self attention -> feedforward -> layer normalisation blocks.
  feedforward_hidden_dim : `int`, required.
      The middle dimension of the FeedForward network. The input and output
      dimensions are fixed to ensure sizes match up for the self attention layers.
  num_attention_heads : `int`, required.
      The number of attention heads to use per layer.
  positional_encoding : `str`, optional, (default = `None`)
      Specifies the type of positional encodings to use. Your options are
       * `None` to have no positional encodings.
       * `"sinusoidal"` to have sinusoidal encodings, as described in https://api.semanticscholar.org/CorpusID:13756489.
       * `"embedding"` to treat positional encodings as learnable parameters
      Without positional encoding, the self attention layers have no idea of absolute or relative
      position (as they are just computing pairwise similarity between vectors of elements),
      which can be important features for many tasks.
  positional_embedding_size : `int`, optional, (default = `512`)
      The number of positional embeddings.
  dropout_prob : `float`, optional, (default = `0.1`)
      The dropout probability for the feedforward network.
  activation : `str`, (default = `"relu"`)
      The activation function of intermediate layers. Must be either `"relu"` or `"gelu"`.
class_docstr_tok: ["Implements","a","stacked","self-attention","encoder","similar","to","the","Transformer","architecture","in","[","Attention","is","all","you","Need","]","(","https",":","\/\/www.semanticscholar.org\/paper\/Attention-Is-All-You-Need-Vaswani-Shazeer\/0737da0767d77606169cbf4187b83e1ab62f6077",")",".","This","class","adapts","the","Transformer","from","torch.nn","for","use","in","AllenNLP",".","Optionally",",","it","adds","positional","encodings",".","Registered","as","a","`","Seq2SeqEncoder","`","with","name","``","pytorch_transformer","''",".","#","Parameters","input_dim",":","`","int","`",",","required",".","The","input","dimension","of","the","encoder",".","num_layers",":","`","int","`",",","required",".","The","number","of","stacked","self","attention","-",">","feedforward","-",">","layer","normalisation","blocks",".","feedforward_hidden_dim",":","`","int","`",",","required",".","The","middle","dimension","of","the","FeedForward","network",".","The","input","and","output","dimensions","are","fixed","to","ensure","sizes","match","up","for","the","self","attention","layers",".","num_attention_heads",":","`","int","`",",","required",".","The","number","of","attention","heads","to","use","per","layer",".","positional_encoding",":","`","str","`",",","optional",",","(","default","=","`","None","`",")","Specifies","the","type","of","positional","encodings","to","use",".","Your","options","are","*","`","None","`","to","have","no","positional","encodings",".","*","`","``","sinusoidal","''","`","to","have","sinusoidal","encodings",",","as","described","in","https",":","\/\/api.semanticscholar.org\/CorpusID:13756489",".","*","`","``","embedding","''","`","to","treat","positional","encodings","as","learnable","parameters","Without","positional","encoding",",","the","self","attention","layers","have","no","idea","of","absolute","or","relative","position","(","as","they","are","just","computing","pairwise","similarity","between","vectors","of","elements",")",",","which","can","be","important","features","for","many","tasks",".","positional_embedding_size",":","`","int","`",",","optional",",","(","default","=","`","512","`",")","The","number","of","positional","embeddings",".","dropout_prob",":","`","float","`",",","optional",",","(","default","=","`","0.1","`",")","The","dropout","probability","for","the","feedforward","network",".","activation",":","`","str","`",",","(","default","=","`","``","relu","''","`",")","The","activation","function","of","intermediate","layers",".","Must","be","either","`","``","relu","''","`","or","`","``","gelu","''","`","."]
code: def forward(self, inputs: torch.Tensor, mask: torch.BoolTensor):
      output = inputs
      if self._sinusoidal_positional_encoding:
          output = add_positional_features(output)
      if self._positional_embedding is not None:
          position_ids = torch.arange(
              inputs.size(1), dtype=torch.long, device=output.device
          )
          position_ids = position_ids.unsqueeze(0).expand(
              inputs.shape[:-1]
          )
          output = output + self._positional_embedding(position_ids)
  
      # For some reason the torch transformer expects the shape (sequence, batch, features), not the more
      # familiar (batch, sequence, features), so we have to fix it.
      output = output.permute(1, 0, 2)
      # For some other reason, the torch transformer takes the mask backwards.
      mask = ~mask
      output = self._transformer(output, src_key_padding_mask=mask)
      output = output.permute(1, 0, 2)
  
      return output
  
file_path: allennlp/allennlp/modules/seq2seq_encoders/pytorch_transformer_wrapper.py
imports: from typing import Optional
  import torch
  from torch import nn
  from allennlp.modules.seq2seq_encoders.seq2seq_encoder import Seq2SeqEncoder
  from allennlp.nn.util import add_positional_features
label_desc: Use image node_id 5 for calling the PytorchTransformer obj's underlying member method code with example usage: obj.forward(inputs, mask) and returns: output