class_docstr: Construct the embeddings from word, position and token_type embeddings.
  Details in the paper:
  [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding, Devlin et al, 2019]
  (https://api.semanticscholar.org/CorpusID:52967399)
  
  # Parameters
  
  vocab_size : `int`
      The size of the input vocab.
  embedding_size : `int`
      The `embedding_dim` of all the embedding layers.
  pad_token_id : `int` (default = `0`)
      The token id of the `<pad>` token.
  max_position_embeddings : `int` (default = `512`)
      The maximum number of positions.
  type_vocab_size : `int` (default = `2`)
      The size of the input token_type vocab.
  dropout : `int` (default = `0.1`)
      The probability of an element to be zeroed.
  output_size : `int`, optional (default = `None`)
      Optionally apply a linear transform after the dropout, projecting to `output_size`.
class_docstr_tok: ["Construct","the","embeddings","from","word",",","position","and","token_type","embeddings",".","Details","in","the","paper",":","[","BERT",":","Pre-training","of","Deep","Bidirectional","Transformers","for","Language","Understanding",",","Devlin","et","al",",","2019","]","(","https",":","\/\/api.semanticscholar.org\/CorpusID:52967399",")","#","Parameters","vocab_size",":","`","int","`","The","size","of","the","input","vocab",".","embedding_size",":","`","int","`","The","`","embedding_dim","`","of","all","the","embedding","layers",".","pad_token_id",":","`","int","`","(","default","=","`","0","`",")","The","token","id","of","the","`","<","pad",">","`","token",".","max_position_embeddings",":","`","int","`","(","default","=","`","512","`",")","The","maximum","number","of","positions",".","type_vocab_size",":","`","int","`","(","default","=","`","2","`",")","The","size","of","the","input","token_type","vocab",".","dropout",":","`","int","`","(","default","=","`","0.1","`",")","The","probability","of","an","element","to","be","zeroed",".","output_size",":","`","int","`",",","optional","(","default","=","`","None","`",")","Optionally","apply","a","linear","transform","after","the","dropout",",","projecting","to","`","output_size","`","."]
docstr: # Parameters
  input_ids : `torch.Tensor`
      Shape `batch_size x seq_len`
  attention_mask : `torch.Tensor`
      Shape `batch_size x seq_len`. This parameter is ignored, but it is here for compatibility.
  token_type_ids : `torch.Tensor`, optional
      Shape `batch_size x seq_len`
  position_ids : `torch.Tensor`, optional
      Shape `batch_size x seq_len`
docstr_tok: ["#","Parameters","input_ids",":","`","torch.Tensor","`","Shape","`","batch_size","x","seq_len","`","attention_mask",":","`","torch.Tensor","`","Shape","`","batch_size","x","seq_len","`",".","This","parameter","is","ignored",",","but","it","is","here","for","compatibility",".","token_type_ids",":","`","torch.Tensor","`",",","optional","Shape","`","batch_size","x","seq_len","`","position_ids",":","`","torch.Tensor","`",",","optional","Shape","`","batch_size","x","seq_len","`"]
code: def forward(  # type: ignore
      self,
      input_ids: torch.Tensor,
      token_type_ids: Optional[torch.Tensor] = None,
      position_ids: Optional[torch.Tensor] = None,
      attention_mask: Optional[torch.Tensor] = None,
  ) -> torch.Tensor:
      """
      # Parameters
      input_ids : `torch.Tensor`
          Shape `batch_size x seq_len`
      attention_mask : `torch.Tensor`
          Shape `batch_size x seq_len`. This parameter is ignored, but it is here for compatibility.
      token_type_ids : `torch.Tensor`, optional
          Shape `batch_size x seq_len`
      position_ids : `torch.Tensor`, optional
          Shape `batch_size x seq_len`
      """
  
      input_shape = input_ids.size()
      device = input_ids.device
      seq_length = input_shape[1]
  
      embedding_inputs = [input_ids]
  
      if attention_mask is None:
          attention_mask = (
              input_ids
              != self.embeddings["word_embeddings"].padding_idx
          )
  
      if "position_embeddings" in self.embeddings:
          if position_ids is None:
              padding_idx = self.embeddings[
                  "position_embeddings"
              ].padding_idx
              if padding_idx is None:
                  position_ids = torch.arange(
                      seq_length, dtype=torch.long, device=device
                  )
                  position_ids = position_ids.unsqueeze(0).expand(
                      input_shape
                  )
              else:
                  # In the RoBERTa case, position indices start with padding_idx + 1. Also, RoBERTa likes
                  # to respect padding in its position ids.
                  position_ids = (
                      torch.arange(
                          seq_length, dtype=torch.long, device=device
                      )
                      + 1
                  )
                  position_ids = (
                      position_ids.unsqueeze(0).expand(input_shape)
                      * attention_mask
                  )
                  position_ids += padding_idx
          embedding_inputs.append(position_ids)
  
      if "token_type_embeddings" in self.embeddings:
          if token_type_ids is None:
              token_type_ids = torch.zeros(
                  input_shape, dtype=torch.long, device=device
              )
          embedding_inputs.append(token_type_ids)
  
      embeddings = super().forward(*embedding_inputs)
  
      if hasattr(self, "linear_transform"):
          embeddings = self.linear_transform(embeddings)
  
      return embeddings
  
file_path: allennlp/allennlp/modules/transformer/transformer_embeddings.py
imports: from typing import Optional, TYPE_CHECKING
  import torch
  from allennlp.common import FromParams
  from allennlp.modules.transformer.layer_norm import LayerNorm
  from allennlp.modules.transformer.transformer_module import TransformerModule
label_desc: Use image node_id 2 for calling the TransformerEmbeddings obj's underlying member method code with example usage: obj.forward(input_ids, token_type_ids, position_ids, attention_mask) and returns: embeddings