class_docstr: Torchscriptable version of `BertModel` from Huggingface transformers v2.3.0
  https://github.com/huggingface/transformers/blob/v2.3.0/transformers/modeling_bert.py # noqa
  
  Modifies the `forward` function
  
  Changes to `forward` function ::
      Typings for input, modifications to device, change output type to
      Tuple[Tensor, Tensor, List[Tensor]]
class_docstr_tok: ["Torchscriptable","version","of","`","BertModel","`","from","Huggingface","transformers","v2.3.0","https",":","\/\/github.com\/huggingface\/transformers\/blob\/v2.3.0\/transformers\/modeling_bert.py","#","noqa","Modifies","the","`","forward","`","function","Changes","to","`","forward","`","function",":",":","Typings","for","input",",","modifications","to","device",",","change","output","type","to","Tuple","[","Tensor",",","Tensor",",","List","[","Tensor","]","]"]
docstr: Forward pass on the Model.
  The model can behave as an encoder (with only self-attention) as well
  as a decoder, in which case a layer of cross-attention is added between
  the self-attention layers, following the architecture described in
  `Attention is all you need`_ by Ashish Vaswani, Noam Shazeer, Niki Parmar,
  Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and
  Illia Polosukhin.
  To behave as an decoder the model needs to be initialized with the
  `is_decoder` argument of the configuration set to `True`; an
  `encoder_hidden_states` is expected as an input to the forward pass.
  .. _`Attention is all you need`:
      https://arxiv.org/abs/1706.03762
docstr_tok: ["Forward","pass","on","the","Model",".","The","model","can","behave","as","an","encoder","(","with","only","self-attention",")","as","well","as","a","decoder",",","in","which","case","a","layer","of","cross-attention","is","added","between","the","self-attention","layers",",","following","the","architecture","described","in","`","Attention","is","all","you","need","`","_","by","Ashish","Vaswani",",","Noam","Shazeer",",","Niki","Parmar",",","Jakob","Uszkoreit",",","Llion","Jones",",","Aidan","N.","Gomez",",","Lukasz","Kaiser","and","Illia","Polosukhin",".","To","behave","as","an","decoder","the","model","needs","to","be","initialized","with","the","`","is_decoder","`","argument","of","the","configuration","set","to","`","True","`",";","an","`","encoder_hidden_states","`","is","expected","as","an","input","to","the","forward","pass",".","..","_","`","Attention","is","all","you","need","`",":","https",":","\/\/arxiv.org\/abs\/1706.03762"]
code: def forward(
      self,
      input_ids: Tensor,
      attention_mask: Optional[Tensor] = None,
      token_type_ids: Optional[Tensor] = None,
      position_ids: Optional[Tensor] = None,
      head_mask: Optional[Tensor] = None,
      inputs_embeds: Optional[Tensor] = None,
      encoder_hidden_states: Optional[Tensor] = None,
      encoder_attention_mask: Optional[Tensor] = None,
  ) -> Tuple[Tensor, Tensor, List[Tensor]]:
      """Forward pass on the Model.
      The model can behave as an encoder (with only self-attention) as well
      as a decoder, in which case a layer of cross-attention is added between
      the self-attention layers, following the architecture described in
      `Attention is all you need`_ by Ashish Vaswani, Noam Shazeer, Niki Parmar,
      Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and
      Illia Polosukhin.
      To behave as an decoder the model needs to be initialized with the
      `is_decoder` argument of the configuration set to `True`; an
      `encoder_hidden_states` is expected as an input to the forward pass.
      .. _`Attention is all you need`:
          https://arxiv.org/abs/1706.03762
      """
      if input_ids is not None and inputs_embeds is not None:
          raise ValueError(
              "You cannot specify both input_ids and inputs_embeds at the same time"
          )
      elif input_ids is not None:
          input_shape = input_ids.size()
      elif inputs_embeds is not None:
          input_shape = inputs_embeds.size()[:-1]
      else:
          raise ValueError(
              "You have to specify either input_ids or inputs_embeds"
          )
  
      device = (
          inputs_embeds.device
          if inputs_embeds is not None
          else input_ids.device
      )
  
      if attention_mask is None:
          attention_mask = torch.ones(input_shape, device=device)
      if token_type_ids is None:
          token_type_ids = torch.zeros(
              input_shape, dtype=torch.long, device=device
          )
  
      # We can provide a self-attention mask of dimensions
      # [batch_size, from_seq_length, to_seq_length]
      # ourselves in which case we just need to make it broadcastable to all heads.
      if attention_mask.dim() == 3:
          extended_attention_mask = attention_mask[:, None, :, :]
      elif attention_mask.dim() == 2:
          # Provided a padding mask of dimensions [batch_size, seq_length]
          # - if the model is a decoder, apply a causal mask in addition to
          # the padding mask
          # - if the model is an encoder, make the mask broadcastable to
          # [batch_size, num_heads, seq_length, seq_length]
          extended_attention_mask = attention_mask[:, None, None, :]
      else:
          raise ValueError(
              f"Wrong shape for input_ids (shape {input_shape}) or "
              + f"attention_mask (shape {attention_mask.shape})"
          )
  
      # Since attention_mask is 1.0 for positions we want to attend and 0.0 for
      # masked positions, this operation will create a tensor which is 0.0 for
      # positions we want to attend and -10000.0 for masked positions.
      # Since we are adding it to the raw scores before the softmax, this is
      # effectively the same as removing these entirely.
      # Python builtin next is currently not supported in Torchscript
      if not torch.jit.is_scripting():
          extended_attention_mask = extended_attention_mask.to(
              dtype=next(self.parameters()).dtype
          )  # fp16 compatibility
      extended_attention_mask = (
          1.0 - extended_attention_mask
      ) * -10000.0
  
      # If a 2D ou 3D attention mask is provided for the cross-attention
      # we need to make broadcastabe to
      # [batch_size, num_heads, seq_length, seq_length]
      encoder_extended_attention_mask = None
  
      embedding_output = self.embeddings(
          input_ids=input_ids,
          position_ids=position_ids,
          token_type_ids=token_type_ids,
          inputs_embeds=inputs_embeds,
      )
      encoder_outputs = self.encoder(
          embedding_output,
          attention_mask=extended_attention_mask,
          encoder_hidden_states=encoder_hidden_states,
          encoder_attention_mask=encoder_extended_attention_mask,
      )
      sequence_output = encoder_outputs[0]
      pooled_output = self.pooler(sequence_output)
  
      # add hidden_states and attentions if they are here
      outputs = (sequence_output, pooled_output, encoder_outputs[1:])
      return outputs
  
imports: import math
  from typing import List, Optional, Tuple
  import torch
  from mmf.utils.patch import restore_saved_modules, safecopy_modules
  from torch import nn, Tensor
label_desc: Use image node_id 2 for calling the BertModelJit obj's underlying member method code with example usage: obj.forward(input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask) and returns: outputs