docstr: Generate the representation given the inputs.
  
  This is used in training or fine-tuning a bert model.
  
  Parameters
  ----------
  inputs
      - layout = 'NT'
          Shape (batch_size, seq_length)
      - layout = 'TN'
          Shape (seq_length, batch_size)
  token_types
      - layout = 'NT'
          Shape (batch_size, seq_length)
      - layout = 'TN'
          Shape (seq_length, batch_size)
  
      If the inputs contain two sequences, we will set different token types for the first
       sentence and the second sentence.
  valid_length
      The valid length of each sequence
      Shape (batch_size,)
  masked_positions
      The masked position of the sequence with respect to flattened batch
      Shape (N, ) for N masked positions across whole batch.
  
  Returns
  -------
  contextual_embedding
      - layout = 'NT'
          Shape (batch_size, seq_length, units).
      - layout = 'TN'
          Shape (seq_length, batch_size, units).
  pooled_out
      Shape (batch_size, units)
  mlm_scores :
      Shape (N, vocab_size)
docstr_tok: ["Generate","the","representation","given","the","inputs",".","This","is","used","in","training","or","fine-tuning","a","bert","model",".","Parameters","--","--","--","--","--","inputs","-","layout","=","'NT'","Shape","(","batch_size",",","seq_length",")","-","layout","=","'TN'","Shape","(","seq_length",",","batch_size",")","token_types","-","layout","=","'NT'","Shape","(","batch_size",",","seq_length",")","-","layout","=","'TN'","Shape","(","seq_length",",","batch_size",")","If","the","inputs","contain","two","sequences",",","we","will","set","different","token","types","for","the","first","sentence","and","the","second","sentence",".","valid_length","The","valid","length","of","each","sequence","Shape","(","batch_size",",",")","masked_positions","The","masked","position","of","the","sequence","with","respect","to","flattened","batch","Shape","(","N",",",")","for","N","masked","positions","across","whole","batch",".","Returns","--","--","--","-","contextual_embedding","-","layout","=","'NT'","Shape","(","batch_size",",","seq_length",",","units",")",".","-","layout","=","'TN'","Shape","(","seq_length",",","batch_size",",","units",")",".","pooled_out","Shape","(","batch_size",",","units",")","mlm_scores",":","Shape","(","N",",","vocab_size",")"]
code: def forward(
      self, inputs, token_types, valid_length, masked_positions
  ):
      """Generate the representation given the inputs.
  
      This is used in training or fine-tuning a bert model.
  
      Parameters
      ----------
      inputs
          - layout = 'NT'
              Shape (batch_size, seq_length)
          - layout = 'TN'
              Shape (seq_length, batch_size)
      token_types
          - layout = 'NT'
              Shape (batch_size, seq_length)
          - layout = 'TN'
              Shape (seq_length, batch_size)
  
          If the inputs contain two sequences, we will set different token types for the first
           sentence and the second sentence.
      valid_length
          The valid length of each sequence
          Shape (batch_size,)
      masked_positions
          The masked position of the sequence with respect to flattened batch
          Shape (N, ) for N masked positions across whole batch.
  
      Returns
      -------
      contextual_embedding
          - layout = 'NT'
              Shape (batch_size, seq_length, units).
          - layout = 'TN'
              Shape (seq_length, batch_size, units).
      pooled_out
          Shape (batch_size, units)
      mlm_scores :
          Shape (N, vocab_size)
      """
      assert (
          len(inputs) % 2 == 0
      ), "Model expects QuickThought paired inputs"
      contextual_embeddings, pooled_out = self.backbone_model(
          inputs, token_types, valid_length
      )
      if self.layout == "NT":
          mlm_features = contextual_embeddings.flatten(0, 1)[
              masked_positions
          ]
      else:
          mlm_features = th.transpose(
              contextual_embeddings, 0, 1
          ).flatten(0, 1)[masked_positions]
      mlm_scores = self.mlm_decoder(mlm_features)
      qt_embeddings = self.quickthought(pooled_out)
      qt_similarity = self._cosine_similarity(
          qt_embeddings[: len(inputs) // 2],
          qt_embeddings[len(inputs) // 2 :],
      )
      return (
          contextual_embeddings,
          pooled_out,
          mlm_scores,
          qt_similarity,
      )
  
imports: import logging
  import torch
  from ..attention_cell import gen_self_attn_mask
  from .transformer import TransformerEncoderLayer
  from ..layers import get_activation, sequence_mask
  from ...utils.registry import Registry
  from ...models.bert import bert_cfg_reg
label_desc: Use image node_id 3 for calling the QTBertForPretrain obj's underlying member method code with example usage: obj.forward(inputs, token_types, valid_length, masked_positions) and returns: contextual_embeddings,  pooled_out,  mlm_scores,  qt_similarity