class_docstr: An integrated model combined with a generator and a discriminator.  Generator here
  produces a corrupted tokens playing as fake data to fool a discriminator whose
  objective is to distinguish whether each token in the input sentence it accepts
  is the same as the original. It is a classification task instead of prediction
  task as other pretrained models such as bert.
class_docstr_tok: ["An","integrated","model","combined","with","a","generator","and","a","discriminator",".","Generator","here","produces","a","corrupted","tokens","playing","as","fake","data","to","fool","a","discriminator","whose","objective","is","to","distinguish","whether","each","token","in","the","input","sentence","it","accepts","is","the","same","as","the","original",".","It","is","a","classification","task","instead","of","prediction","task","as","other","pretrained","models","such","as","bert","."]
docstr: Getting the mlm scores of each masked positions from a generator,
  then produces the corrupted tokens sampling from a gumbel distribution.
  We also get the ground-truth and scores of the replaced token detection
  which is output by a discriminator. The ground-truth is an array with same
  shape as the input using 1 stand for original token and 0 for replacement.
  
  Notice: There is a problem when the masked positions have duplicate indexs.
  Try to avoid that in the data preprocessing process. In addition, loss calculation
  should be done in the training scripts as well.
  
  Parameters
  ----------
  inputs
      The masked input
  
      - layout = 'NT'
          Shape (batch_size, seq_length)
      - layout = 'TN'
          Shape (seq_length, batch_size)
  
  token_types
      The token types. If the inputs contain two sequences, we will set different token types
      for the first sentence and the second sentence.
  
      - layout = 'NT'
          Shape (batch_size, seq_length)
      - layout = 'TN'
          Shape (seq_length, batch_size)
  
  valid_length
      The valid length of each sequence.
      Shape (batch_size,)
  original_tokens
      The original tokens that appear in the unmasked input sequence.
      Shape (batch_size, num_masked_positions).
  masked_positions :
      The masked position of the sequence.
      Shape (batch_size, num_masked_positions).
  
  Returns
  -------
  mlm_scores
      The masked language model score.
      Shape (batch_size, num_masked_positions, vocab_size)
  rtd_scores
      The replaced-token-detection score. Predicts whether the tokens are replaced or not.
  
      - layout = 'NT'
          Shape (batch_size, seq_length)
      - layout = 'TN'
          Shape (seq_length, batch_size)
  
  replaced_inputs
      Shape (batch_size, num_masked_positions)
  labels
      - layout = 'NT'
          Shape (batch_size, seq_length)
      - layout = 'TN'
          Shape (seq_length, batch_size)
docstr_tok: ["Getting","the","mlm","scores","of","each","masked","positions","from","a","generator",",","then","produces","the","corrupted","tokens","sampling","from","a","gumbel","distribution",".","We","also","get","the","ground-truth","and","scores","of","the","replaced","token","detection","which","is","output","by","a","discriminator",".","The","ground-truth","is","an","array","with","same","shape","as","the","input","using","1","stand","for","original","token","and","0","for","replacement",".","Notice",":","There","is","a","problem","when","the","masked","positions","have","duplicate","indexs",".","Try","to","avoid","that","in","the","data","preprocessing","process",".","In","addition",",","loss","calculation","should","be","done","in","the","training","scripts","as","well",".","Parameters","--","--","--","--","--","inputs","The","masked","input","-","layout","=","'NT'","Shape","(","batch_size",",","seq_length",")","-","layout","=","'TN'","Shape","(","seq_length",",","batch_size",")","token_types","The","token","types",".","If","the","inputs","contain","two","sequences",",","we","will","set","different","token","types","for","the","first","sentence","and","the","second","sentence",".","-","layout","=","'NT'","Shape","(","batch_size",",","seq_length",")","-","layout","=","'TN'","Shape","(","seq_length",",","batch_size",")","valid_length","The","valid","length","of","each","sequence",".","Shape","(","batch_size",",",")","original_tokens","The","original","tokens","that","appear","in","the","unmasked","input","sequence",".","Shape","(","batch_size",",","num_masked_positions",")",".","masked_positions",":","The","masked","position","of","the","sequence",".","Shape","(","batch_size",",","num_masked_positions",")",".","Returns","--","--","--","-","mlm_scores","The","masked","language","model","score",".","Shape","(","batch_size",",","num_masked_positions",",","vocab_size",")","rtd_scores","The","replaced-token-detection","score",".","Predicts","whether","the","tokens","are","replaced","or","not",".","-","layout","=","'NT'","Shape","(","batch_size",",","seq_length",")","-","layout","=","'TN'","Shape","(","seq_length",",","batch_size",")","replaced_inputs","Shape","(","batch_size",",","num_masked_positions",")","labels","-","layout","=","'NT'","Shape","(","batch_size",",","seq_length",")","-","layout","=","'TN'","Shape","(","seq_length",",","batch_size",")"]
code: def forward(
      self,
      inputs,
      token_types,
      valid_length,
      original_tokens,
      masked_positions,
  ):
      """Getting the mlm scores of each masked positions from a generator,
      then produces the corrupted tokens sampling from a gumbel distribution.
      We also get the ground-truth and scores of the replaced token detection
      which is output by a discriminator. The ground-truth is an array with same
      shape as the input using 1 stand for original token and 0 for replacement.
  
      Notice: There is a problem when the masked positions have duplicate indexs.
      Try to avoid that in the data preprocessing process. In addition, loss calculation
      should be done in the training scripts as well.
  
      Parameters
      ----------
      inputs
          The masked input
  
          - layout = 'NT'
              Shape (batch_size, seq_length)
          - layout = 'TN'
              Shape (seq_length, batch_size)
  
      token_types
          The token types. If the inputs contain two sequences, we will set different token types
          for the first sentence and the second sentence.
  
          - layout = 'NT'
              Shape (batch_size, seq_length)
          - layout = 'TN'
              Shape (seq_length, batch_size)
  
      valid_length
          The valid length of each sequence.
          Shape (batch_size,)
      original_tokens
          The original tokens that appear in the unmasked input sequence.
          Shape (batch_size, num_masked_positions).
      masked_positions :
          The masked position of the sequence.
          Shape (batch_size, num_masked_positions).
  
      Returns
      -------
      mlm_scores
          The masked language model score.
          Shape (batch_size, num_masked_positions, vocab_size)
      rtd_scores
          The replaced-token-detection score. Predicts whether the tokens are replaced or not.
  
          - layout = 'NT'
              Shape (batch_size, seq_length)
          - layout = 'TN'
              Shape (seq_length, batch_size)
  
      replaced_inputs
          Shape (batch_size, num_masked_positions)
      labels
          - layout = 'NT'
              Shape (batch_size, seq_length)
          - layout = 'TN'
              Shape (seq_length, batch_size)
  
      """
      if self._uniform_generator:
          # generate the corrupt tokens randomly with a mlm_scores vector whose value is all 0
          zero_logits = np.zeros(
              (1, 1, self.vocab_size), dtype=self._dtype
          )
          mlm_scores = np.expand_dims(
              np.zeros_like(masked_positions, dtype=self._dtype),
              axis=-1,
          )
          mlm_scores = mlm_scores + zero_logits
      else:
          _, _, mlm_scores = self.generator(
              inputs, token_types, valid_length, masked_positions
          )
  
      corrupted_tokens, fake_data, labels = self.get_corrupted_tokens(
          inputs, original_tokens, masked_positions, mlm_scores
      )
      # The discriminator takes the same input as the generator and the token_ids are
      # replaced with fake data
      _, _, rtd_scores = self.discriminator(
          fake_data, token_types, valid_length
      )
      return mlm_scores, rtd_scores, corrupted_tokens, labels
  
imports: import os
  from typing import Tuple, Optional, List
  import mxnet
  from mxnet import use_np, np, npx
  from mxnet.gluon import HybridBlock, nn
  from .base import BACKBONE_REGISTRY
  from .transformer import TransformerEncoderLayer
  from ..op import gumbel_softmax, select_vectors_by_position, add_vectors_by_position, update_vectors_by_position
  from ..base import get_model_zoo_home_dir, get_repo_model_zoo_url, get_model_zoo_checksum_dir
  from ..layers import PositionalEmbedding, get_activation
  from ..initializer import TruncNorm
  from ..utils.config import CfgNode
  from ..utils.misc import load_checksum_stats, download
  from ..utils.registry import Registry
  from ..attention_cell import gen_self_attn_mask
  from ..data.tokenizers import HuggingFaceWordPieceTokenizer
label_desc: Use image node_id 2 for calling the ElectraForPretrain obj's underlying member method code with example usage: obj.forward(inputs, token_types, valid_length, original_tokens, masked_positions) and returns: mlm_scores,  rtd_scores,  corrupted_tokens,  labels