class_docstr: An integrated model combined with a generator and a discriminator.  Generator here
  produces a corrupted tokens playing as fake data to fool a discriminator whose
  objective is to distinguish whether each token in the input sentence it accepts
  is the same as the original. It is a classification task instead of prediction
  task as other pretrained models such as bert.
class_docstr_tok: ["An","integrated","model","combined","with","a","generator","and","a","discriminator",".","Generator","here","produces","a","corrupted","tokens","playing","as","fake","data","to","fool","a","discriminator","whose","objective","is","to","distinguish","whether","each","token","in","the","input","sentence","it","accepts","is","the","same","as","the","original",".","It","is","a","classification","task","instead","of","prediction","task","as","other","pretrained","models","such","as","bert","."]
docstr: Parameters
  ----------
  disc_cfg :
      Config for discriminator model including scaled size for generator
  uniform_generator :
      Wether to get a generator with uniform weights, the mlm_scores from
      which are totally random. In this case , a discriminator learns from
      a random 15% of the input tokens distinct from the subset.
  tied_generator :
      Whether to tie backbone model weights of generator and discriminator.
      The size of G and D are required to be same if set to True.
  tied_embeddings :
      Whether to tie the embeddings of generator and discriminator
  disallow_correct :
      Whether the correct smaples of generator are allowed,
      that is 15% of tokens are always fake.
  temperature :
      Temperature of gumbel distribution for sampling from generator
  weight_initializer
  bias_initializer
docstr_tok: ["Parameters","--","--","--","--","--","disc_cfg",":","Config","for","discriminator","model","including","scaled","size","for","generator","uniform_generator",":","Wether","to","get","a","generator","with","uniform","weights",",","the","mlm_scores","from","which","are","totally","random",".","In","this","case",",","a","discriminator","learns","from","a","random","15","%","of","the","input","tokens","distinct","from","the","subset",".","tied_generator",":","Whether","to","tie","backbone","model","weights","of","generator","and","discriminator",".","The","size","of","G","and","D","are","required","to","be","same","if","set","to","True",".","tied_embeddings",":","Whether","to","tie","the","embeddings","of","generator","and","discriminator","disallow_correct",":","Whether","the","correct","smaples","of","generator","are","allowed",",","that","is","15","%","of","tokens","are","always","fake",".","temperature",":","Temperature","of","gumbel","distribution","for","sampling","from","generator","weight_initializer","bias_initializer"]
code: def __init__(
      self,
      disc_cfg,
      uniform_generator=False,
      tied_generator=False,
      tied_embeddings=True,
      disallow_correct=False,
      temperature=1.0,
      gumbel_eps=1e-9,
      dtype="float32",
      weight_initializer=None,
      bias_initializer=None,
  ):
      """
  
      Parameters
      ----------
      disc_cfg :
          Config for discriminator model including scaled size for generator
      uniform_generator :
          Wether to get a generator with uniform weights, the mlm_scores from
          which are totally random. In this case , a discriminator learns from
          a random 15% of the input tokens distinct from the subset.
      tied_generator :
          Whether to tie backbone model weights of generator and discriminator.
          The size of G and D are required to be same if set to True.
      tied_embeddings :
          Whether to tie the embeddings of generator and discriminator
      disallow_correct :
          Whether the correct smaples of generator are allowed,
          that is 15% of tokens are always fake.
      temperature :
          Temperature of gumbel distribution for sampling from generator
      weight_initializer
      bias_initializer
      """
      super().__init__()
      self._uniform_generator = uniform_generator
      self._tied_generator = tied_generator
      self._tied_embeddings = tied_embeddings
      self._disallow_correct = disallow_correct
      self._temperature = temperature
      self._gumbel_eps = gumbel_eps
      self._dtype = dtype
  
      self.disc_cfg = disc_cfg
      self.vocab_size = disc_cfg.MODEL.vocab_size
      self.gen_cfg = get_generator_cfg(disc_cfg)
      self.discriminator = ElectraDiscriminator(
          disc_cfg,
          weight_initializer=weight_initializer,
          bias_initializer=bias_initializer,
      )
      self.disc_backbone = self.discriminator.backbone_model
  
      if not uniform_generator and not tied_generator:
          self.generator = ElectraGenerator(
              self.gen_cfg,
              weight_initializer=weight_initializer,
              bias_initializer=bias_initializer,
          )
          if tied_embeddings:
              self.generator.tie_embeddings(
                  self.disc_backbone.word_embed.collect_params(),
                  self.disc_backbone.token_type_embed.collect_params(),
                  self.disc_backbone.token_pos_embed.collect_params(),
                  self.disc_backbone.embed_layer_norm.collect_params(),
              )
  
      elif tied_generator:
          # Reuse the weight of the discriminator backbone model
          self.generator = ElectraGenerator(
              self.gen_cfg,
              weight_initializer=weight_initializer,
              bias_initializer=bias_initializer,
          )
          # TODO(sxjscience, zheyu) Verify
          self.generator.backbone_model = self.disc_backbone
      elif uniform_generator:
          # get the mlm_scores randomly over vocab
          self.generator = None
  
imports: import os
  from typing import Tuple, Optional, List
  import mxnet
  from mxnet import use_np, np, npx
  from mxnet.gluon import HybridBlock, nn
  from .base import BACKBONE_REGISTRY
  from .transformer import TransformerEncoderLayer
  from ..op import gumbel_softmax, select_vectors_by_position, add_vectors_by_position, update_vectors_by_position
  from ..base import get_model_zoo_home_dir, get_repo_model_zoo_url, get_model_zoo_checksum_dir
  from ..layers import PositionalEmbedding, get_activation
  from ..initializer import TruncNorm
  from ..utils.config import CfgNode
  from ..utils.misc import load_checksum_stats, download
  from ..utils.registry import Registry
  from ..attention_cell import gen_self_attn_mask
  from ..data.tokenizers import HuggingFaceWordPieceTokenizer
label_desc: Use image node_id 1 to create a new ElectraForPretrain object from inherited base classes: HybridBlock with example: obj = ElectraForPretrain(disc_cfg, uniform_generator, tied_generator, tied_embeddings, disallow_correct, temperature, gumbel_eps, dtype, weight_initializer, bias_initializer)