class_docstr: ELECTRA pre-processes and applies masks pretrain data
  
  Parameters
  ----------
  tokenizer : gluonnlp.data.tokenizers
      Used to tokenize the pretrained text sequence.
  max_seq_length : int
      Maximum sequence length of preprocessed text for pretraining.
  mask_prob : float
      The probability of applying masks on the token in the sequence.
  proposal_distribution : float
      A predefined probability distribution for each position in the sequence.
  replace_prob : float
      The probability of replace current token with a generator-predicted token.
class_docstr_tok: ["ELECTRA","pre-processes","and","applies","masks","pretrain","data","Parameters","--","--","--","--","--","tokenizer",":","gluonnlp.data.tokenizers","Used","to","tokenize","the","pretrained","text","sequence",".","max_seq_length",":","int","Maximum","sequence","length","of","preprocessed","text","for","pretraining",".","mask_prob",":","float","The","probability","of","applying","masks","on","the","token","in","the","sequence",".","proposal_distribution",":","float","A","predefined","probability","distribution","for","each","position","in","the","sequence",".","replace_prob",":","float","The","probability","of","replace","current","token","with","a","generator-predicted","token","."]
docstr: Generate masking positions on-the-fly instead of during preprocessing
  Parameters
  ----------
  input_ids
      The batchified input_ids with shape (batch_size, max_seq_length)
  valid_lengths
      The batchified valid_lengths with shape (batch_size, )
  Returns
  ------
  masked_input_ids
      The masked input sequence with 15% tokens are masked with [MASK]
      shape (batch_size, max_seq_length)
  length_masks
      The masking matrix for the whole sequence that indicates the positions
      are greater than valid_length.
  
      shape (batch_size, max_seq_length)
  unmasked_tokens
      The original tokens that appear in the unmasked input sequence
      shape (batch_size, num_masked_positions)
  masked_positions
      The masking positions in mx.np.ndarray with shape (batch_size, num_masked_positions)
      shape (batch_size, num_masked_positions)
  masked_lm_weights
      The weight matrix containing 0 or 1 to mark the actual effect of masked positions
      shape (batch_size, num_masked_positions)
docstr_tok: ["Generate","masking","positions","on-the-fly","instead","of","during","preprocessing","Parameters","--","--","--","--","--","input_ids","The","batchified","input_ids","with","shape","(","batch_size",",","max_seq_length",")","valid_lengths","The","batchified","valid_lengths","with","shape","(","batch_size",",",")","Returns","--","--","--","masked_input_ids","The","masked","input","sequence","with","15","%","tokens","are","masked","with","[","MASK","]","shape","(","batch_size",",","max_seq_length",")","length_masks","The","masking","matrix","for","the","whole","sequence","that","indicates","the","positions","are","greater","than","valid_length",".","shape","(","batch_size",",","max_seq_length",")","unmasked_tokens","The","original","tokens","that","appear","in","the","unmasked","input","sequence","shape","(","batch_size",",","num_masked_positions",")","masked_positions","The","masking","positions","in","mx.np.ndarray","with","shape","(","batch_size",",","num_masked_positions",")","shape","(","batch_size",",","num_masked_positions",")","masked_lm_weights","The","weight","matrix","containing","0","or","1","to","mark","the","actual","effect","of","masked","positions","shape","(","batch_size",",","num_masked_positions",")"]
code: def dynamic_masking(self, input_ids, valid_lengths):
      # TODO(zheyuye), two additional flag `disallow_from_mask` and `already_masked`
      # that control the masking status for each positions in the sequence.
      """
      Generate masking positions on-the-fly instead of during preprocessing
      Parameters
      ----------
      input_ids
          The batchified input_ids with shape (batch_size, max_seq_length)
      valid_lengths
          The batchified valid_lengths with shape (batch_size, )
      Returns
      ------
      masked_input_ids
          The masked input sequence with 15% tokens are masked with [MASK]
          shape (batch_size, max_seq_length)
      length_masks
          The masking matrix for the whole sequence that indicates the positions
          are greater than valid_length.
  
          shape (batch_size, max_seq_length)
      unmasked_tokens
          The original tokens that appear in the unmasked input sequence
          shape (batch_size, num_masked_positions)
      masked_positions
          The masking positions in mx.np.ndarray with shape (batch_size, num_masked_positions)
          shape (batch_size, num_masked_positions)
      masked_lm_weights
          The weight matrix containing 0 or 1 to mark the actual effect of masked positions
          shape (batch_size, num_masked_positions)
      """
      N = self._max_num_masked_position
      # Only valid token without special token are allowed to mask
      valid_candidates = np.ones_like(input_ids, dtype=np.bool)
      ignore_tokens = [
          self.vocab.cls_id,
          self.vocab.sep_id,
          self.vocab.pad_id,
      ]
  
      for ignore_token in ignore_tokens:
          # TODO(zheyuye), Update when operation += supported
          valid_candidates = valid_candidates * np.not_equal(
              input_ids, ignore_token
          )
      valid_lengths = valid_lengths.astype(np.float32)
      valid_candidates = valid_candidates.astype(np.float32)
      num_masked_position = mxnp.maximum(
          1, np.minimum(N, round(valid_lengths * self._mask_prob))
      )
  
      # Get the masking probability of each position
      sample_probs = self._proposal_distribution * valid_candidates
      sample_probs /= mxnp.sum(sample_probs, axis=-1, keepdims=True)
      sample_probs = npx.stop_gradient(sample_probs)
      gumbels = mxnp.random.gumbel(np.zeros_like(sample_probs))
      # Following the instruction of official repo to avoid deduplicate postions
      # with Top_k Sampling as https://github.com/google-research/electra/issues/41
      masked_positions = npx.topk(
          mxnp.log(sample_probs) + gumbels,
          k=N,
          axis=-1,
          ret_typ="indices",
          dtype=np.int32,
      )
  
      masked_weights = npx.sequence_mask(
          mxnp.ones_like(masked_positions),
          sequence_length=num_masked_position,
          use_sequence_length=True,
          axis=1,
          value=0,
      )
      masked_positions = masked_positions * masked_weights
      length_masks = npx.sequence_mask(
          mxnp.ones_like(input_ids, dtype=np.float32),
          sequence_length=valid_lengths,
          use_sequence_length=True,
          axis=1,
          value=0,
      )
      unmasked_tokens = (
          select_vectors_by_position(input_ids, masked_positions)
          * masked_weights
      )
      masked_weights = masked_weights.astype(np.float32)
      replaced_positions = (
          mxnp.random.uniform(
              mxnp.zeros_like(masked_positions),
              mxnp.ones_like(masked_positions),
          )
          < self._replace_prob
      ) * masked_positions
      # dealing with multiple zero values in replaced_positions which causes
      # the [CLS] being replaced
      filled = mxnp.where(
          replaced_positions, self.vocab.mask_id, self.vocab.cls_id
      ).astype(np.int32)
      # Masking token by replacing with [MASK]
      masked_input_ids = update_vectors_by_position(
          input_ids, filled, replaced_positions
      )
  
      # Note: It is likely have multiple zero values in masked_positions if number of masked of
      # positions not reached the maximum. However, this example hardly exists since valid_length
      # is almost always equal to max_seq_length
      masked_input = self.MaskedInput(
          input_ids=masked_input_ids,
          masks=length_masks,
          unmasked_tokens=unmasked_tokens,
          masked_positions=masked_positions,
          masked_weights=masked_weights,
      )
      return masked_input
  
imports: import io
  import os
  import re
  import random
  import logging
  import collections
  import numpy
  from mxnet import np, npx
  from mxnet.gluon import HybridBlock
  from mxnet.gluon.data import ArrayDataset
  import gluonnlp.data.batchify
  from gluonnlp.utils.misc import glob
  from gluonnlp.data.loading import NumpyDataset, DatasetLoader
  from gluonnlp.data.sampler import SplitSampler, FixedBucketSampler
  from gluonnlp.op import select_vectors_by_position, update_vectors_by_position
  from gluonnlp.initializer import TruncNorm
  from gluonnlp.models.electra import ElectraModel, ElectraForPretrain, get_pretrained_electra
label_desc: Use image node_id 2 for calling the ElectraMasker obj's underlying member method code with example usage: obj.dynamic_masking(input_ids, valid_lengths) and returns: masked_input