class_docstr: UL2 Dataset from https://arxiv.org/abs/2205.05131.
  Consists of three different objectives:
  1. Short span masking with small probabilities (ex: T5). Typically max ngram size of 5 with 0.15 mask prob.
  2. Extreme span masking with either large probabilities or large ngram sizes or both.
  3. Prefx-LM as in the T5 or LM-adapted T5 (prompt-tuning paper).
class_docstr_tok: ["UL2","Dataset","from","https",":","\/\/arxiv.org\/abs\/2205.05131",".","Consists","of","three","different","objectives",":","1",".","Short","span","masking","with","small","probabilities","(","ex",":","T5",")",".","Typically","max","ngram","size","of","5","with","0.15","mask","prob",".","2",".","Extreme","span","masking","with","either","large","probabilities","or","large","ngram","sizes","or","both",".","3",".","Prefx-LM","as","in","the","T5","or","LM-adapted","T5","(","prompt-tuning","paper",")","."]
code: def __init__(
      self,
      cfg,
      trainer,
      tokenizer,
      name,
      indexed_dataset,
      data_prefix,
      num_epochs,
      max_num_samples,
      max_seq_length,
      max_seq_length_dec,
      seed,
      masked_lm_prob=0.15,
      extreme_masked_lm_prob=0.5,
      short_seq_prob=0.0,
      min_ngram_size=2,
      max_ngram_size=10,
      mean_ngram_size=3,
      extreme_max_ngram_size=128,
      extreme_min_ngram_size=32,
      extreme_mean_ngram_size=64,
      prefix_lm_pivot_mean=0.25,  # This is represented as a percentage of the total length.
      ngram_span_length_distribution=LengthDistribution.geometric,
      extreme_ngram_span_length_distribution=LengthDistribution.truncated_normal,
      permutation=False,
      whole_word_masking=True,
      favor_long_ngrams=False,
      respect_document_boundaries=True,
      documents=None,
  ):
      super().__init__(
          cfg=cfg,
          trainer=trainer,
          tokenizer=tokenizer,
          name=name,
          indexed_dataset=indexed_dataset,
          data_prefix=data_prefix,
          num_epochs=num_epochs,
          max_num_samples=max_num_samples,
          max_seq_length=max_seq_length
          - 1,  # -1 to account for the added mask type token
          max_seq_length_dec=max_seq_length_dec,
          seed=seed,
          masked_lm_prob=masked_lm_prob,
          short_seq_prob=short_seq_prob,
          max_ngram_size=max_ngram_size,
          mean_ngram_size=None,  # TODO: Determin if we want to actually pass mean ngram as an override to max here.
          geometric_dist=ngram_span_length_distribution
          == LengthDistribution.geometric,
          permutation=permutation,
          whole_word_masking=whole_word_masking,
          favor_long_ngrams=favor_long_ngrams,
          respect_document_boundaries=respect_document_boundaries,
          documents=documents,
      )
      self.mean_ngram_size = mean_ngram_size
      self.min_ngram_size = min_ngram_size
      self.extreme_masked_lm_prob = extreme_masked_lm_prob
      self.extreme_min_ngram_size = extreme_min_ngram_size
      self.extreme_max_ngram_size = extreme_max_ngram_size
      self.extreme_mean_ngram_size = extreme_mean_ngram_size
      self.ngram_span_length_distribution = (
          ngram_span_length_distribution
      )
      self.extreme_ngram_span_length_distribution = (
          extreme_ngram_span_length_distribution
      )
      self.prefix_lm_pivot_mean = prefix_lm_pivot_mean
  
file_path: NeMo/nemo/collections/nlp/data/language_modeling/megatron/ul2_dataset.py
imports: import numpy
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.collections.nlp.data.language_modeling.megatron.dataset_utils import create_extreme_masked_lm_predictions
  from nemo.collections.nlp.data.language_modeling.megatron.length_distribution_type import LengthDistribution
  from nemo.collections.nlp.data.language_modeling.megatron.lm_adapted_t5_dataset import T5LMAdaptedDataset
  from nemo.collections.nlp.data.language_modeling.megatron.t5_dataset import T5Dataset
label_desc: Use image node_id 1 to create a new UL2Dataset object from inherited base classes: T5Dataset with example: obj = UL2Dataset(cfg, trainer, tokenizer, name, indexed_dataset, data_prefix, num_epochs, max_num_samples, max_seq_length, max_seq_length_dec, seed, masked_lm_prob, extreme_masked_lm_prob, short_seq_prob, min_ngram_size, max_ngram_size, mean_ngram_size, extreme_max_ngram_size, extreme_min_ngram_size, extreme_mean_ngram_size, prefix_lm_pivot_mean, ngram_span_length_distribution, extreme_ngram_span_length_distribution, permutation, whole_word_masking, favor_long_ngrams, respect_document_boundaries, documents)