docstr: src_file_name: Path to a JSONL T0 dataset file.
  tokenizer: Tokenizer for the dataset. Instance of a class that inherits TokenizerSpec (ex: YTTM, SentencePiece).
  max_src_seq_length: Maximum length of the source sequences. Lines above this length will be truncated.
  max_tgt_seq_length: Maximum length of the target sequences. Lines above this length will be truncated.
  replace_bos_with_pad: Whether the decoder starts with a pad token. This is needed for Google's T5 models that may be converted from HF.
  add_bos_to_input: Whether to add the bos_id to the input sequence.
  add_eos_to_input: Whether to add the eos_id to the input sequence.
  seed: Random seed for data shuffling.
  max_num_samples: Maximum number of samples to load. This can be > dataset length if you want to oversample data. If None, all samples will be loaded.
docstr_tok: ["src_file_name",":","Path","to","a","JSONL","T0","dataset","file",".","tokenizer",":","Tokenizer","for","the","dataset",".","Instance","of","a","class","that","inherits","TokenizerSpec","(","ex",":","YTTM",",","SentencePiece",")",".","max_src_seq_length",":","Maximum","length","of","the","source","sequences",".","Lines","above","this","length","will","be","truncated",".","max_tgt_seq_length",":","Maximum","length","of","the","target","sequences",".","Lines","above","this","length","will","be","truncated",".","replace_bos_with_pad",":","Whether","the","decoder","starts","with","a","pad","token",".","This","is","needed","for","Google","'s","T5","models","that","may","be","converted","from","HF",".","add_bos_to_input",":","Whether","to","add","the","bos_id","to","the","input","sequence",".","add_eos_to_input",":","Whether","to","add","the","eos_id","to","the","input","sequence",".","seed",":","Random","seed","for","data","shuffling",".","max_num_samples",":","Maximum","number","of","samples","to","load",".","This","can","be",">","dataset","length","if","you","want","to","oversample","data",".","If","None",",","all","samples","will","be","loaded","."]
code: def __init__(
      self,
      file_path: str,
      tokenizer: TokenizerSpec,
      max_src_seq_length: int = 512,
      max_tgt_seq_length: int = 512,
      replace_bos_with_pad: bool = False,
      add_bos_to_input: bool = False,
      add_eos_to_input: bool = False,
      max_num_samples: int = None,
      seed: int = 1234,
  ):
      """
      src_file_name: Path to a JSONL T0 dataset file.
      tokenizer: Tokenizer for the dataset. Instance of a class that inherits TokenizerSpec (ex: YTTM, SentencePiece).
      max_src_seq_length: Maximum length of the source sequences. Lines above this length will be truncated.
      max_tgt_seq_length: Maximum length of the target sequences. Lines above this length will be truncated.
      replace_bos_with_pad: Whether the decoder starts with a pad token. This is needed for Google's T5 models that may be converted from HF.
      add_bos_to_input: Whether to add the bos_id to the input sequence.
      add_eos_to_input: Whether to add the eos_id to the input sequence.
      seed: Random seed for data shuffling.
      max_num_samples: Maximum number of samples to load. This can be > dataset length if you want to oversample data. If None, all samples will be loaded.
      """
      self.tokenizer = tokenizer
      self.file_path = file_path
      self.max_src_seq_length = max_src_seq_length
      self.max_tgt_seq_length = max_tgt_seq_length
      self.replace_bos_with_pad = replace_bos_with_pad
      self.add_bos_to_input = add_bos_to_input
      self.add_eos_to_input = add_eos_to_input
      self.max_num_samples = max_num_samples
      self.seed = seed
  
      self.indexed_dataset = JSONLMemMapDataset(
          dataset_paths=[file_path], tokenizer=None, header_lines=0
      )
  
      # Will be None after this call if `max_num_samples` is None
      self._build_samples_mapping()
  
file_path: NeMo/nemo/collections/nlp/data/language_modeling/t0_dataset.py
imports: import numpy
  import torch
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.collections.nlp.data.language_modeling.megatron.dataset_utils import get_samples_mapping
  from nemo.collections.nlp.data.language_modeling.text_memmap_dataset import JSONLMemMapDataset, TextMemMapDataset
  from nemo.core.classes import Dataset
label_desc: Use image node_id 1 to create a new T0Dataset object from inherited base classes: Dataset with example: obj = T0Dataset(file_path, tokenizer, max_src_seq_length, max_tgt_seq_length, replace_bos_with_pad, add_bos_to_input, add_eos_to_input, max_num_samples, seed)