class_docstr: Abstract class for TextMemmapSequenceToSequenceDataset and BinarizedMemmapSequenceToSequenceDataset.
  This class is not meant to be used standalone and just as an abstract class for the two subclasses.
class_docstr_tok: ["Abstract","class","for","TextMemmapSequenceToSequenceDataset","and","BinarizedMemmapSequenceToSequenceDataset",".","This","class","is","not","meant","to","be","used","standalone","and","just","as","an","abstract","class","for","the","two","subclasses","."]
docstr: src_file_name: Path to a single source file on disk. This is either the path to a raw text file or the prefix to the processed src_file_name.bin/idx files.
  src_file_name: Path to a single target file on disk. This is either the path to a raw text file or the prefix to the processed tgt_file_name.bin/idx files.
  src_tokenizer: Tokenizer for the source dataset. Instance of a class that inherits TokenizerSpec (ex: YTTM, SentencePiece).
  tgt_tokenizer: Tokenizer for the target dataset. Instance of a class that inherits TokenizerSpec (ex: YTTM, SentencePiece).
  max_src_seq_length: Maximum length of the source sequences. Lines above this length will be truncated.
  max_tgt_seq_length: Maximum length of the target sequences. Lines above this length will be truncated.
  seed: Random seed for data shuffling.
  max_num_samples: Maximum number of samples to load. This can be > dataset length if you want to oversample data. If None, all samples will be loaded.
  prepend_id: If not None, prepend this id to the encoder input.
docstr_tok: ["src_file_name",":","Path","to","a","single","source","file","on","disk",".","This","is","either","the","path","to","a","raw","text","file","or","the","prefix","to","the","processed","src_file_name.bin\/idx","files",".","src_file_name",":","Path","to","a","single","target","file","on","disk",".","This","is","either","the","path","to","a","raw","text","file","or","the","prefix","to","the","processed","tgt_file_name.bin\/idx","files",".","src_tokenizer",":","Tokenizer","for","the","source","dataset",".","Instance","of","a","class","that","inherits","TokenizerSpec","(","ex",":","YTTM",",","SentencePiece",")",".","tgt_tokenizer",":","Tokenizer","for","the","target","dataset",".","Instance","of","a","class","that","inherits","TokenizerSpec","(","ex",":","YTTM",",","SentencePiece",")",".","max_src_seq_length",":","Maximum","length","of","the","source","sequences",".","Lines","above","this","length","will","be","truncated",".","max_tgt_seq_length",":","Maximum","length","of","the","target","sequences",".","Lines","above","this","length","will","be","truncated",".","seed",":","Random","seed","for","data","shuffling",".","max_num_samples",":","Maximum","number","of","samples","to","load",".","This","can","be",">","dataset","length","if","you","want","to","oversample","data",".","If","None",",","all","samples","will","be","loaded",".","prepend_id",":","If","not","None",",","prepend","this","id","to","the","encoder","input","."]
code: def __init__(
      self,
      src_file_name: str,
      tgt_file_name: str,
      src_tokenizer: TokenizerSpec,
      tgt_tokenizer: TokenizerSpec,
      max_src_seq_length: int,
      max_tgt_seq_length: int,
      seed: int = 1234,
      add_bos_to_enc: bool = True,
      add_eos_to_enc: bool = True,
      max_num_samples: int = None,
      prepend_id: int = None,
  ):
      """
      src_file_name: Path to a single source file on disk. This is either the path to a raw text file or the prefix to the processed src_file_name.bin/idx files.
      src_file_name: Path to a single target file on disk. This is either the path to a raw text file or the prefix to the processed tgt_file_name.bin/idx files.
      src_tokenizer: Tokenizer for the source dataset. Instance of a class that inherits TokenizerSpec (ex: YTTM, SentencePiece).
      tgt_tokenizer: Tokenizer for the target dataset. Instance of a class that inherits TokenizerSpec (ex: YTTM, SentencePiece).
      max_src_seq_length: Maximum length of the source sequences. Lines above this length will be truncated.
      max_tgt_seq_length: Maximum length of the target sequences. Lines above this length will be truncated.
      seed: Random seed for data shuffling.
      max_num_samples: Maximum number of samples to load. This can be > dataset length if you want to oversample data. If None, all samples will be loaded.
      prepend_id: If not None, prepend this id to the encoder input.
      """
      super().__init__(
          src_file_name=src_file_name,
          tgt_file_name=tgt_file_name,
          src_tokenizer=src_tokenizer,
          tgt_tokenizer=tgt_tokenizer,
          max_src_seq_length=max_src_seq_length,
          max_tgt_seq_length=max_tgt_seq_length,
      )
      self.seed = seed
      self.max_num_samples = max_num_samples
      self.add_bos_to_enc = add_bos_to_enc
      self.add_eos_to_enc = add_eos_to_enc
      self.prepend_id = prepend_id
  
      logging.info(
          f"Desired number of samples : {self.max_num_samples}"
      )
      logging.info(
          f"Source Dataset Length : {len(self.src_indexed_dataset)}"
      )
      logging.info(
          f"Target Dataset Length : {len(self.tgt_indexed_dataset)}"
      )
  
imports: import os
  import numpy
  import torch
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.collections.nlp.data.language_modeling.megatron.dataset_utils import get_indexed_dataset_, get_samples_mapping
  from nemo.collections.nlp.data.language_modeling.text_memmap_dataset import TextMemMapDataset
  from nemo.core.classes import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new IndexedSequenceToSequenceDataset object from inherited base classes: SequenceToSequenceDataset with example: obj = IndexedSequenceToSequenceDataset(src_file_name, tgt_file_name, src_tokenizer, tgt_tokenizer, max_src_seq_length, max_tgt_seq_length, seed, add_bos_to_enc, add_eos_to_enc, max_num_samples, prepend_id)