class_docstr: Memory-mapped text sequence to sequence dataset. Operates pre-tokenized binarized data files.
class_docstr_tok: ["Memory-mapped","text","sequence","to","sequence","dataset",".","Operates","pre-tokenized","binarized","data","files","."]
docstr: src_dataset_prefix: Path to the *prefix* of a single source bin/idx file on disk. This necessitates the existance src_file_prefix.bin and src_file_prefix.idx.
  tgt_dataset_prefix: Path to the *prefix* of a single target aligned with source bin/idx file on disk. This necessitates the existance tgt_file_prefix.bin and tgt_file_prefix.idx.
  src_tokenizer: Tokenizer for the source dataset. Instance of a class that inherits TokenizerSpec (ex: YTTM, SentencePiece).
  tgt_tokenizer: Tokenizer for the target dataset. Instance of a class that inherits TokenizerSpec (ex: YTTM, SentencePiece).
  max_src_seq_length: Maximum length of the source sequences. Lines above this length will be truncated.
  max_tgt_seq_length: Maximum length of the target sequences. Lines above this length will be truncated.
  seed: Random seed for data shuffling.
  max_num_samples: Maximum number of samples to load. This can be > dataset length if you want to oversample data. If None, all samples will be loaded.
  add_bos_to_enc: Add BOS token to the encoder input.
  add_eos_to_enc: Add EOS token to the encoder input.
  prepend_id: If not None, prepend this id to the encoder input.
docstr_tok: ["src_dataset_prefix",":","Path","to","the","*","prefix","*","of","a","single","source","bin\/idx","file","on","disk",".","This","necessitates","the","existance","src_file_prefix.bin","and","src_file_prefix.idx",".","tgt_dataset_prefix",":","Path","to","the","*","prefix","*","of","a","single","target","aligned","with","source","bin\/idx","file","on","disk",".","This","necessitates","the","existance","tgt_file_prefix.bin","and","tgt_file_prefix.idx",".","src_tokenizer",":","Tokenizer","for","the","source","dataset",".","Instance","of","a","class","that","inherits","TokenizerSpec","(","ex",":","YTTM",",","SentencePiece",")",".","tgt_tokenizer",":","Tokenizer","for","the","target","dataset",".","Instance","of","a","class","that","inherits","TokenizerSpec","(","ex",":","YTTM",",","SentencePiece",")",".","max_src_seq_length",":","Maximum","length","of","the","source","sequences",".","Lines","above","this","length","will","be","truncated",".","max_tgt_seq_length",":","Maximum","length","of","the","target","sequences",".","Lines","above","this","length","will","be","truncated",".","seed",":","Random","seed","for","data","shuffling",".","max_num_samples",":","Maximum","number","of","samples","to","load",".","This","can","be",">","dataset","length","if","you","want","to","oversample","data",".","If","None",",","all","samples","will","be","loaded",".","add_bos_to_enc",":","Add","BOS","token","to","the","encoder","input",".","add_eos_to_enc",":","Add","EOS","token","to","the","encoder","input",".","prepend_id",":","If","not","None",",","prepend","this","id","to","the","encoder","input","."]
code: def __init__(
      self,
      src_dataset_prefix: str,
      tgt_dataset_prefix: str,
      src_tokenizer: TokenizerSpec,
      tgt_tokenizer: TokenizerSpec,
      max_src_seq_length: int,
      max_tgt_seq_length: int,
      seed: int = 1234,
      max_num_samples: int = None,
      add_bos_to_enc: bool = True,
      add_eos_to_enc: bool = True,
      prepend_id: int = None,
  ):
      """
      src_dataset_prefix: Path to the *prefix* of a single source bin/idx file on disk. This necessitates the existance src_file_prefix.bin and src_file_prefix.idx.
      tgt_dataset_prefix: Path to the *prefix* of a single target aligned with source bin/idx file on disk. This necessitates the existance tgt_file_prefix.bin and tgt_file_prefix.idx.
      src_tokenizer: Tokenizer for the source dataset. Instance of a class that inherits TokenizerSpec (ex: YTTM, SentencePiece).
      tgt_tokenizer: Tokenizer for the target dataset. Instance of a class that inherits TokenizerSpec (ex: YTTM, SentencePiece).
      max_src_seq_length: Maximum length of the source sequences. Lines above this length will be truncated.
      max_tgt_seq_length: Maximum length of the target sequences. Lines above this length will be truncated.
      seed: Random seed for data shuffling.
      max_num_samples: Maximum number of samples to load. This can be > dataset length if you want to oversample data. If None, all samples will be loaded.
      add_bos_to_enc: Add BOS token to the encoder input.
      add_eos_to_enc: Add EOS token to the encoder input.
      prepend_id: If not None, prepend this id to the encoder input.
      """
      self.src_dataset_prefix = src_dataset_prefix
      self.tgt_dataset_prefix = tgt_dataset_prefix
      self.seed = seed
      self.max_num_samples = max_num_samples
      super().__init__(
          src_file_name=src_dataset_prefix,
          tgt_file_name=tgt_dataset_prefix,
          src_tokenizer=src_tokenizer,
          tgt_tokenizer=tgt_tokenizer,
          max_src_seq_length=max_src_seq_length,
          max_tgt_seq_length=max_tgt_seq_length,
          seed=seed,
          max_num_samples=max_num_samples,
          add_bos_to_enc=add_bos_to_enc,
          add_eos_to_enc=add_eos_to_enc,
          prepend_id=prepend_id,
      )
  
imports: import os
  import numpy
  import torch
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.collections.nlp.data.language_modeling.megatron.dataset_utils import get_indexed_dataset_, get_samples_mapping
  from nemo.collections.nlp.data.language_modeling.text_memmap_dataset import TextMemMapDataset
  from nemo.core.classes import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new BinarizedMemmapSequenceToSequenceDataset object from inherited base classes: IndexedSequenceToSequenceDataset with example: obj = BinarizedMemmapSequenceToSequenceDataset(src_dataset_prefix, tgt_dataset_prefix, src_tokenizer, tgt_tokenizer, max_src_seq_length, max_tgt_seq_length, seed, max_num_samples, add_bos_to_enc, add_eos_to_enc, prepend_id)