class_docstr: Memory-mapped text sequence to sequence dataset. Operates on raw text files and tokenizes the text on-the-fly.
class_docstr_tok: ["Memory-mapped","text","sequence","to","sequence","dataset",".","Operates","on","raw","text","files","and","tokenizes","the","text","on-the-fly","."]
docstr: src_file_name: Path to a single source file on disk. The file should contain one sentence per line and be raw text.
  tgt_file_name: Path to a single target file on disk. The file should contain one sentence per line aligned with src_file_name and be raw text.
  src_tokenizer: Tokenizer for the source dataset. Instance of a class that inherits TokenizerSpec (ex: YTTM, SentencePiece).
  tgt_tokenizer: Tokenizer for the target dataset. Instance of a class that inherits TokenizerSpec (ex: YTTM, SentencePiece).
  max_src_seq_length: Maximum length of the source sequences. Lines above this length will be truncated.
  max_tgt_seq_length: Maximum length of the target sequences. Lines above this length will be truncated.
  seed: Random seed for data shuffling.
  max_num_samples: Maximum number of samples to load. This can be > dataset length if you want to oversample data. If None, all samples will be loaded.
  add_bos_to_enc: Add BOS token to the encoder input.
  add_eos_to_enc: Add EOS token to the encoder input.
  prepend_id: If not None, prepend this id to the encoder input.
docstr_tok: ["src_file_name",":","Path","to","a","single","source","file","on","disk",".","The","file","should","contain","one","sentence","per","line","and","be","raw","text",".","tgt_file_name",":","Path","to","a","single","target","file","on","disk",".","The","file","should","contain","one","sentence","per","line","aligned","with","src_file_name","and","be","raw","text",".","src_tokenizer",":","Tokenizer","for","the","source","dataset",".","Instance","of","a","class","that","inherits","TokenizerSpec","(","ex",":","YTTM",",","SentencePiece",")",".","tgt_tokenizer",":","Tokenizer","for","the","target","dataset",".","Instance","of","a","class","that","inherits","TokenizerSpec","(","ex",":","YTTM",",","SentencePiece",")",".","max_src_seq_length",":","Maximum","length","of","the","source","sequences",".","Lines","above","this","length","will","be","truncated",".","max_tgt_seq_length",":","Maximum","length","of","the","target","sequences",".","Lines","above","this","length","will","be","truncated",".","seed",":","Random","seed","for","data","shuffling",".","max_num_samples",":","Maximum","number","of","samples","to","load",".","This","can","be",">","dataset","length","if","you","want","to","oversample","data",".","If","None",",","all","samples","will","be","loaded",".","add_bos_to_enc",":","Add","BOS","token","to","the","encoder","input",".","add_eos_to_enc",":","Add","EOS","token","to","the","encoder","input",".","prepend_id",":","If","not","None",",","prepend","this","id","to","the","encoder","input","."]
code: def __init__(
      self,
      src_file_name: str,
      tgt_file_name: str,
      src_tokenizer: TokenizerSpec,
      tgt_tokenizer: TokenizerSpec,
      max_src_seq_length: int,
      max_tgt_seq_length: int,
      seed: int = 1234,
      max_num_samples: int = None,
      add_bos_to_enc: bool = True,
      add_eos_to_enc: bool = True,
      prepend_id: int = None,
  ):
      """
      src_file_name: Path to a single source file on disk. The file should contain one sentence per line and be raw text.
      tgt_file_name: Path to a single target file on disk. The file should contain one sentence per line aligned with src_file_name and be raw text.
      src_tokenizer: Tokenizer for the source dataset. Instance of a class that inherits TokenizerSpec (ex: YTTM, SentencePiece).
      tgt_tokenizer: Tokenizer for the target dataset. Instance of a class that inherits TokenizerSpec (ex: YTTM, SentencePiece).
      max_src_seq_length: Maximum length of the source sequences. Lines above this length will be truncated.
      max_tgt_seq_length: Maximum length of the target sequences. Lines above this length will be truncated.
      seed: Random seed for data shuffling.
      max_num_samples: Maximum number of samples to load. This can be > dataset length if you want to oversample data. If None, all samples will be loaded.
      add_bos_to_enc: Add BOS token to the encoder input.
      add_eos_to_enc: Add EOS token to the encoder input.
      prepend_id: If not None, prepend this id to the encoder input.
      """
      self.seed = seed
      self.max_num_samples = max_num_samples
      super().__init__(
          src_file_name=src_file_name,
          tgt_file_name=tgt_file_name,
          src_tokenizer=src_tokenizer,
          tgt_tokenizer=tgt_tokenizer,
          max_src_seq_length=max_src_seq_length,
          max_tgt_seq_length=max_tgt_seq_length,
          seed=seed,
          max_num_samples=max_num_samples,
          add_bos_to_enc=add_bos_to_enc,
          add_eos_to_enc=add_eos_to_enc,
          prepend_id=prepend_id,
      )
  
imports: import os
  import numpy
  import torch
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.collections.nlp.data.language_modeling.megatron.dataset_utils import get_indexed_dataset_, get_samples_mapping
  from nemo.collections.nlp.data.language_modeling.text_memmap_dataset import TextMemMapDataset
  from nemo.core.classes import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new TextMemmapSequenceToSequenceDataset object from inherited base classes: IndexedSequenceToSequenceDataset with example: obj = TextMemmapSequenceToSequenceDataset(src_file_name, tgt_file_name, src_tokenizer, tgt_tokenizer, max_src_seq_length, max_tgt_seq_length, seed, max_num_samples, add_bos_to_enc, add_eos_to_enc, prepend_id)