class_docstr: Sequence to Sequence Dataset in memory.
  Similar to SequenceToSequenceDataset but with the same input format as GPTSFTDataset
class_docstr_tok: ["Sequence","to","Sequence","Dataset","in","memory",".","Similar","to","SequenceToSequenceDataset","but","with","the","same","input","format","as","GPTSFTDataset"]
docstr: index_mapping_dir: Directory to save the index mapping to. If None, will write to the same folder as the dataset.
  hf_dataset: Whether to load the json file with the HuggingFace dataset. otherwise, will load the jsonl file with the JSONLMemMapDataset.
docstr_tok: ["index_mapping_dir",":","Directory","to","save","the","index","mapping","to",".","If","None",",","will","write","to","the","same","folder","as","the","dataset",".","hf_dataset",":","Whether","to","load","the","json","file","with","the","HuggingFace","dataset",".","otherwise",",","will","load","the","jsonl","file","with","the","JSONLMemMapDataset","."]
code: def __init__(
      self,
      file_path: str,
      src_tokenizer: TokenizerSpec,
      tgt_tokenizer: TokenizerSpec,
      max_src_seq_length: int,
      max_tgt_seq_length: int,
      add_bos_to_input: bool = True,
      add_eos_to_input: bool = True,
      replace_bos_with_pad: bool = False,
      index_mapping_dir: str = None,
      memmap_workers: Optional[int] = None,
      hf_dataset: bool = False,
  ):
      """
      index_mapping_dir: Directory to save the index mapping to. If None, will write to the same folder as the dataset.
      hf_dataset: Whether to load the json file with the HuggingFace dataset. otherwise, will load the jsonl file with the JSONLMemMapDataset.
      """
      super().__init__()
      self.file_path = file_path
      self.src_tokenizer = src_tokenizer
      self.tgt_tokenizer = tgt_tokenizer
      self.max_src_seq_length = max_src_seq_length
      self.max_tgt_seq_length = max_tgt_seq_length
      self.add_bos_to_input = add_bos_to_input
      self.add_eos_to_input = add_eos_to_input
      self.replace_bos_with_pad = replace_bos_with_pad
      assert self.max_src_seq_length > 0
      assert self.max_tgt_seq_length > 0
  
      # check file exists
      if not os.path.exists(self.file_path):
          raise FileNotFoundError(
              f"Data file {self.file_path} not found"
          )
  
      if hf_dataset:
          self.indexed_dataset = load_dataset(
              "json",
              data_files=file_path,
              cache_dir=index_mapping_dir,
              num_proc=memmap_workers,
              split="train",
          )
      else:
          self.indexed_dataset = JSONLMemMapDataset(
              dataset_paths=[file_path],
              tokenizer=None,
              header_lines=0,
              index_mapping_dir=index_mapping_dir,
              workers=memmap_workers,
          )
  
file_path: NeMo/nemo/collections/nlp/data/language_modeling/megatron/t5_sft_dataset.py
imports: import json
  import os
  from typing import Optional
  import numpy
  import torch
  from datasets import load_dataset
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.collections.nlp.data.language_modeling.text_memmap_dataset import JSONLMemMapDataset
  from nemo.core.classes import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new T5SFTDataset object from inherited base classes: Dataset with example: obj = T5SFTDataset(file_path, src_tokenizer, tgt_tokenizer, max_src_seq_length, max_tgt_seq_length, add_bos_to_input, add_eos_to_input, replace_bos_with_pad, index_mapping_dir, memmap_workers, hf_dataset)