class_docstr: A dataset class that converts raw data to a dataset that can be used by NeuralMachineTranslationModel.
  
  Args:
      filepath: .tsv file to sequence + label.
          the first line is header (sentence [tab] label)
          each line should be [sentence][tab][label]
      encoder_tokenizer: encoder tokenizer object such as AutoTokenizer
      decoder_tokenizer: decoder tokenizer object. If using BART or end to end model, set this to encoder_tokenizer
      max_seq_length: max sequence length including bos and eos tokens
      num_samples: number of samples you want to use for the dataset. If -1, use all dataset. Useful for testing.
      convert_labels: if true, converts labels for masked lm and updates pad_id to -100
          for hf masked loss
class_docstr_tok: ["A","dataset","class","that","converts","raw","data","to","a","dataset","that","can","be","used","by","NeuralMachineTranslationModel",".","Args",":","filepath",":",".tsv","file","to","sequence","+","label",".","the","first","line","is","header","(","sentence","[","tab","]","label",")","each","line","should","be","[","sentence","]","[","tab","]","[","label","]","encoder_tokenizer",":","encoder","tokenizer","object","such","as","AutoTokenizer","decoder_tokenizer",":","decoder","tokenizer","object",".","If","using","BART","or","end","to","end","model",",","set","this","to","encoder_tokenizer","max_seq_length",":","max","sequence","length","including","bos","and","eos","tokens","num_samples",":","number","of","samples","you","want","to","use","for","the","dataset",".","If","-1",",","use","all","dataset",".","Useful","for","testing",".","convert_labels",":","if","true",",","converts","labels","for","masked","lm","and","updates","pad_id","to","-100","for","hf","masked","loss"]
code: def __init__(
      self,
      filepath: str,
      encoder_tokenizer: AutoTokenizer,
      decoder_tokenizer: AutoTokenizer,
      encoder_add_special_tokens: bool,
      decoder_add_special_tokens: bool,
      max_seq_length: int,
      num_samples: int = -1,
      convert_labels: bool = False,
  ):
      self.filepath = filepath
      self.encoder_tokenizer = encoder_tokenizer
      self.decoder_tokenizer = decoder_tokenizer
      self.encoder_add_special_tokens = encoder_add_special_tokens
      self.decoder_add_special_tokens = decoder_add_special_tokens
      self.max_seq_length = max_seq_length
      self.num_samples = num_samples
      self.convert_labels = convert_labels
  
      if num_samples == 0:
          raise ValueError(
              "num_samples has to be positive.", num_samples
          )
  
      if self.max_seq_length and self.max_seq_length <= 2:
          self.max_seq_length = None
  
      if not os.path.exists(filepath):
          raise FileNotFoundError(
              f"{filepath} not found. The filepath must be set in train_ds.filepath and validation_ds.filepath."
          )
  
      with open(filepath) as f:
          lines = f.readlines()[1:]
  
      if num_samples > 0:
          lines = lines[:num_samples]
  
      input_ids, input_masks, label_ids = [], [], []
      for line in lines:
          try:
              sentence, label = line.split("\t")
          except ValueError:
              raise ValueError(
                  "Each line of input file should contain the format [sentence][tab][label]."
              )
          ids, mask = self.text_to_ids(
              sentence,
              tokenizer=encoder_tokenizer,
              add_special_tokens=encoder_add_special_tokens,
          )
          input_ids.append(ids)
          input_masks.append(mask)
          label_ids.append(
              self.text_to_ids(
                  label,
                  tokenizer=decoder_tokenizer,
                  add_special_tokens=decoder_add_special_tokens,
              )[0]
          )
  
      self.input_ids = np.asarray(input_ids)
      self.input_masks = np.asarray(input_masks)
      self.label_ids = np.asarray(label_ids)
  
file_path: NeMo/nemo/collections/nlp/data/text2sparql/text2sparql_dataset.py
imports: import os
  from typing import Dict, List, Optional, Tuple
  import numpy
  from nemo.collections.common.tokenizers import AutoTokenizer
  from nemo.core.classes import Dataset
  from nemo.core.neural_types import ChannelType, MaskType, NeuralType
label_desc: Use image node_id 2 to create a new Text2SparqlDataset object from inherited base classes: Dataset with example: obj = Text2SparqlDataset(filepath, encoder_tokenizer, decoder_tokenizer, encoder_add_special_tokens, decoder_add_special_tokens, max_seq_length, num_samples, convert_labels)