class_docstr: A dataset class that converts raw data to a dataset that can be used by NeuralMachineTranslationModel.
  
  Args:
      filepath: .tsv file to sequence + label.
          the first line is header (sentence [tab] label)
          each line should be [sentence][tab][label]
      encoder_tokenizer: encoder tokenizer object such as AutoTokenizer
      decoder_tokenizer: decoder tokenizer object. If using BART or end to end model, set this to encoder_tokenizer
      max_seq_length: max sequence length including bos and eos tokens
      num_samples: number of samples you want to use for the dataset. If -1, use all dataset. Useful for testing.
      convert_labels: if true, converts labels for masked lm and updates pad_id to -100
          for hf masked loss
class_docstr_tok: ["A","dataset","class","that","converts","raw","data","to","a","dataset","that","can","be","used","by","NeuralMachineTranslationModel",".","Args",":","filepath",":",".tsv","file","to","sequence","+","label",".","the","first","line","is","header","(","sentence","[","tab","]","label",")","each","line","should","be","[","sentence","]","[","tab","]","[","label","]","encoder_tokenizer",":","encoder","tokenizer","object","such","as","AutoTokenizer","decoder_tokenizer",":","decoder","tokenizer","object",".","If","using","BART","or","end","to","end","model",",","set","this","to","encoder_tokenizer","max_seq_length",":","max","sequence","length","including","bos","and","eos","tokens","num_samples",":","number","of","samples","you","want","to","use","for","the","dataset",".","If","-1",",","use","all","dataset",".","Useful","for","testing",".","convert_labels",":","if","true",",","converts","labels","for","masked","lm","and","updates","pad_id","to","-100","for","hf","masked","loss"]
docstr: Converts text to ids. Truncates and adds padding.
docstr_tok: ["Converts","text","to","ids",".","Truncates","and","adds","padding","."]
code: def text_to_ids(
      self,
      text: str,
      tokenizer: AutoTokenizer,
      add_special_tokens=False,
  ) -> Tuple[List[int], List[int]]:
      """Converts text to ids. Truncates and adds padding."""
      text_tokens = tokenizer.text_to_ids(text.strip())
      num_special_tokens = 2 if add_special_tokens else 0
  
      if (
          self.max_seq_length
          and self.max_seq_length > num_special_tokens
      ):
          text_tokens = text_tokens[
              : self.max_seq_length - num_special_tokens
          ]
      if add_special_tokens:
          text_tokens = (
              [tokenizer.bos_id] + text_tokens + [tokenizer.eos_id]
          )
      mask = [1] * len(text_tokens)
  
      if (
          self.max_seq_length
          and self.max_seq_length > num_special_tokens
      ):
          pad_length = self.max_seq_length - len(text_tokens)
          text_tokens += [tokenizer.pad_id] * pad_length
          mask += [0] * pad_length
  
      return text_tokens, mask
  
file_path: NeMo/nemo/collections/nlp/data/text2sparql/text2sparql_dataset.py
imports: import os
  from typing import Dict, List, Optional, Tuple
  import numpy
  from nemo.collections.common.tokenizers import AutoTokenizer
  from nemo.core.classes import Dataset
  from nemo.core.neural_types import ChannelType, MaskType, NeuralType
label_desc: Use image node_id 3 for calling the Text2SparqlDataset obj's underlying member method code with example usage: obj.text_to_ids(text, tokenizer, add_special_tokens) and returns: text_tokens,  mask