class_docstr: Creates dataset to use to do end-to-end inference
  
  Args:
      input_file: path to the raw data file (e.g., train.tsv). For more info about the data format, refer to the `text_normalization doc <https://github.com/NVIDIA/NeMo/blob/main/docs/source/nlp/text_normalization.rst>`.
      mode: should be one of the values ['tn', 'itn', 'joint'].  `tn` mode is for TN only. `itn` mode is for ITN only. `joint` is for training a system that can do both TN and ITN at the same time.
      lang: Language of the dataset
class_docstr_tok: ["Creates","dataset","to","use","to","do","end-to-end","inference","Args",":","input_file",":","path","to","the","raw","data","file","(","e.g.",",","train.tsv",")",".","For","more","info","about","the","data","format",",","refer","to","the","`","text_normalization","doc","<","https",":","\/\/github.com\/NVIDIA\/NeMo\/blob\/main\/docs\/source\/nlp\/text_normalization.rst",">","`",".","mode",":","should","be","one","of","the","values","[","'tn","'",",","'itn","'",",","'joint","'","]",".","`","tn","`","mode","is","for","TN","only",".","`","itn","`","mode","is","for","ITN","only",".","`","joint","`","is","for","training","a","system","that","can","do","both","TN","and","ITN","at","the","same","time",".","lang",":","Language","of","the","dataset"]
code: def __init__(self, input_file: str, mode: str, lang: str):
      self.lang = lang
      insts = read_data_file(input_file, lang=lang)
      processor = MosesProcessor(lang_id=lang)
      # Build inputs and targets
      (
          self.directions,
          self.inputs,
          self.targets,
          self.classes,
          self.nb_spans,
          self.span_starts,
          self.span_ends,
      ) = (
          [],
          [],
          [],
          [],
          [],
          [],
          [],
      )
      for classes, w_words, s_words in insts:
          # Extract words that are not punctuations
          for direction in constants.INST_DIRECTIONS:
              if direction == constants.INST_BACKWARD:
                  if mode == constants.TN_MODE:
                      continue
                  # ITN mode
                  (
                      processed_w_words,
                      processed_s_words,
                      processed_classes,
                      processed_nb_spans,
                      processed_s_span_starts,
                      processed_s_span_ends,
                  ) = ([], [], [], 0, [], [])
                  s_word_idx = 0
                  for cls, w_word, s_word in zip(
                      classes, w_words, s_words
                  ):
                      if s_word == constants.SIL_WORD:
                          continue
                      elif s_word == constants.SELF_WORD:
                          processed_s_words.append(w_word)
                      else:
                          processed_s_words.append(s_word)
  
                      s_word_last = processor.tokenize(
                          processed_s_words.pop()
                      ).split()
                      processed_s_words.append(" ".join(s_word_last))
                      num_tokens = len(s_word_last)
                      processed_nb_spans += 1
                      processed_classes.append(cls)
                      processed_s_span_starts.append(s_word_idx)
                      s_word_idx += num_tokens
                      processed_s_span_ends.append(s_word_idx)
                      processed_w_words.append(w_word)
  
                  self.span_starts.append(processed_s_span_starts)
                  self.span_ends.append(processed_s_span_ends)
                  self.classes.append(processed_classes)
                  self.nb_spans.append(processed_nb_spans)
                  input_words = " ".join(processed_s_words)
                  # Update self.directions, self.inputs, self.targets
                  self.directions.append(direction)
                  self.inputs.append(input_words)
                  self.targets.append(
                      processed_w_words
                  )  # is list of lists where inner list contains target tokens (not words)
              # TN mode
              elif direction == constants.INST_FORWARD:
                  if mode == constants.ITN_MODE:
                      continue
                  (
                      processed_w_words,
                      processed_s_words,
                      processed_classes,
                      processed_nb_spans,
                      w_span_starts,
                      w_span_ends,
                  ) = ([], [], [], 0, [], [])
                  w_word_idx = 0
                  for cls, w_word, s_word in zip(
                      classes, w_words, s_words
                  ):
                      # TN forward mode
                      # this is done for cases like `do n't`, this w_word will be treated as 2 tokens
                      w_word = processor.tokenize(w_word).split()
                      num_tokens = len(w_word)
                      if s_word in constants.SPECIAL_WORDS:
                          processed_s_words.append(" ".join(w_word))
                      else:
                          processed_s_words.append(s_word)
                      w_span_starts.append(w_word_idx)
                      w_word_idx += num_tokens
                      w_span_ends.append(w_word_idx)
                      processed_nb_spans += 1
                      processed_classes.append(cls)
                      processed_w_words.extend(w_word)
  
                  self.span_starts.append(w_span_starts)
                  self.span_ends.append(w_span_ends)
                  self.classes.append(processed_classes)
                  self.nb_spans.append(processed_nb_spans)
                  input_words = " ".join(processed_w_words)
                  # Update self.directions, self.inputs, self.targets
                  self.directions.append(direction)
                  self.inputs.append(input_words)
                  self.targets.append(
                      processed_s_words
                  )  # is list of lists where inner list contains target tokens (not words)
  
      self.examples = list(
          zip(
              self.directions,
              self.inputs,
              self.targets,
              self.classes,
              self.nb_spans,
              self.span_starts,
              self.span_ends,
          )
      )
  
file_path: NeMo/nemo/collections/nlp/data/text_normalization/test_dataset.py
imports: from collections import defaultdict
  from typing import List
  from nemo.collections.common.tokenizers.moses_tokenizers import MosesProcessor
  from nemo.collections.nlp.data.text_normalization import constants
  from nemo.collections.nlp.data.text_normalization.utils import normalize_str, read_data_file, remove_puncts
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new TextNormalizationTestDataset object  with example: obj = TextNormalizationTestDataset(input_file, mode, lang)