class_docstr: Creates dataset to use to train a DuplexDecoderModel.
  Converts from raw data to an instance that can be used by Dataloader.
  For dataset to use to do end-to-end inference, see TextNormalizationTestDataset.
  
  Args:
      input_file: path to the raw data file (e.g., train.tsv).
          For more info about the data format, refer to the
          `text_normalization doc <https://github.com/NVIDIA/NeMo/blob/main/docs/source/nlp/text_normalization/nn_text_normalization.rst>`.
      raw_instances: processed raw instances in the Google TN dataset format (used for tarred dataset)
      tokenizer: tokenizer of the model that will be trained on the dataset
      tokenizer_name: name of the tokenizer,
      mode: should be one of the values ['tn', 'itn', 'joint'].  `tn` mode is for TN only.
          `itn` mode is for ITN only. `joint` is for training a system that can do both TN and ITN at the same time.
      max_len: maximum length of sequence in tokens. The code will discard any training instance whose input or
          output is longer than the specified max_len.
      decoder_data_augmentation (bool): a flag indicates whether to augment the dataset with additional data
          instances that may help the decoder become more robust against the tagger's errors.
          Refer to the doc for more info.
      lang: language of the dataset
      use_cache: Enables caching to use pickle format to store and read data from
      max_insts: Maximum number of instances (-1 means no limit)
      do_tokenize: Tokenize each instance (set to False for Tarred dataset)
      initial_shuffle: Set to True to shuffle the data
class_docstr_tok: ["Creates","dataset","to","use","to","train","a","DuplexDecoderModel",".","Converts","from","raw","data","to","an","instance","that","can","be","used","by","Dataloader",".","For","dataset","to","use","to","do","end-to-end","inference",",","see","TextNormalizationTestDataset",".","Args",":","input_file",":","path","to","the","raw","data","file","(","e.g.",",","train.tsv",")",".","For","more","info","about","the","data","format",",","refer","to","the","`","text_normalization","doc","<","https",":","\/\/github.com\/NVIDIA\/NeMo\/blob\/main\/docs\/source\/nlp\/text_normalization\/nn_text_normalization.rst",">","`",".","raw_instances",":","processed","raw","instances","in","the","Google","TN","dataset","format","(","used","for","tarred","dataset",")","tokenizer",":","tokenizer","of","the","model","that","will","be","trained","on","the","dataset","tokenizer_name",":","name","of","the","tokenizer",",","mode",":","should","be","one","of","the","values","[","'tn","'",",","'itn","'",",","'joint","'","]",".","`","tn","`","mode","is","for","TN","only",".","`","itn","`","mode","is","for","ITN","only",".","`","joint","`","is","for","training","a","system","that","can","do","both","TN","and","ITN","at","the","same","time",".","max_len",":","maximum","length","of","sequence","in","tokens",".","The","code","will","discard","any","training","instance","whose","input","or","output","is","longer","than","the","specified","max_len",".","decoder_data_augmentation","(","bool",")",":","a","flag","indicates","whether","to","augment","the","dataset","with","additional","data","instances","that","may","help","the","decoder","become","more","robust","against","the","tagger","'s","errors",".","Refer","to","the","doc","for","more","info",".","lang",":","language","of","the","dataset","use_cache",":","Enables","caching","to","use","pickle","format","to","store","and","read","data","from","max_insts",":","Maximum","number","of","instances","(","-1","means","no","limit",")","do_tokenize",":","Tokenize","each","instance","(","set","to","False","for","Tarred","dataset",")","initial_shuffle",":","Set","to","True","to","shuffle","the","data"]
docstr: Converts raw instances to DecoderDataInstance
  
  raw_instances: raw entries: (semiotic class, written words, spoken words)
  decoder_data_augmentation (bool): a flag indicates whether to augment the dataset with additional data
      instances that may help the decoder become more robust against the tagger's errors.
      Refer to the doc for more info.
  
  Returns:
      converted instances and all semiotic classes present in the data
docstr_tok: ["Converts","raw","instances","to","DecoderDataInstance","raw_instances",":","raw","entries",":","(","semiotic","class",",","written","words",",","spoken","words",")","decoder_data_augmentation","(","bool",")",":","a","flag","indicates","whether","to","augment","the","dataset","with","additional","data","instances","that","may","help","the","decoder","become","more","robust","against","the","tagger","'s","errors",".","Refer","to","the","doc","for","more","info",".","Returns",":","converted","instances","and","all","semiotic","classes","present","in","the","data"]
code: def __process_raw_entries(
      self, raw_instances: List[Tuple[str]], decoder_data_augmentation
  ):
      """
      Converts raw instances to DecoderDataInstance
  
      raw_instances: raw entries: (semiotic class, written words, spoken words)
      decoder_data_augmentation (bool): a flag indicates whether to augment the dataset with additional data
          instances that may help the decoder become more robust against the tagger's errors.
          Refer to the doc for more info.
  
      Returns:
          converted instances and all semiotic classes present in the data
      """
      all_semiotic_classes = set([])
      insts = []
      for classes, w_words, s_words in tqdm(raw_instances):
          for ix, (_class, w_word, s_word) in enumerate(
              zip(classes, w_words, s_words)
          ):
              all_semiotic_classes.update([_class])
              if s_word in constants.SPECIAL_WORDS:
                  continue
              for inst_dir in constants.INST_DIRECTIONS:
                  if (
                      inst_dir == constants.INST_BACKWARD
                      and self.mode == constants.TN_MODE
                  ):
                      continue
                  if (
                      inst_dir == constants.INST_FORWARD
                      and self.mode == constants.ITN_MODE
                  ):
                      continue
                  # Create a DecoderDataInstance
                  inst = DecoderDataInstance(
                      w_words,
                      s_words,
                      inst_dir,
                      start_idx=ix,
                      end_idx=ix + 1,
                      lang=self.lang,
                      semiotic_class=_class,
                  )
                  insts.append(inst)
  
                  if decoder_data_augmentation:
                      noise_left = random.randint(1, 2)
                      noise_right = random.randint(1, 2)
                      inst = DecoderDataInstance(
                          w_words,
                          s_words,
                          inst_dir,
                          start_idx=ix - noise_left,
                          end_idx=ix + 1 + noise_right,
                          semiotic_class=_class,
                          lang=self.lang,
                      )
                      insts.append(inst)
  
      all_semiotic_classes = list(all_semiotic_classes)
      all_semiotic_classes.sort()
      return insts, all_semiotic_classes
  
imports: import io
  import os
  import pickle
  import random
  from collections import OrderedDict
  from typing import List, Optional, Tuple
  import braceexpand
  import numpy
  import torch
  import webdataset
  from torch.utils.data import IterableDataset
  from tqdm import tqdm
  from transformers import PreTrainedTokenizerBase
  from nemo.collections.common.tokenizers.moses_tokenizers import MosesProcessor
  from nemo.collections.nlp.data.text_normalization import constants
  from nemo.collections.nlp.data.text_normalization.utils import read_data_file
  from nemo.core.classes import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 2 for calling the TextNormalizationDecoderDataset obj's underlying member method code with example usage: obj.__process_raw_entries(raw_instances, decoder_data_augmentation) and returns: insts,  all_semiotic_classes