class_docstr: Creates dataset to use to train a DuplexTaggerModel.
  
  Converts from raw data to an instance that can be used by Dataloader.
  
  For dataset to use to do end-to-end inference, see TextNormalizationTestDataset.
  
  Args:
      input_file: path to the raw data file (e.g., train.tsv). For more info about the data format, refer to the `text_normalization doc <https://github.com/NVIDIA/NeMo/blob/main/docs/source/nlp/text_normalization.rst>`.
      tokenizer: tokenizer of the model that will be trained on the dataset
      tokenizer_name: name of the tokenizer,
      mode: should be one of the values ['tn', 'itn', 'joint'].  `tn` mode is for TN only. `itn` mode is for ITN only. `joint` is for training a system that can do both TN and ITN at the same time.
      tagger_data_augmentation (bool): a flag indicates whether to augment the dataset with additional data instances
      lang: language of the dataset
      use_cache: Enables caching to use pickle format to store and read data from,
      max_insts: Maximum number of instances (-1 means no limit)
class_docstr_tok: ["Creates","dataset","to","use","to","train","a","DuplexTaggerModel",".","Converts","from","raw","data","to","an","instance","that","can","be","used","by","Dataloader",".","For","dataset","to","use","to","do","end-to-end","inference",",","see","TextNormalizationTestDataset",".","Args",":","input_file",":","path","to","the","raw","data","file","(","e.g.",",","train.tsv",")",".","For","more","info","about","the","data","format",",","refer","to","the","`","text_normalization","doc","<","https",":","\/\/github.com\/NVIDIA\/NeMo\/blob\/main\/docs\/source\/nlp\/text_normalization.rst",">","`",".","tokenizer",":","tokenizer","of","the","model","that","will","be","trained","on","the","dataset","tokenizer_name",":","name","of","the","tokenizer",",","mode",":","should","be","one","of","the","values","[","'tn","'",",","'itn","'",",","'joint","'","]",".","`","tn","`","mode","is","for","TN","only",".","`","itn","`","mode","is","for","ITN","only",".","`","joint","`","is","for","training","a","system","that","can","do","both","TN","and","ITN","at","the","same","time",".","tagger_data_augmentation","(","bool",")",":","a","flag","indicates","whether","to","augment","the","dataset","with","additional","data","instances","lang",":","language","of","the","dataset","use_cache",":","Enables","caching","to","use","pickle","format","to","store","and","read","data","from",",","max_insts",":","Maximum","number","of","instances","(","-1","means","no","limit",")"]
code: def __init__(
      self,
      input_file: str,
      tokenizer: PreTrainedTokenizerBase,
      tokenizer_name: str,
      mode: str,
      tagger_data_augmentation: bool,
      lang: str,
      max_seq_length: int,
      use_cache: bool = False,
      max_insts: int = -1,
  ):
      assert mode in constants.MODES
      assert lang in constants.SUPPORTED_LANGS
      self.mode = mode
      self.lang = lang
      self.use_cache = use_cache
      self.max_insts = max_insts
  
      # Get cache path
      data_dir, filename = os.path.split(input_file)
      tokenizer_name_normalized = tokenizer_name.replace("/", "_")
      cached_data_file = os.path.join(
          data_dir,
          f"cached_tagger_{filename}_{tokenizer_name_normalized}_{lang}_{max_insts}_{max_seq_length}.pkl",
      )
  
      if use_cache and os.path.exists(cached_data_file):
          logging.warning(
              f"Processing of {input_file} is skipped as caching is enabled and a cache file "
              f"{cached_data_file} already exists."
          )
          with open(cached_data_file, "rb") as f:
              data = pickle.load(f)
              (
                  self.insts,
                  self.tag2id,
                  self.encodings,
                  self.labels,
              ) = data
      else:
          # Read the input raw data file, returns list of sentences parsed as list of class, w_words, s_words
          raw_insts = read_data_file(input_file, lang=lang)
          if max_insts >= 0:
              raw_insts = raw_insts[:max_insts]
  
          # Convert raw instances to TaggerDataInstance
          insts = []
          for _, w_words, s_words in tqdm(raw_insts):
              for inst_dir in constants.INST_DIRECTIONS:
                  if (
                      inst_dir == constants.INST_BACKWARD
                      and mode == constants.TN_MODE
                  ):
                      continue
                  if (
                      inst_dir == constants.INST_FORWARD
                      and mode == constants.ITN_MODE
                  ):
                      continue
  
                  # filter out examples that are longer than the maximum sequence length value
                  if (
                      len(
                          tokenizer(
                              w_words,
                              is_split_into_words=True,
                              padding=False,
                              truncation=True,
                          )["input_ids"]
                      )
                      >= max_seq_length
                      or len(
                          tokenizer(
                              s_words,
                              is_split_into_words=True,
                              padding=False,
                              truncation=True,
                          )["input_ids"]
                      )
                      >= max_seq_length
                  ):
                      continue
  
                  # Create a new TaggerDataInstance
                  inst = TaggerDataInstance(
                      w_words, s_words, inst_dir, lang=self.lang
                  )
                  insts.append(inst)
                  # Data Augmentation (if enabled)
                  if tagger_data_augmentation:
                      filtered_w_words, filtered_s_words = [], []
                      for ix, (w, s) in enumerate(
                          zip(w_words, s_words)
                      ):
                          if not s in constants.SPECIAL_WORDS:
                              filtered_w_words.append(w)
                              filtered_s_words.append(s)
                      if len(filtered_s_words) > 1:
                          inst = TaggerDataInstance(
                              filtered_w_words,
                              filtered_s_words,
                              inst_dir,
                              lang,
                          )
                          insts.append(inst)
  
          self.insts = insts
          texts = [inst.input_words for inst in insts]
          tags = [inst.labels for inst in insts]
  
          # Tags Mapping
          self.tag2id = {
              tag: id for id, tag in enumerate(constants.ALL_TAG_LABELS)
          }
  
          # Finalize
          self.encodings = tokenizer(
              texts,
              is_split_into_words=True,
              padding=False,
              truncation=True,
          )
          self.labels = self.encode_tags(tags, self.encodings)
  
          # Write to cache (if use_cache)
          if use_cache:
              with open(cached_data_file, "wb") as out_file:
                  data = (
                      self.insts,
                      self.tag2id,
                      self.encodings,
                      self.labels,
                  )
                  pickle.dump(
                      data, out_file, protocol=pickle.HIGHEST_PROTOCOL
                  )
  
file_path: NeMo/nemo/collections/nlp/data/text_normalization/tagger_dataset.py
imports: import os
  import pickle
  from tqdm import tqdm
  from transformers import PreTrainedTokenizerBase
  from nemo.collections.common.tokenizers.moses_tokenizers import MosesProcessor
  from nemo.collections.nlp.data.text_normalization import constants
  from nemo.collections.nlp.data.text_normalization.utils import read_data_file
  from nemo.core.classes import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 1 to create a new TextNormalizationTaggerDataset object from inherited base classes: Dataset with example: obj = TextNormalizationTaggerDataset(input_file, tokenizer, tokenizer_name, mode, tagger_data_augmentation, lang, max_seq_length, use_cache, max_insts)