class_docstr: Creates dataset to use to train a DuplexDecoderModel.
  Converts from raw data to an instance that can be used by Dataloader.
  For dataset to use to do end-to-end inference, see TextNormalizationTestDataset.
  
  Args:
      input_file: path to the raw data file (e.g., train.tsv).
          For more info about the data format, refer to the
          `text_normalization doc <https://github.com/NVIDIA/NeMo/blob/main/docs/source/nlp/text_normalization/nn_text_normalization.rst>`.
      raw_instances: processed raw instances in the Google TN dataset format (used for tarred dataset)
      tokenizer: tokenizer of the model that will be trained on the dataset
      tokenizer_name: name of the tokenizer,
      mode: should be one of the values ['tn', 'itn', 'joint'].  `tn` mode is for TN only.
          `itn` mode is for ITN only. `joint` is for training a system that can do both TN and ITN at the same time.
      max_len: maximum length of sequence in tokens. The code will discard any training instance whose input or
          output is longer than the specified max_len.
      decoder_data_augmentation (bool): a flag indicates whether to augment the dataset with additional data
          instances that may help the decoder become more robust against the tagger's errors.
          Refer to the doc for more info.
      lang: language of the dataset
      use_cache: Enables caching to use pickle format to store and read data from
      max_insts: Maximum number of instances (-1 means no limit)
      do_tokenize: Tokenize each instance (set to False for Tarred dataset)
      initial_shuffle: Set to True to shuffle the data
class_docstr_tok: ["Creates","dataset","to","use","to","train","a","DuplexDecoderModel",".","Converts","from","raw","data","to","an","instance","that","can","be","used","by","Dataloader",".","For","dataset","to","use","to","do","end-to-end","inference",",","see","TextNormalizationTestDataset",".","Args",":","input_file",":","path","to","the","raw","data","file","(","e.g.",",","train.tsv",")",".","For","more","info","about","the","data","format",",","refer","to","the","`","text_normalization","doc","<","https",":","\/\/github.com\/NVIDIA\/NeMo\/blob\/main\/docs\/source\/nlp\/text_normalization\/nn_text_normalization.rst",">","`",".","raw_instances",":","processed","raw","instances","in","the","Google","TN","dataset","format","(","used","for","tarred","dataset",")","tokenizer",":","tokenizer","of","the","model","that","will","be","trained","on","the","dataset","tokenizer_name",":","name","of","the","tokenizer",",","mode",":","should","be","one","of","the","values","[","'tn","'",",","'itn","'",",","'joint","'","]",".","`","tn","`","mode","is","for","TN","only",".","`","itn","`","mode","is","for","ITN","only",".","`","joint","`","is","for","training","a","system","that","can","do","both","TN","and","ITN","at","the","same","time",".","max_len",":","maximum","length","of","sequence","in","tokens",".","The","code","will","discard","any","training","instance","whose","input","or","output","is","longer","than","the","specified","max_len",".","decoder_data_augmentation","(","bool",")",":","a","flag","indicates","whether","to","augment","the","dataset","with","additional","data","instances","that","may","help","the","decoder","become","more","robust","against","the","tagger","'s","errors",".","Refer","to","the","doc","for","more","info",".","lang",":","language","of","the","dataset","use_cache",":","Enables","caching","to","use","pickle","format","to","store","and","read","data","from","max_insts",":","Maximum","number","of","instances","(","-1","means","no","limit",")","do_tokenize",":","Tokenize","each","instance","(","set","to","False","for","Tarred","dataset",")","initial_shuffle",":","Set","to","True","to","shuffle","the","data"]
docstr: Creates a batch
  
  Args:
      batch_size: the size of the batch
docstr_tok: ["Creates","a","batch","Args",":","batch_size",":","the","size","of","the","batch"]
code: def batchify(self, batch_size: int):
      """
      Creates a batch
  
      Args:
          batch_size: the size of the batch
      """
      logging.info("Padding the data and creating batches...")
  
      long_examples_filtered = 0
      inputs_all = [inst.input_str.strip() for inst in self.insts]
      targets_all = [inst.output_str.strip() for inst in self.insts]
      batch, batches = [], []
      for idx in tqdm(range(len(self.insts))):
          # exclude examples that are longer than maximum sequence length
          # Input
          _input = self.tokenizer([inputs_all[idx]])
          input_len = len(_input["input_ids"][0])
          if input_len > self.max_seq_len:
              long_examples_filtered += 1
              continue
  
          # Target
          _target = self.tokenizer([targets_all[idx]])
          target_len = len(_target["input_ids"][0])
          if target_len > self.max_seq_len:
              long_examples_filtered += 1
              continue
  
          batch.append(self.insts[idx])
  
          if len(batch) == batch_size:
              inputs = [inst.input_str.strip() for inst in batch]
              inputs_center = [
                  inst.input_center_str.strip() for inst in batch
              ]
              targets = [inst.output_str.strip() for inst in batch]
  
              # Here we assume that every input_file contains examples from every semiotic class
              classes = [
                  [self.label_ids_semiotic[inst.semiotic_class]]
                  for inst in batch
              ]
              directions = [
                  [constants.DIRECTIONS_TO_ID[inst.direction]]
                  for inst in batch
              ]
  
              batch = self.tokenizer(inputs, padding=True)
              batch["input_center"] = self.tokenizer(
                  inputs_center, padding=True
              )["input_ids"]
              batch["direction"] = directions
              batch["semiotic_class_id"] = classes
  
              labels = self.tokenizer(targets, padding=True)[
                  "input_ids"
              ]
              batch["decoder_input_ids"] = np.insert(
                  [x[:-1] for x in labels],
                  0,
                  self.tokenizer.pad_token_id,
                  axis=-1,
              )
  
              # use LABEL_PAD_TOKEN_ID to disregard padded values for the loss calculations
              batch["labels"] = [
                  [
                      x if x != 0 else constants.LABEL_PAD_TOKEN_ID
                      for x in l
                  ]
                  for l in labels
              ]
              batches.append(batch)
              batch = []
  
      logging.info(f"long_examples_filtered: {long_examples_filtered}")
      self.batches = batches
  
imports: import io
  import os
  import pickle
  import random
  from collections import OrderedDict
  from typing import List, Optional, Tuple
  import braceexpand
  import numpy
  import torch
  import webdataset
  from torch.utils.data import IterableDataset
  from tqdm import tqdm
  from transformers import PreTrainedTokenizerBase
  from nemo.collections.common.tokenizers.moses_tokenizers import MosesProcessor
  from nemo.collections.nlp.data.text_normalization import constants
  from nemo.collections.nlp.data.text_normalization.utils import read_data_file
  from nemo.core.classes import Dataset
  from nemo.utils import logging
label_desc: Use image node_id 6 for calling the TextNormalizationDecoderDataset obj's underlying member method code with example usage: obj.batchify(batch_size) without return types