code: def __init__(
      self,
      tokenizer_args,
      input_files={"txt": None, "label": None},
      input_text=None,
      vocab=None,
      evaluation=False,
      dictionary=None,
      *args,
      **kwargs,
  ):
      super().__init__(*args, **kwargs)  # forwards all unused arguments
      self.args = tokenizer_args
      self.eval = evaluation
      self.dictionary = dictionary
      self.vocab = vocab
  
      # get input files
      txt_file = input_files["txt"]
      label_file = input_files["label"]
  
      # Load data and process it
      # set up text from file or input string
      assert txt_file is not None or input_text is not None
      if input_text is None:
          with open(txt_file) as f:
              text = "".join(f.readlines()).rstrip()
      else:
          text = input_text
  
      text_chunks = NEWLINE_WHITESPACE_RE.split(text)
      text_chunks = [pt.rstrip() for pt in text_chunks]
      text_chunks = [pt for pt in text_chunks if pt]
      if label_file is not None:
          with open(label_file) as f:
              labels = "".join(f.readlines()).rstrip()
              labels = NEWLINE_WHITESPACE_RE.split(labels)
              labels = [pt.rstrip() for pt in labels]
              labels = [map(int, pt) for pt in labels if pt]
      else:
          labels = [[0 for _ in pt] for pt in text_chunks]
  
      skip_newline = self.args.get("skip_newline", False)
      self.data = [
          [
              (
                  WHITESPACE_RE.sub(" ", char),
                  label,
              )  # substitute special whitespaces
              for char, label in zip(pt, pc)
              if not (skip_newline and char == "\n")
          ]  # check if newline needs to be eaten
          for pt, pc in zip(text_chunks, labels)
      ]
  
      # remove consecutive whitespaces
      self.data = [filter_consecutive_whitespaces(x) for x in self.data]
  
imports: from bisect import bisect_right
  from copy import copy
  import numpy
  import random
  import logging
  import re
  import torch
  from torch.utils.data import Dataset
  from .vocab import Vocab
  from stanza.models.common.utils import sort_with_indices, unsort
label_desc: Use image node_id 1 to create a new TokenizationDataset object  with example: obj = TokenizationDataset(tokenizer_args, input_files, input_text, vocab, evaluation, dictionary)