class_docstr: This is the training version of the dataset.
class_docstr_tok: ["This","is","the","training","version","of","the","dataset","."]
code: def __init__(
      self,
      args,
      input_files={"txt": None, "label": None},
      input_text=None,
      vocab=None,
      evaluation=False,
      dictionary=None,
  ):
      super().__init__(
          args, input_files, input_text, vocab, evaluation, dictionary
      )
  
      self.vocab = vocab if vocab is not None else self.init_vocab()
  
      # data comes in a list of paragraphs, where each paragraph is a list of units with unit-level labels.
      # At evaluation time, each paragraph is treated as single "sentence" as we don't know a priori where
      # sentence breaks occur. We make prediction from left to right for each paragraph and move forward to
      # the last predicted sentence break to start afresh.
      self.sentences = [
          self.para_to_sentences(para) for para in self.data
      ]
  
      self.init_sent_ids()
      logger.debug(f"{len(self.sentence_ids)} sentences loaded.")
  
imports: from bisect import bisect_right
  from copy import copy
  import numpy
  import random
  import logging
  import re
  import torch
  from torch.utils.data import Dataset
  from .vocab import Vocab
  from stanza.models.common.utils import sort_with_indices, unsort
label_desc: Use image node_id 1 to create a new DataLoader object from inherited base classes: TokenizationDataset with example: obj = DataLoader(args, input_files, input_text, vocab, evaluation, dictionary)