docstr: Pretokenized text can be provided in 2 manners:
  
  1.) str, tokenized by whitespace, sentence split by newline
  2.) list of token lists, each token list represents a sentence
  
  generate dictionary data structure
docstr_tok: ["Pretokenized","text","can","be","provided","in","2","manners",":","1",".",")","str",",","tokenized","by","whitespace",",","sentence","split","by","newline","2",".",")","list","of","token","lists",",","each","token","list","represents","a","sentence","generate","dictionary","data","structure"]
code: def process_pre_tokenized_text(self, input_src):
      """
      Pretokenized text can be provided in 2 manners:
  
      1.) str, tokenized by whitespace, sentence split by newline
      2.) list of token lists, each token list represents a sentence
  
      generate dictionary data structure
      """
  
      document = []
      if isinstance(input_src, str):
          sentences = [
              sent.strip().split()
              for sent in input_src.strip().split("\n")
              if len(sent.strip()) > 0
          ]
      elif isinstance(input_src, list):
          sentences = input_src
      idx = 0
      for sentence in sentences:
          sent = []
          for token_id, token in enumerate(sentence):
              sent.append(
                  {
                      doc.ID: (token_id + 1,),
                      doc.TEXT: token,
                      doc.MISC: f"start_char={idx}|end_char={idx + len(token)}",
                  }
              )
              idx += len(token) + 1
          document.append(sent)
      raw_text = " ".join(
          [" ".join(sentence) for sentence in sentences]
      )
      return raw_text, document
  
imports: import io
  import logging
  import torch
  from stanza.models.tokenization.data import TokenizationDataset
  from stanza.models.tokenization.trainer import Trainer
  from stanza.models.tokenization.utils import output_predictions
  from stanza.pipeline._constants import *
  from stanza.pipeline.processor import UDProcessor, register_processor
  from stanza.pipeline.registry import PROCESSOR_VARIANTS
  from stanza.models.common import doc
  from stanza.pipeline.external.jieba import JiebaTokenizer
  from stanza.pipeline.external.spacy import SpacyTokenizer
  from stanza.pipeline.external.sudachipy import SudachiPyTokenizer
  from stanza.pipeline.external.pythainlp import PyThaiNLPTokenizer
label_desc: Use image node_id 2 for calling the TokenizeProcessor obj's underlying member method code with example usage: obj.process_pre_tokenized_text(input_src) and returns: raw_text,  document