code: def process(self, document):
      if not (
          isinstance(document, str)
          or isinstance(document, doc.Document)
          or (
              self.config.get("pretokenized")
              or self.config.get("no_ssplit", False)
          )
      ):
          raise ValueError(
              "If neither 'pretokenized' or 'no_ssplit' option is enabled, the input to the TokenizerProcessor must be a string or a Document object.  Got %s"
              % str(type(document))
          )
  
      if isinstance(document, doc.Document):
          if self.config.get("pretokenized"):
              return document
          document = document.text
  
      if self.config.get("pretokenized"):
          raw_text, document = self.process_pre_tokenized_text(document)
          return doc.Document(document, raw_text)
  
      if hasattr(self, "_variant"):
          return self._variant.process(document)
  
      raw_text = (
          "\n\n".join(document)
          if isinstance(document, list)
          else document
      )
  
      max_seq_len = self.config.get(
          "max_seqlen", TokenizeProcessor.MAX_SEQ_LENGTH_DEFAULT
      )
  
      # set up batches
      batches = TokenizationDataset(
          self.config,
          input_text=raw_text,
          vocab=self.vocab,
          evaluation=True,
          dictionary=self.trainer.dictionary,
      )
      # get dict data
      with torch.no_grad():
          _, _, _, document = output_predictions(
              None,
              self.trainer,
              batches,
              self.vocab,
              None,
              max_seq_len,
              orig_text=raw_text,
              no_ssplit=self.config.get("no_ssplit", False),
              num_workers=self.config.get("num_workers", 0),
              postprocessor=self._postprocessor,
          )
  
      # replace excessively long tokens with <UNK> to avoid downstream GPU memory issues in POS
      for sentence in document:
          for token in sentence:
              if len(token["text"]) > max_seq_len:
                  token["text"] = TOKEN_TOO_LONG_REPLACEMENT
  
      return doc.Document(document, raw_text)
  
imports: import io
  import logging
  import torch
  from stanza.models.tokenization.data import TokenizationDataset
  from stanza.models.tokenization.trainer import Trainer
  from stanza.models.tokenization.utils import output_predictions
  from stanza.pipeline._constants import *
  from stanza.pipeline.processor import UDProcessor, register_processor
  from stanza.pipeline.registry import PROCESSOR_VARIANTS
  from stanza.models.common import doc
  from stanza.pipeline.external.jieba import JiebaTokenizer
  from stanza.pipeline.external.spacy import SpacyTokenizer
  from stanza.pipeline.external.sudachipy import SudachiPyTokenizer
  from stanza.pipeline.external.pythainlp import PyThaiNLPTokenizer
label_desc: Use image node_id 3 for calling the TokenizeProcessor obj's underlying member method code with example usage: obj.process(document) and returns: doc, doc, self, document