docstr:         The tokenizer cannot use UDProcessor's sentence-level cross-document batching interface, and requires special handling.
          Essentially, this method concatenates the text of multiple documents with "
  
  ", tokenizes it with the neural tokenizer,
          then splits the result into the original Documents and recovers the original character offsets.
          
docstr_tok: ["The","tokenizer","can","not","use","UDProcessor","'s","sentence-level","cross-document","batching","interface",",","and","requires","special","handling",".","Essentially",",","this","method","concatenates","the","text","of","multiple","documents","with","``","''",",","tokenizes","it","with","the","neural","tokenizer",",","then","splits","the","result","into","the","original","Documents","and","recovers","the","original","character","offsets","."]
code: def bulk_process(self, docs):
      """
      The tokenizer cannot use UDProcessor's sentence-level cross-document batching interface, and requires special handling.
      Essentially, this method concatenates the text of multiple documents with "\n\n", tokenizes it with the neural tokenizer,
      then splits the result into the original Documents and recovers the original character offsets.
      """
      if hasattr(self, "_variant"):
          return self._variant.bulk_process(docs)
  
      if self.config.get("pretokenized"):
          res = []
          for document in docs:
              raw_text, document = self.process_pre_tokenized_text(
                  document.text
              )
              res.append(doc.Document(document, raw_text))
          return res
  
      combined_text = "\n\n".join([thisdoc.text for thisdoc in docs])
      processed_combined = self.process(
          doc.Document([], text=combined_text)
      )
  
      # postprocess sentences and tokens to reset back pointers and char offsets
      charoffset = 0
      sentst = senten = 0
      for thisdoc in docs:
          while senten < len(
              processed_combined.sentences
          ) and processed_combined.sentences[senten].tokens[
              -1
          ].end_char - charoffset <= len(
              thisdoc.text
          ):
              senten += 1
  
          sentences = processed_combined.sentences[sentst:senten]
          thisdoc.sentences = sentences
          for sent in sentences:
              # fix doc back pointers for sentences
              sent._doc = thisdoc
  
              # fix char offsets for tokens and words
              for token in sent.tokens:
                  token._start_char -= charoffset
                  token._end_char -= charoffset
                  if (
                      token.words
                  ):  # not-yet-processed MWT can leave empty tokens
                      for word in token.words:
                          word._start_char -= charoffset
                          word._end_char -= charoffset
  
          thisdoc.num_tokens = sum(
              len(sent.tokens) for sent in sentences
          )
          thisdoc.num_words = sum(len(sent.words) for sent in sentences)
          sentst = senten
  
          charoffset += len(thisdoc.text) + 2
  
      return docs
  
imports: import io
  import logging
  import torch
  from stanza.models.tokenization.data import TokenizationDataset
  from stanza.models.tokenization.trainer import Trainer
  from stanza.models.tokenization.utils import output_predictions
  from stanza.pipeline._constants import *
  from stanza.pipeline.processor import UDProcessor, register_processor
  from stanza.pipeline.registry import PROCESSOR_VARIANTS
  from stanza.models.common import doc
  from stanza.pipeline.external.jieba import JiebaTokenizer
  from stanza.pipeline.external.spacy import SpacyTokenizer
  from stanza.pipeline.external.sudachipy import SudachiPyTokenizer
  from stanza.pipeline.external.pythainlp import PyThaiNLPTokenizer
label_desc: Use image node_id 4 for calling the TokenizeProcessor obj's underlying member method code with example usage: obj.bulk_process(docs) and returns: docs, self, res