class_docstr: Labels each token in a sequence as belonging to 1 of N token classes.
  
  :param config: A :py:class:`finetune.config.Settings` object or None (for default config).
  :param \**kwargs: key-value pairs of config items to override.
class_docstr_tok: ["Labels","each","token","in","a","sequence","as","belonging","to","1","of","N","token","classes",".",":","param","config",":","A",":","py",":","class",":","`","finetune.config.Settings","`","object","or","None","(","for","default","config",")",".",":","param","\\","*","*","kwargs",":","key-value","pairs","of","config","items","to","override","."]
docstr: If self.config.max_document_chars is set, "pre-chunk" any documents that
  are longer than that into multiple "sub documents" to more easily process
  large documents through prediction.
  
  Args:
      texts: Text of each document
  
  Returns:
      new_texts: Text of each document after pre chunking
      split_indices: Indices of documents that were split by pre chunking
docstr_tok: ["If","self.config.max_document_chars","is","set",",","``","pre-chunk","''","any","documents","that","are","longer","than","that","into","multiple","``","sub","documents","''","to","more","easily","process","large","documents","through","prediction",".","Args",":","texts",":","Text","of","each","document","Returns",":","new_texts",":","Text","of","each","document","after","pre","chunking","split_indices",":","Indices","of","documents","that","were","split","by","pre","chunking"]
code: def _pre_chunk_document(
      self, texts: List[str]
  ) -> Tuple[List[str], List[List[int]]]:
      """
      If self.config.max_document_chars is set, "pre-chunk" any documents that
      are longer than that into multiple "sub documents" to more easily process
      large documents through prediction.
  
      Args:
          texts: Text of each document
  
      Returns:
          new_texts: Text of each document after pre chunking
          split_indices: Indices of documents that were split by pre chunking
      """
      max_doc_len = self.config.max_document_chars
      new_texts = []
      split_indices = []
      offset = 0
      # Split documents into "sub documents" if any are too long
      for doc_idx, doc in enumerate(texts):
          if len(doc) > max_doc_len:
              num_splits = math.ceil(len(doc) / max_doc_len)
              for split_idx in range(num_splits):
                  new_texts.append(
                      doc[
                          split_idx
                          * max_doc_len : (split_idx + 1)
                          * max_doc_len
                      ]
                  )
              split_indices.append(
                  list(
                      range(
                          doc_idx + offset,
                          doc_idx + offset + num_splits,
                      )
                  )
              )
              offset += num_splits - 1
          else:
              new_texts.append(doc)
              split_indices.append([doc_idx + offset])
  
      return new_texts, split_indices
  
imports: import copy
  import os
  from collections import Counter, defaultdict
  import math
  from typing import Dict, List, Tuple, Union
  import tensorflow
  import numpy
  from finetune.base import BaseModel
  from finetune.encoding.target_encoders import SequenceLabelingEncoder, SequenceMultiLabelingEncoder
  from finetune.nn.target_blocks import sequence_labeler
  from finetune.nn.crf import sequence_decode
  from finetune.encoding.sequence_encoder import finetune_to_indico_sequence
  from finetune.encoding.input_encoder import get_spacy
  from finetune.input_pipeline import BasePipeline
  from finetune.util.metrics import sequences_overlap
  from finetune.encoding.input_encoder import tokenize_context
label_desc: Use image node_id 5 for calling the SequenceLabeler obj's underlying member method code with example usage: obj._pre_chunk_document(texts) and returns: new_texts,  split_indices