docstr: If config.max_document_chars is set, documents are "pre chunked" into "sub
  documents" prior to creation of the input pipeline. This test asserts that
  the splitting and rejoining behaves as expected.
docstr_tok: ["If","config.max_document_chars","is","set",",","documents","are","``","pre","chunked","''","into","``","sub","documents","''","prior","to","creation","of","the","input","pipeline",".","This","test","asserts","that","the","splitting","and","rejoining","behaves","as","expected","."]
code: def test_pre_chunking(self):
      """
      If config.max_document_chars is set, documents are "pre chunked" into "sub
      documents" prior to creation of the input pipeline. This test asserts that
      the splitting and rejoining behaves as expected.
      """
      max_doc_len = 250
  
      # Create a mix of short and long sequences for prediction
      test_sequence = "I am a dog. A dog that's incredibly bright. I can talk, read, and write! "
      test_sequences = [
          test_sequence,
          test_sequence * 10,
          test_sequence,
          test_sequence * 10,
      ]
  
      # Use animal test data to train model
      path = os.path.join(
          os.path.dirname(__file__), "data", "testdata.json"
      )
      with open(path, "rt") as fp:
          text, labels = json.load(fp)
      self.model.finetune(text * 10, labels * 10)
  
      # Split inputs into "sub documents"
      self.model.config.max_document_chars = max_doc_len
      split_sequences, split_indices = self.model._pre_chunk_document(
          test_sequences
      )
  
      # Predict on the newly split sequences. We unset config.max_document chars
      # prior to calling predict so that we don't repeat the pre chunking that we
      # did explicitly above. This has the potential to be brittle if the functionality
      # of predict in sequence_labeler.py changes
      self.model.config.max_document_chars = None
      split_preds = self.model.predict(split_sequences)
  
      # Rejoin predictions using split_indices
      self.model.config.max_document_chars = max_doc_len
      preds = self.model._merge_chunked_preds(
          deepcopy(split_preds), split_indices
      )
  
      # Assert that documents were split properly
      assert split_indices == [[0], [1, 2, 3], [4], [5, 6, 7]]
      assert len(split_preds) == 8
      assert len(preds) == 4
  
      # Assert that the offset is calculated correctly for start/end indices after
      # merging predictions by specifically looking at the last "sub document" of
      # second (index 1) document, which was split into three "sub documents"
      sub_split_preds = split_preds[3]
      sub_preds = [
          pred for pred in preds[1] if pred["start"] >= max_doc_len * 2
      ]
      for split_pred, pred in zip(sub_split_preds, sub_preds):
          assert split_pred["start"] + max_doc_len * 2 == pred["start"]
          assert split_pred["end"] + max_doc_len * 2 == pred["end"]
  
imports: import os
  import unittest
  import logging
  from copy import copy, deepcopy
  from pathlib import Path
  import codecs
  import json
  import random
  import time
  import weakref
  import gc
  import pytest
  from pytest import approx
  import tensorflow
  import numpy
  from sklearn.metrics import accuracy_score
  from sklearn.model_selection import train_test_split
  import requests
  from bs4 import BeautifulSoup
  from bs4.element import Tag
  from finetune import SequenceLabeler
  from finetune.base_models import GPT
  from finetune.config import get_config
  from finetune.encoding.sequence_encoder import finetune_to_indico_sequence
  from finetune.util.metrics import sequence_labeling_token_precision, sequence_labeling_token_recall, sequence_labeling_overlap_precision, sequence_labeling_overlap_recall
label_desc: Use image node_id 17 for calling the TestSequenceLabeler obj's underlying member method code with example usage: obj.test_pre_chunking() without return types