docstr: If documents are "pre chunked" due to config.max_document_chars being set,
  predictions are impacted when a document is split in the middle of a non-pad
  token. This test asserts that the impact is minimal.
docstr_tok: ["If","documents","are","``","pre","chunked","''","due","to","config.max_document_chars","being","set",",","predictions","are","impacted","when","a","document","is","split","in","the","middle","of","a","non-pad","token",".","This","test","asserts","that","the","impact","is","minimal","."]
code: def test_max_document_chars(self):
      """
      If documents are "pre chunked" due to config.max_document_chars being set,
      predictions are impacted when a document is split in the middle of a non-pad
      token. This test asserts that the impact is minimal.
      """
      # Use animal test data to train model
      test_sequence = [
          "I am a dog. A dog that's incredibly bright. I can talk, read, and write! "
          * 10
      ]
      path = os.path.join(
          os.path.dirname(__file__), "data", "testdata.json"
      )
      with open(path, "rt") as fp:
          text, labels = json.load(fp)
      self.model.finetune(text * 10, labels * 10)
  
      # Predict without config.max_document_chars
      preds = self.model.predict(test_sequence)
  
      # Predict with config.max_document_chars set to split long sequence
      self.model.config.max_document_chars = 250
      pre_chunk_preds = self.model.predict(test_sequence)
  
      # Ensure that the number of predictions of the span "dog" is almost identical
      # between preds and pre_chunk_preds. The reason these values are not identical
      # is because with pre chunking, "dog" can be split across multiple sub documents
      num_dog_preds = len([p for p in preds[0] if p["text"] == "dog"])
      num_dog_pre_chunk_preds = len(
          [p for p in pre_chunk_preds[0] if p["text"] == "dog"]
      )
      assert num_dog_pre_chunk_preds / num_dog_preds >= 0.95
  
imports: import os
  import unittest
  import logging
  from copy import copy, deepcopy
  from pathlib import Path
  import codecs
  import json
  import random
  import time
  import weakref
  import gc
  import pytest
  from pytest import approx
  import tensorflow
  import numpy
  from sklearn.metrics import accuracy_score
  from sklearn.model_selection import train_test_split
  import requests
  from bs4 import BeautifulSoup
  from bs4.element import Tag
  from finetune import SequenceLabeler
  from finetune.base_models import GPT
  from finetune.config import get_config
  from finetune.encoding.sequence_encoder import finetune_to_indico_sequence
  from finetune.util.metrics import sequence_labeling_token_precision, sequence_labeling_token_recall, sequence_labeling_overlap_precision, sequence_labeling_overlap_recall
label_desc: Use image node_id 19 for calling the TestSequenceLabeler obj's underlying member method code with example usage: obj.test_max_document_chars() without return types