class_docstr: Class to split documents into chunks of max_seq_len symbols so that the length will not exceed
  maximal sequence length to feed into BERT
class_docstr_tok: ["Class","to","split","documents","into","chunks","of","max_seq_len","symbols","so","that","the","length","will","not","exceed","maximal","sequence","length","to","feed","into","BERT"]
docstr: Args:
      vocab_file: vocab file of pretrained transformer model
      max_seq_len: maximal length of chunks into which the document is split
      lowercase: whether to lowercase text
      batch_size: how many chunks are in batch
docstr_tok: ["Args",":","vocab_file",":","vocab","file","of","pretrained","transformer","model","max_seq_len",":","maximal","length","of","chunks","into","which","the","document","is","split","lowercase",":","whether","to","lowercase","text","batch_size",":","how","many","chunks","are","in","batch"]
code: def __init__(
      self,
      vocab_file: str,
      max_seq_len: int = 400,
      lowercase: bool = False,
      batch_size: int = 2,
      **kwargs,
  ):
      """
      Args:
          vocab_file: vocab file of pretrained transformer model
          max_seq_len: maximal length of chunks into which the document is split
          lowercase: whether to lowercase text
          batch_size: how many chunks are in batch
      """
      self.max_seq_len = max_seq_len
      self.batch_size = batch_size
      self.re_tokenizer = re.compile(r"[\w']+|[^\w ]")
      self.tokenizer = AutoTokenizer.from_pretrained(
          vocab_file, do_lower_case=True
      )
      self.punct_ext = (
          punctuation
          + " "
          + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
      )
      self.russian_letters = "абвгдеёжзийклмнопрстуфхцчшщъыьэюя"
      self.lowercase = lowercase
  
file_path: DeepPavlov/deeppavlov/models/entity_extraction/ner_chunker.py
imports: import re
  from logging import getLogger
  from string import punctuation
  from typing import List, Tuple, Union, Any
  from nltk import sent_tokenize
  from transformers import AutoTokenizer
  from deeppavlov.core.common.registry import register
  from deeppavlov.core.models.component import Component
  from deeppavlov.core.common.chainer import Chainer
  from deeppavlov.models.entity_extraction.entity_detection_parser import EntityDetectionParser
label_desc: Use image node_id 1 to create a new NerChunker object from inherited base classes: Component with example: obj = NerChunker(vocab_file, max_seq_len, lowercase, batch_size)