class_docstr: Class to split documents into chunks of max_seq_len symbols so that the length will not exceed
  maximal sequence length to feed into BERT
class_docstr_tok: ["Class","to","split","documents","into","chunks","of","max_seq_len","symbols","so","that","the","length","will","not","exceed","maximal","sequence","length","to","feed","into","BERT"]
docstr: This method splits each document in the batch into chunks wuth the maximal length of max_seq_len
  
  Args:
      docs_batch: batch of documents
  Returns:
      batch of lists of document chunks for each document
      batch of lists of numbers of documents which correspond to chunks
docstr_tok: ["This","method","splits","each","document","in","the","batch","into","chunks","wuth","the","maximal","length","of","max_seq_len","Args",":","docs_batch",":","batch","of","documents","Returns",":","batch","of","lists","of","document","chunks","for","each","document","batch","of","lists","of","numbers","of","documents","which","correspond","to","chunks"]
code: def __call__(
      self, docs_batch: List[str]
  ) -> Tuple[
      List[List[str]],
      List[List[int]],
      List[
          List[
              Union[
                  List[
                      Union[
                          Tuple[int, int],
                          Tuple[Union[int, Any], Union[int, Any]],
                      ]
                  ],
                  List[Tuple[Union[int, Any], Union[int, Any]]],
                  List[Tuple[int, int]],
              ]
          ]
      ],
      List[List[Union[List[Any], List[str]]]],
      List[List[str]],
  ]:
      """
      This method splits each document in the batch into chunks wuth the maximal length of max_seq_len
  
      Args:
          docs_batch: batch of documents
      Returns:
          batch of lists of document chunks for each document
          batch of lists of numbers of documents which correspond to chunks
      """
      (
          text_batch_list,
          nums_batch_list,
          sentences_offsets_batch_list,
          sentences_batch_list,
      ) = ([], [], [], [])
      (
          text_batch,
          nums_batch,
          sentences_offsets_batch,
          sentences_batch,
      ) = ([], [], [], [])
      for n, doc in enumerate(docs_batch):
          if self.lowercase:
              doc = doc.lower()
          start = 0
          text = ""
          sentences_list = []
          sentences_offsets_list = []
          cur_len = 0
          doc_pieces = doc.split("\n")
          doc_pieces = [
              self.sanitize(doc_piece) for doc_piece in doc_pieces
          ]
          doc_pieces = [
              doc_piece
              for doc_piece in doc_pieces
              if len(doc_piece) > 1
          ]
          if doc_pieces:
              sentences = []
              for doc_piece in doc_pieces:
                  sentences += sent_tokenize(doc_piece)
              for sentence in sentences:
                  sentence_tokens = re.findall(
                      self.re_tokenizer, sentence
                  )
                  sentence_len = sum(
                      [
                          len(
                              self.tokenizer.encode_plus(
                                  token, add_special_tokens=False
                              )["input_ids"]
                          )
                          for token in sentence_tokens
                      ]
                  )
                  if cur_len + sentence_len < self.max_seq_len:
                      text += f"{sentence} "
                      cur_len += sentence_len
                      end = start + len(sentence)
                      sentences_offsets_list.append((start, end))
                      sentences_list.append(sentence)
                      start = end + 1
                  else:
                      text = text.strip()
                      if text:
                          text_batch.append(text)
                          sentences_offsets_batch.append(
                              sentences_offsets_list
                          )
                          sentences_batch.append(sentences_list)
                          nums_batch.append(n)
  
                      if sentence_len < self.max_seq_len:
                          text = f"{sentence} "
                          cur_len = sentence_len
                          start = 0
                          end = start + len(sentence)
                          sentences_offsets_list = [(start, end)]
                          sentences_list = [sentence]
                          start = end + 1
                      else:
                          text = ""
                          sentence_chunks = sentence.split(" ")
                          for chunk in sentence_chunks:
                              chunk_tokens = re.findall(
                                  self.re_tokenizer, chunk
                              )
                              chunk_len = sum(
                                  [
                                      len(
                                          self.tokenizer.encode_plus(
                                              token,
                                              add_special_tokens=False,
                                          )["input_ids"]
                                      )
                                      for token in chunk_tokens
                                  ]
                              )
                              if cur_len + chunk_len < self.max_seq_len:
                                  text += f"{chunk} "
                                  cur_len += chunk_len + 1
                                  end = start + len(chunk)
                                  sentences_offsets_list.append(
                                      (start, end)
                                  )
                                  sentences_list.append(chunk)
                                  start = end + 1
                              else:
                                  text = text.strip()
                                  if text:
                                      text_batch.append(text)
                                      sentences_offsets_batch.append(
                                          sentences_offsets_list
                                      )
                                      sentences_batch.append(
                                          sentences_list
                                      )
                                      nums_batch.append(n)
  
                                  text = f"{chunk} "
                                  cur_len = chunk_len
                                  start = 0
                                  end = start + len(chunk)
                                  sentences_offsets_list = [
                                      (start, end)
                                  ]
                                  sentences_list = [chunk]
                                  start = end + 1
  
              text = text.strip().strip(",")
              if text:
                  text_batch.append(text)
                  nums_batch.append(n)
                  sentences_offsets_batch.append(sentences_offsets_list)
                  sentences_batch.append(sentences_list)
          else:
              text_batch.append("а")
              nums_batch.append(n)
              sentences_offsets_batch.append([(0, len(doc))])
              sentences_batch.append([doc])
  
      num_batches = len(text_batch) // self.batch_size + int(
          len(text_batch) % self.batch_size > 0
      )
      for jj in range(num_batches):
          text_batch_list.append(
              text_batch[
                  jj * self.batch_size : (jj + 1) * self.batch_size
              ]
          )
          nums_batch_list.append(
              nums_batch[
                  jj * self.batch_size : (jj + 1) * self.batch_size
              ]
          )
          sentences_offsets_batch_list.append(
              sentences_offsets_batch[
                  jj * self.batch_size : (jj + 1) * self.batch_size
              ]
          )
          sentences_batch_list.append(
              sentences_batch[
                  jj * self.batch_size : (jj + 1) * self.batch_size
              ]
          )
  
      return (
          text_batch_list,
          nums_batch_list,
          sentences_offsets_batch_list,
          sentences_batch_list,
      )
  
file_path: DeepPavlov/deeppavlov/models/entity_extraction/ner_chunker.py
imports: import re
  from logging import getLogger
  from string import punctuation
  from typing import List, Tuple, Union, Any
  from nltk import sent_tokenize
  from transformers import AutoTokenizer
  from deeppavlov.core.common.registry import register
  from deeppavlov.core.models.component import Component
  from deeppavlov.core.common.chainer import Chainer
  from deeppavlov.models.entity_extraction.entity_detection_parser import EntityDetectionParser
label_desc: Use image node_id 2 for calling the NerChunker obj's underlying member method code with example usage: obj.__call__(docs_batch) and returns: text_batch_list,  nums_batch_list,  sentences_offsets_batch_list,  sentences_batch_list