class_docstr: Class for linking of entity substrings in the document to entities in Wikidata
class_docstr_tok: ["Class","for","linking","of","entity","substrings","in","the","document","to","entities","in","Wikidata"]
docstr: Args:
      text_batch_list: list of document chunks
      nums_batch_list: nums of documents
      sentences_offsets_batch_list: indices of start and end symbols of sentences in text
      sentences_batch_list: list of sentences from texts
  Returns:
      doc_entity_substr_batch: entity substrings
      doc_entity_offsets_batch: indices of start and end symbols of entities in text
      doc_tags_batch: entity tags (PER, LOC, ORG)
      doc_sentences_offsets_batch: indices of start and end symbols of sentences in text
      doc_sentences_batch: list of sentences from texts
docstr_tok: ["Args",":","text_batch_list",":","list","of","document","chunks","nums_batch_list",":","nums","of","documents","sentences_offsets_batch_list",":","indices","of","start","and","end","symbols","of","sentences","in","text","sentences_batch_list",":","list","of","sentences","from","texts","Returns",":","doc_entity_substr_batch",":","entity","substrings","doc_entity_offsets_batch",":","indices","of","start","and","end","symbols","of","entities","in","text","doc_tags_batch",":","entity","tags","(","PER",",","LOC",",","ORG",")","doc_sentences_offsets_batch",":","indices","of","start","and","end","symbols","of","sentences","in","text","doc_sentences_batch",":","list","of","sentences","from","texts"]
code: def __call__(
      self,
      text_batch_list: List[List[str]],
      nums_batch_list: List[List[int]],
      sentences_offsets_batch_list: List[List[List[Tuple[int, int]]]],
      sentences_batch_list: List[List[List[str]]],
  ):
      """
      Args:
          text_batch_list: list of document chunks
          nums_batch_list: nums of documents
          sentences_offsets_batch_list: indices of start and end symbols of sentences in text
          sentences_batch_list: list of sentences from texts
      Returns:
          doc_entity_substr_batch: entity substrings
          doc_entity_offsets_batch: indices of start and end symbols of entities in text
          doc_tags_batch: entity tags (PER, LOC, ORG)
          doc_sentences_offsets_batch: indices of start and end symbols of sentences in text
          doc_sentences_batch: list of sentences from texts
      """
      (
          entity_substr_batch_list,
          entity_offsets_batch_list,
          entity_positions_batch_list,
          tags_batch_list,
          entity_probas_batch_list,
          text_len_batch_list,
          text_tokens_len_batch_list,
      ) = ([], [], [], [], [], [], [])
      for text_batch, sentences_offsets_batch, sentences_batch in zip(
          text_batch_list,
          sentences_offsets_batch_list,
          sentences_batch_list,
      ):
          text_batch = [
              text.replace("\xad", " ") for text in text_batch
          ]
  
          (
              ner_tokens_batch,
              ner_tokens_offsets_batch,
              ner_probas_batch,
              probas_batch,
          ) = self.ner(text_batch)
          (
              entity_substr_batch,
              entity_positions_batch,
              entity_probas_batch,
          ) = self.ner_parser(
              ner_tokens_batch, ner_probas_batch, probas_batch
          )
          if self.ner2:
              (
                  ner_tokens_batch2,
                  ner_tokens_offsets_batch2,
                  ner_probas_batch2,
                  probas_batch2,
              ) = self.ner2(text_batch)
              (
                  entity_substr_batch2,
                  entity_positions_batch2,
                  entity_probas_batch2,
              ) = self.ner_parser2(
                  ner_tokens_batch2, ner_probas_batch2, probas_batch2
              )
              (
                  entity_substr_batch,
                  entity_positions_batch,
                  entity_probas_batch,
              ) = self.merge_annotations(
                  entity_substr_batch,
                  entity_positions_batch,
                  entity_probas_batch,
                  entity_substr_batch2,
                  entity_positions_batch2,
                  entity_probas_batch2,
              )
  
          entity_pos_tags_probas_batch = [
              [
                  (
                      entity_substr.lower(),
                      entity_substr_positions,
                      tag,
                      entity_proba,
                  )
                  for tag, entity_substr_list in entity_substr_dict.items()
                  for entity_substr, entity_substr_positions, entity_proba in zip(
                      entity_substr_list,
                      entity_positions_dict[tag],
                      entity_probas_dict[tag],
                  )
              ]
              for entity_substr_dict, entity_positions_dict, entity_probas_dict in zip(
                  entity_substr_batch,
                  entity_positions_batch,
                  entity_probas_batch,
              )
          ]
  
          (
              entity_substr_batch,
              entity_offsets_batch,
              entity_positions_batch,
              tags_batch,
              probas_batch,
          ) = ([], [], [], [], [])
          for entity_pos_tags_probas, ner_tokens_offsets_list in zip(
              entity_pos_tags_probas_batch, ner_tokens_offsets_batch
          ):
              if entity_pos_tags_probas:
                  entity_offsets_list = []
                  (
                      entity_substr_list,
                      entity_positions_list,
                      tags_list,
                      probas_list,
                  ) = zip(*entity_pos_tags_probas)
                  for entity_positions in entity_positions_list:
                      start_offset = ner_tokens_offsets_list[
                          entity_positions[0]
                      ][0]
                      end_offset = ner_tokens_offsets_list[
                          entity_positions[-1]
                      ][1]
                      entity_offsets_list.append(
                          (start_offset, end_offset)
                      )
              else:
                  (
                      entity_substr_list,
                      entity_offsets_list,
                      entity_positions_list,
                  ) = ([], [], [])
                  tags_list, probas_list = [], []
              entity_substr_batch.append(list(entity_substr_list))
              entity_offsets_batch.append(list(entity_offsets_list))
              entity_positions_batch.append(list(entity_positions_list))
              tags_batch.append(list(tags_list))
              probas_batch.append(list(probas_list))
  
          entity_substr_batch_list.append(entity_substr_batch)
          tags_batch_list.append(tags_batch)
          entity_offsets_batch_list.append(entity_offsets_batch)
          entity_positions_batch_list.append(entity_positions_batch)
          entity_probas_batch_list.append(probas_batch)
          text_len_batch_list.append([len(text) for text in text_batch])
          text_tokens_len_batch_list.append(
              [len(ner_tokens) for ner_tokens in ner_tokens_batch]
          )
  
      (
          doc_entity_substr_batch,
          doc_tags_batch,
          doc_entity_offsets_batch,
          doc_probas_batch,
      ) = ([], [], [], [])
      (
          doc_entity_positions_batch,
          doc_sentences_offsets_batch,
          doc_sentences_batch,
      ) = ([], [], [])
      (
          doc_entity_substr,
          doc_tags,
          doc_probas,
          doc_entity_offsets,
          doc_entity_positions,
      ) = ([], [], [], [], [])
      doc_sentences_offsets, doc_sentences = [], []
      cur_doc_num = 0
      text_len_sum = 0
      text_tokens_len_sum = 0
      for (
          entity_substr_batch,
          tags_batch,
          probas_batch,
          entity_offsets_batch,
          entity_positions_batch,
          sentences_offsets_batch,
          sentences_batch,
          text_len_batch,
          text_tokens_len_batch,
          nums_batch,
      ) in zip(
          entity_substr_batch_list,
          tags_batch_list,
          entity_probas_batch_list,
          entity_offsets_batch_list,
          entity_positions_batch_list,
          sentences_offsets_batch_list,
          sentences_batch_list,
          text_len_batch_list,
          text_tokens_len_batch_list,
          nums_batch_list,
      ):
          for (
              entity_substr_list,
              tag_list,
              probas_list,
              entity_offsets_list,
              entity_positions_list,
              sentences_offsets_list,
              sentences_list,
              text_len,
              text_tokens_len,
              doc_num,
          ) in zip(
              entity_substr_batch,
              tags_batch,
              probas_batch,
              entity_offsets_batch,
              entity_positions_batch,
              sentences_offsets_batch,
              sentences_batch,
              text_len_batch,
              text_tokens_len_batch,
              nums_batch,
          ):
              if doc_num == cur_doc_num:
                  doc_entity_substr += entity_substr_list
                  doc_tags += tag_list
                  doc_probas += probas_list
                  doc_entity_offsets += [
                      (
                          start_offset + text_len_sum,
                          end_offset + text_len_sum,
                      )
                      for start_offset, end_offset in entity_offsets_list
                  ]
                  doc_sentences_offsets += [
                      (
                          start_offset + text_len_sum,
                          end_offset + text_len_sum,
                      )
                      for start_offset, end_offset in sentences_offsets_list
                  ]
                  doc_entity_positions += [
                      [pos + text_tokens_len_sum for pos in positions]
                      for positions in entity_positions_list
                  ]
                  doc_sentences += sentences_list
                  text_len_sum += text_len + 1
                  text_tokens_len_sum += text_tokens_len
              else:
                  doc_entity_substr_batch.append(doc_entity_substr)
                  doc_tags_batch.append(doc_tags)
                  doc_probas_batch.append(doc_probas)
                  doc_entity_offsets_batch.append(doc_entity_offsets)
                  doc_entity_positions_batch.append(
                      doc_entity_positions
                  )
                  doc_sentences_offsets_batch.append(
                      doc_sentences_offsets
                  )
                  doc_sentences_batch.append(doc_sentences)
                  doc_entity_substr = entity_substr_list
                  doc_tags = tag_list
                  doc_probas = probas_list
                  doc_entity_offsets = entity_offsets_list
                  doc_sentences_offsets = sentences_offsets_list
                  doc_sentences = sentences_list
                  cur_doc_num = doc_num
                  text_len_sum = text_len + 1
                  text_tokens_len_sum = text_tokens_len
  
      doc_entity_substr_batch.append(doc_entity_substr)
      doc_tags_batch.append(doc_tags)
      doc_probas_batch.append(doc_probas)
      doc_entity_offsets_batch.append(doc_entity_offsets)
      doc_entity_positions_batch.append(doc_entity_positions)
      doc_sentences_offsets_batch.append(doc_sentences_offsets)
      doc_sentences_batch.append(doc_sentences)
  
      return (
          doc_entity_substr_batch,
          doc_entity_offsets_batch,
          doc_entity_positions_batch,
          doc_tags_batch,
          doc_sentences_offsets_batch,
          doc_sentences_batch,
          doc_probas_batch,
      )
  
file_path: DeepPavlov/deeppavlov/models/entity_extraction/ner_chunker.py
imports: import re
  from logging import getLogger
  from string import punctuation
  from typing import List, Tuple, Union, Any
  from nltk import sent_tokenize
  from transformers import AutoTokenizer
  from deeppavlov.core.common.registry import register
  from deeppavlov.core.models.component import Component
  from deeppavlov.core.common.chainer import Chainer
  from deeppavlov.models.entity_extraction.entity_detection_parser import EntityDetectionParser
label_desc: Use image node_id 2 for calling the NerChunkModel obj's underlying member method code with example usage: obj.__call__(text_batch_list, nums_batch_list, sentences_offsets_batch_list, sentences_batch_list) and returns: doc_entity_substr_batch,  doc_entity_offsets_batch,  doc_entity_positions_batch,  doc_tags_batch,  doc_sentences_offsets_batch,  doc_sentences_batch,  doc_probas_batch