docstr: Tokenize and create masks; recalculate the entity positions regarding the document boarders.
  Args:
      tokens: List of tokens of each document: List[List[tokens in doc]]
      entity_pos: start and end positions of the entities' mentions
      entity_tags: NER tag of the entities
  Return:
      input_ids: List[List[int]],
      attention_mask: List[List[int]],
      entity_poss: List[
                      List[
                          List[(entity1_mention1_start_id, entity1_mention1_end_id), ...],
                          List[(entity2_mention1_start_id, entity2_mention1_end_id), ...]
                      ]
                  ]
      entity_tags: List[List[int]]
      nf_samples: List[int] - contains the information about whether the corresponding sample is real sample or
          fake (for testing): 0 means the sample is real, 1 - it is fake.
docstr_tok: ["Tokenize","and","create","masks",";","recalculate","the","entity","positions","regarding","the","document","boarders",".","Args",":","tokens",":","List","of","tokens","of","each","document",":","List","[","List","[","tokens","in","doc","]","]","entity_pos",":","start","and","end","positions","of","the","entities","'","mentions","entity_tags",":","NER","tag","of","the","entities","Return",":","input_ids",":","List","[","List","[","int","]","]",",","attention_mask",":","List","[","List","[","int","]","]",",","entity_poss",":","List","[","List","[","List","[","(","entity1_mention1_start_id",",","entity1_mention1_end_id",")",",","...","]",",","List","[","(","entity2_mention1_start_id",",","entity2_mention1_end_id",")",",","...","]","]","]","entity_tags",":","List","[","List","[","int","]","]","nf_samples",":","List","[","int","]","-","contains","the","information","about","whether","the","corresponding","sample","is","real","sample","or","fake","(","for","testing",")",":","0","means","the","sample","is","real",",","1","-","it","is","fake","."]
code: def __call__(
      self,
      tokens: Union[Tuple, List[List[str]]],
      entity_pos: Union[Tuple, List[List[Tuple]]],
      entity_tags: Union[Tuple, List[List[str]]],
  ) -> Tuple[List, List, List, List, List]:
      """
      Tokenize and create masks; recalculate the entity positions regarding the document boarders.
      Args:
          tokens: List of tokens of each document: List[List[tokens in doc]]
          entity_pos: start and end positions of the entities' mentions
          entity_tags: NER tag of the entities
      Return:
          input_ids: List[List[int]],
          attention_mask: List[List[int]],
          entity_poss: List[
                          List[
                              List[(entity1_mention1_start_id, entity1_mention1_end_id), ...],
                              List[(entity2_mention1_start_id, entity2_mention1_end_id), ...]
                          ]
                      ]
          entity_tags: List[List[int]]
          nf_samples: List[int] - contains the information about whether the corresponding sample is real sample or
              fake (for testing): 0 means the sample is real, 1 - it is fake.
      """
  
      _ = self.tokenizer.add_special_tokens(self.special_tokens_dict)
  
      (
          input_ids,
          attention_mask,
          upd_entity_pos,
          upd_entity_tags,
          nf_samples,
      ) = ([], [], [], [], [])
  
      # this workaround is for proper testing: for an unknown reason during test in test_quick_start.py
      # each input list is transformed into a tuple, e.g., tokens -> tuple(tokens, ).
      # todo: refactoring
      if (
          type(tokens) == tuple
          and type(entity_pos) == tuple
          and type(entity_tags) == tuple
      ):
          tokens = tokens[0]
          entity_pos = entity_pos[0]
          entity_tags = entity_tags[0]
  
      for n_sample, (doc, ent_pos, ent_tags) in enumerate(
          zip(tokens, entity_pos, entity_tags)
      ):
          # valid scenario
          if isinstance(ent_pos, list) and len(ent_pos) == 2:
              count = 0
              doc_wordpiece_tokens = []
  
              entity1_pos_start = list(zip(*ent_pos[0]))[
                  0
              ]  # first entity mentions' start positions
              entity1_pos_end = list(zip(*ent_pos[0]))[
                  1
              ]  # first entity mentions' end positions
              entity2_pos_start = list(zip(*ent_pos[1]))[
                  0
              ]  # second entity mentions' start positions
              entity2_pos_end = list(zip(*ent_pos[1]))[
                  1
              ]  # second entity mentions' end positions
  
              (
                  upd_entity1_pos_start,
                  upd_entity2_pos_start,
                  upd_entity1_pos_end,
                  upd_entity2_pos_end,
              ) = ([], [], [], [])
              for n, token in enumerate(doc):
                  if n in entity1_pos_start:
                      doc_wordpiece_tokens.append(self.special_token)
                      upd_entity1_pos_start.append(count)
                      count += 1
  
                  if n in entity1_pos_end:
                      doc_wordpiece_tokens.append(self.special_token)
                      count += 1
                      upd_entity1_pos_end.append(count)
  
                  if n in entity2_pos_start:
                      doc_wordpiece_tokens.append(self.special_token)
                      upd_entity2_pos_start.append(count)
                      count += 1
  
                  if n in entity2_pos_end:
                      doc_wordpiece_tokens.append(self.special_token)
                      count += 1
                      upd_entity2_pos_end.append(count)
  
                  word_tokens = self.tokenizer.tokenize(token)
                  doc_wordpiece_tokens += word_tokens
                  count += len(word_tokens)
  
              # special case when the entity is the last in the doc
              if len(doc) in entity1_pos_end:
                  doc_wordpiece_tokens.append(self.special_token)
                  count += 1
                  upd_entity1_pos_end.append(count)
              if len(doc) in entity2_pos_end:
                  doc_wordpiece_tokens.append(self.special_token)
                  count += 1
                  upd_entity2_pos_end.append(count)
                  word_tokens = self.tokenizer.tokenize(token)
                  doc_wordpiece_tokens += word_tokens
                  count += len(word_tokens)
  
              upd_entity_1_pos = list(
                  zip(upd_entity1_pos_start, upd_entity1_pos_end)
              )
              upd_entity_2_pos = list(
                  zip(upd_entity2_pos_start, upd_entity2_pos_end)
              )
  
              # text entities for self check
              upd_entity1_text = [
                  doc_wordpiece_tokens[ent_m[0] : ent_m[1]]
                  for ent_m in upd_entity_1_pos
              ]
              upd_entity2_text = [
                  doc_wordpiece_tokens[ent_m[0] : ent_m[1]]
                  for ent_m in upd_entity_2_pos
              ]
  
              enc_entity_tags = self.encode_ner_tag(ent_tags)
  
              encoding = self.tokenizer.encode_plus(
                  doc_wordpiece_tokens[
                      : self.max_seq_length
                  ],  # truncate tokens
                  add_special_tokens=True,
                  truncation=True,
                  max_length=self.max_seq_length,
                  pad_to_max_length=True,
                  return_attention_mask=True,
              )
              upd_entity_pos.append(
                  [upd_entity_1_pos, upd_entity_2_pos]
              )
              nf_samples.append(0)
  
          # api test scenario
          else:
              # for api test: dump values of entity tags and entity pos
              encoding = self.tokenizer.encode_plus(
                  doc,
                  add_special_tokens=True,
                  truncation=True,
                  max_length=self.max_seq_length,
                  pad_to_max_length=True,
                  return_attention_mask=True,
              )
              upd_entity_pos.append([[(0, 1)], [(0, 1)]])
              enc_entity_tags = self.encode_ner_tag(
                  [self.default_tag] * 2
              )
              nf_samples.append(1)
  
          input_ids.append(encoding["input_ids"])
          attention_mask.append(encoding["attention_mask"])
          upd_entity_tags.append(enc_entity_tags)
  
      return (
          input_ids,
          attention_mask,
          upd_entity_pos,
          upd_entity_tags,
          nf_samples,
      )
  
file_path: DeepPavlov/deeppavlov/models/preprocessors/re_preprocessor.py
imports: from logging import getLogger
  from pathlib import Path
  from typing import Tuple, List, Union
  import numpy
  from transformers import BertTokenizer
  from deeppavlov.core.commands.utils import expand_path
  from deeppavlov.core.common.file import read_json
  from deeppavlov.core.common.registry import register
  from deeppavlov.core.models.component import Component
label_desc: Use image node_id 2 for calling the REPreprocessor obj's underlying member method code with example usage: obj.__call__(tokens, entity_pos, entity_tags) and returns: input_ids,  attention_mask,  upd_entity_pos,  upd_entity_tags,  nf_samples