docstr: Args:
      vocab_file: path to vocabulary / name of vocabulary for tokenizer initialization
      special_token: an additional token that will be used for marking the entities in the document
      do_lower_case: set True if lowercasing is needed
      default_tag: used for test purposes to create a valid input
  Return:
      list of feature batches with input_ids, attention_mask, entity_pos, ner_tags
docstr_tok: ["Args",":","vocab_file",":","path","to","vocabulary","\/","name","of","vocabulary","for","tokenizer","initialization","special_token",":","an","additional","token","that","will","be","used","for","marking","the","entities","in","the","document","do_lower_case",":","set","True","if","lowercasing","is","needed","default_tag",":","used","for","test","purposes","to","create","a","valid","input","Return",":","list","of","feature","batches","with","input_ids",",","attention_mask",",","entity_pos",",","ner_tags"]
code: def __init__(
      self,
      vocab_file: str,
      special_token: str = "<ENT>",
      ner_tags=None,
      max_seq_length: int = 512,
      do_lower_case: bool = False,
      default_tag: str = None,
      **kwargs,
  ):
      """
      Args:
          vocab_file: path to vocabulary / name of vocabulary for tokenizer initialization
          special_token: an additional token that will be used for marking the entities in the document
          do_lower_case: set True if lowercasing is needed
          default_tag: used for test purposes to create a valid input
      Return:
          list of feature batches with input_ids, attention_mask, entity_pos, ner_tags
      """
  
      self.special_token = special_token
      self.special_tokens_dict = {
          "additional_special_tokens": [self.special_token]
      }
      self.default_tag = default_tag
  
      if ner_tags is None:
          ner_tags = ["ORG", "TIME", "MISC", "LOC", "PER", "NUM"]
      self.ner2id = {tag: tag_id for tag_id, tag in enumerate(ner_tags)}
      self.max_seq_length = max_seq_length
  
      if Path(vocab_file).is_file():
          vocab_file = str(expand_path(vocab_file))
          self.tokenizer = BertTokenizer(
              vocab_file=vocab_file, do_lower_case=do_lower_case
          )
      else:
          self.tokenizer = BertTokenizer.from_pretrained(
              vocab_file, do_lower_case=do_lower_case
          )
  
file_path: DeepPavlov/deeppavlov/models/preprocessors/re_preprocessor.py
imports: from logging import getLogger
  from pathlib import Path
  from typing import Tuple, List, Union
  import numpy
  from transformers import BertTokenizer
  from deeppavlov.core.commands.utils import expand_path
  from deeppavlov.core.common.file import read_json
  from deeppavlov.core.common.registry import register
  from deeppavlov.core.models.component import Component
label_desc: Use image node_id 1 to create a new REPreprocessor object from inherited base classes: Component with example: obj = REPreprocessor(vocab_file, special_token, ner_tags, max_seq_length, do_lower_case, default_tag)