class_docstr: Constructs an EmbeddingTokenizer.
      
class_docstr_tok: ["Constructs","an","EmbeddingTokenizer","."]
docstr: :param vocab_file: Path to a one-word-per-line vocabulary file
  :type vocab_file: str
  :param do_lower_case: Flag whether to lower case the input
  :type do_lower_case: bool
docstr_tok: [":","param","vocab_file",":","Path","to","a","one-word-per-line","vocabulary","file",":","type","vocab_file",":","str",":","param","do_lower_case",":","Flag","whether","to","lower","case","the","input",":","type","do_lower_case",":","bool"]
code: def __init__(
      self,
      vocab_file,
      do_lower_case=True,
      unk_token="[UNK]",
      sep_token="[SEP]",
      pad_token="[PAD]",
      cls_token="[CLS]",
      mask_token="[MASK]",
      **kwargs,
  ):
      """
      :param vocab_file: Path to a one-word-per-line vocabulary file
      :type vocab_file: str
      :param do_lower_case: Flag whether to lower case the input
      :type do_lower_case: bool
      """
      # TODO check why EmbeddingTokenizer.tokenize gives many UNK, while tokenize_with_metadata() works fine
  
      super().__init__(
          unk_token=unk_token,
          sep_token=sep_token,
          pad_token=pad_token,
          cls_token=cls_token,
          mask_token=mask_token,
          **kwargs,
      )
  
      if not os.path.isfile(vocab_file):
          raise ValueError(
              "Can't find a vocabulary file at path '{}'.".format(
                  vocab_file
              )
          )
      self.vocab = load_vocab(vocab_file)
      self.unk_tok_idx = self.vocab[unk_token]
      self.ids_to_tokens = collections.OrderedDict(
          [(ids, tok) for tok, ids in self.vocab.items()]
      )
      self.do_lower_case = do_lower_case
  
imports: from __future__ import absolute_import, division, print_function, unicode_literals
  import collections
  import json
  import logging
  import os
  import re
  from pathlib import Path
  import numpy
  from transformers import AlbertTokenizer, AlbertTokenizerFast, BertTokenizer, BertTokenizerFast, DistilBertTokenizer, DistilBertTokenizerFast, ElectraTokenizer, ElectraTokenizerFast, RobertaTokenizer, RobertaTokenizerFast, XLMRobertaTokenizer, XLMRobertaTokenizerFast, XLNetTokenizer, XLNetTokenizerFast, CamembertTokenizer, CamembertTokenizerFast, DPRContextEncoderTokenizer, DPRContextEncoderTokenizerFast, DPRQuestionEncoderTokenizer, DPRQuestionEncoderTokenizerFast, BigBirdTokenizer, BigBirdTokenizerFast
  from transformers.models.bert.tokenization_bert import load_vocab
  from transformers.tokenization_utils import PreTrainedTokenizer
  from transformers import AutoConfig
  from farm.data_handler.samples import SampleBasket
  from farm.modeling.wordembedding_utils import load_from_cache, EMBEDDING_VOCAB_FILES_MAP, run_split_on_punc
label_desc: Use image node_id 1 to create a new EmbeddingTokenizer object from inherited base classes: PreTrainedTokenizer with example: obj = EmbeddingTokenizer(vocab_file, do_lower_case, unk_token, sep_token, pad_token, cls_token, mask_token)