class_docstr: Simple Wrapper for Tokenizers from the transformers package. Enables loading of different Tokenizer classes with a uniform interface.
class_docstr_tok: ["Simple","Wrapper","for","Tokenizers","from","the","transformers","package",".","Enables","loading","of","different","Tokenizer","classes","with","a","uniform","interface","."]
docstr: Enables loading of different Tokenizer classes with a uniform interface. Either infer the class from
  model config or define it manually via `tokenizer_class`.
  
  :param pretrained_model_name_or_path:  The path of the saved pretrained model or its name (e.g. `bert-base-uncased`)
  :type pretrained_model_name_or_path: str
  :param revision: The version of model to use from the HuggingFace model hub. Can be tag name, branch name, or commit hash.
  :type revision: str
  :param tokenizer_class: (Optional) Name of the tokenizer class to load (e.g. `BertTokenizer`)
  :type tokenizer_class: str
  :param use_fast: (Optional, False by default) Indicate if FARM should try to load the fast version of the tokenizer (True) or
      use the Python one (False).
      Only DistilBERT, BERT and Electra fast tokenizers are supported.
  :type use_fast: bool
  :param kwargs:
  :return: Tokenizer
docstr_tok: ["Enables","loading","of","different","Tokenizer","classes","with","a","uniform","interface",".","Either","infer","the","class","from","model","config","or","define","it","manually","via","`","tokenizer_class","`",".",":","param","pretrained_model_name_or_path",":","The","path","of","the","saved","pretrained","model","or","its","name","(","e.g",".","`","bert-base-uncased","`",")",":","type","pretrained_model_name_or_path",":","str",":","param","revision",":","The","version","of","model","to","use","from","the","HuggingFace","model","hub",".","Can","be","tag","name",",","branch","name",",","or","commit","hash",".",":","type","revision",":","str",":","param","tokenizer_class",":","(","Optional",")","Name","of","the","tokenizer","class","to","load","(","e.g",".","`","BertTokenizer","`",")",":","type","tokenizer_class",":","str",":","param","use_fast",":","(","Optional",",","False","by","default",")","Indicate","if","FARM","should","try","to","load","the","fast","version","of","the","tokenizer","(","True",")","or","use","the","Python","one","(","False",")",".","Only","DistilBERT",",","BERT","and","Electra","fast","tokenizers","are","supported",".",":","type","use_fast",":","bool",":","param","kwargs",":",":","return",":","Tokenizer"]
code: def load(
      cls,
      pretrained_model_name_or_path,
      revision=None,
      tokenizer_class=None,
      use_fast=True,
      **kwargs,
  ):
      """
      Enables loading of different Tokenizer classes with a uniform interface. Either infer the class from
      model config or define it manually via `tokenizer_class`.
  
      :param pretrained_model_name_or_path:  The path of the saved pretrained model or its name (e.g. `bert-base-uncased`)
      :type pretrained_model_name_or_path: str
      :param revision: The version of model to use from the HuggingFace model hub. Can be tag name, branch name, or commit hash.
      :type revision: str
      :param tokenizer_class: (Optional) Name of the tokenizer class to load (e.g. `BertTokenizer`)
      :type tokenizer_class: str
      :param use_fast: (Optional, False by default) Indicate if FARM should try to load the fast version of the tokenizer (True) or
          use the Python one (False).
          Only DistilBERT, BERT and Electra fast tokenizers are supported.
      :type use_fast: bool
      :param kwargs:
      :return: Tokenizer
      """
      pretrained_model_name_or_path = str(pretrained_model_name_or_path)
      kwargs["revision"] = revision
  
      if tokenizer_class is None:
          tokenizer_class = cls._infer_tokenizer_class(
              pretrained_model_name_or_path
          )
  
      logger.info(f"Loading tokenizer of type '{tokenizer_class}'")
      # return appropriate tokenizer object
      ret = None
      if "AlbertTokenizer" in tokenizer_class:
          if use_fast:
              ret = AlbertTokenizerFast.from_pretrained(
                  pretrained_model_name_or_path,
                  keep_accents=True,
                  **kwargs,
              )
          else:
              ret = AlbertTokenizer.from_pretrained(
                  pretrained_model_name_or_path,
                  keep_accents=True,
                  **kwargs,
              )
      elif "XLMRobertaTokenizer" in tokenizer_class:
          if use_fast:
              ret = XLMRobertaTokenizerFast.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
          else:
              ret = XLMRobertaTokenizer.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
      elif "RobertaTokenizer" in tokenizer_class:
          if use_fast:
              ret = RobertaTokenizerFast.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
          else:
              ret = RobertaTokenizer.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
      elif "DistilBertTokenizer" in tokenizer_class:
          if use_fast:
              ret = DistilBertTokenizerFast.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
          else:
              ret = DistilBertTokenizer.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
      elif "BertTokenizer" in tokenizer_class:
          if use_fast:
              ret = BertTokenizerFast.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
          else:
              ret = BertTokenizer.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
      elif "XLNetTokenizer" in tokenizer_class:
          if use_fast:
              ret = XLNetTokenizerFast.from_pretrained(
                  pretrained_model_name_or_path,
                  keep_accents=True,
                  **kwargs,
              )
          else:
              ret = XLNetTokenizer.from_pretrained(
                  pretrained_model_name_or_path,
                  keep_accents=True,
                  **kwargs,
              )
      elif "ElectraTokenizer" in tokenizer_class:
          if use_fast:
              ret = ElectraTokenizerFast.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
          else:
              ret = ElectraTokenizer.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
      elif tokenizer_class == "EmbeddingTokenizer":
          if use_fast:
              logger.error(
                  "EmbeddingTokenizerFast is not supported! Using EmbeddingTokenizer instead."
              )
              ret = EmbeddingTokenizer.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
          else:
              ret = EmbeddingTokenizer.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
      elif "CamembertTokenizer" in tokenizer_class:
          if use_fast:
              ret = CamembertTokenizerFast.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
          else:
              ret = CamembertTokenizer.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
      elif "DPRQuestionEncoderTokenizer" in tokenizer_class:
          if use_fast:
              ret = DPRQuestionEncoderTokenizerFast.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
          else:
              ret = DPRQuestionEncoderTokenizer.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
      elif "DPRContextEncoderTokenizer" in tokenizer_class:
          if use_fast:
              ret = DPRContextEncoderTokenizerFast.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
          else:
              ret = DPRContextEncoderTokenizer.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
      elif "BigBirdTokenizer" in tokenizer_class:
          if use_fast:
              ret = BigBirdTokenizerFast.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
          else:
              ret = BigBirdTokenizer.from_pretrained(
                  pretrained_model_name_or_path, **kwargs
              )
      if ret is None:
          raise Exception("Unable to load tokenizer")
      else:
          return ret
  
imports: from __future__ import absolute_import, division, print_function, unicode_literals
  import collections
  import json
  import logging
  import os
  import re
  from pathlib import Path
  import numpy
  from transformers import AlbertTokenizer, AlbertTokenizerFast, BertTokenizer, BertTokenizerFast, DistilBertTokenizer, DistilBertTokenizerFast, ElectraTokenizer, ElectraTokenizerFast, RobertaTokenizer, RobertaTokenizerFast, XLMRobertaTokenizer, XLMRobertaTokenizerFast, XLNetTokenizer, XLNetTokenizerFast, CamembertTokenizer, CamembertTokenizerFast, DPRContextEncoderTokenizer, DPRContextEncoderTokenizerFast, DPRQuestionEncoderTokenizer, DPRQuestionEncoderTokenizerFast, BigBirdTokenizer, BigBirdTokenizerFast
  from transformers.models.bert.tokenization_bert import load_vocab
  from transformers.tokenization_utils import PreTrainedTokenizer
  from transformers import AutoConfig
  from farm.data_handler.samples import SampleBasket
  from farm.modeling.wordembedding_utils import load_from_cache, EMBEDDING_VOCAB_FILES_MAP, run_split_on_punc
label_desc: Use image node_id 1 for calling the Tokenizer obj's underlying member method code with example usage: obj.load(cls, pretrained_model_name_or_path, revision, tokenizer_class, use_fast) and returns: ret