class_docstr: Construct a Transformer-XL tokenizer adapted from Vocab class in [the original
  code](https://github.com/kimiyoung/transformer-xl). The Transformer-XL tokenizer is a word-level tokenizer (no
  sub-word tokenization).
  
  This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
  this superclass for more information regarding those methods.
  
  Args:
      special (`List[str]`, *optional*):
          A list of special tokens (to be treated by the original implementation of this tokenizer).
      min_freq (`int`, *optional*, defaults to 0):
          The minimum number of times a token has to be present in order to be kept in the vocabulary (otherwise it
          will be mapped to `unk_token`).
      max_size (`int`, *optional*):
          The maximum size of the vocabulary. If left unset, it will default to the size of the vocabulary found
          after excluding the tokens according to the `min_freq` rule.
      lower_case (`bool`, *optional*, defaults to `False`):
          Whether or not to lowercase the input when tokenizing.
      delimiter (`str`, *optional*):
          The delimiter used between tokens.
      vocab_file (`str`, *optional*):
          File containing the vocabulary (from the original implementation).
      pretrained_vocab_file (`str`, *optional*):
          File containing the vocabulary as saved with the `save_pretrained()` method.
      never_split (`List[str]`, *optional*):
          List of tokens that should never be split. If no list is specified, will simply use the existing special
          tokens.
      unk_token (`str`, *optional*, defaults to `"<unk>"`):
          The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
          token instead.
      eos_token (`str`, *optional*, defaults to `"<eos>"`):
          The end of sequence token.
      additional_special_tokens (`List[str]`, *optional*, defaults to `["<formula>"]`):
          A list of additional special tokens (for the HuggingFace functionality).
      language (`str`, *optional*, defaults to `"en"`):
          The language of this tokenizer (used for mose preprocessing).
class_docstr_tok: ["Construct","a","Transformer-XL","tokenizer","adapted","from","Vocab","class","in","[","the","original","code","]","(","https",":","\/\/github.com\/kimiyoung\/transformer-xl",")",".","The","Transformer-XL","tokenizer","is","a","word-level","tokenizer","(","no","sub-word","tokenization",")",".","This","tokenizer","inherits","from","[","`","PreTrainedTokenizer","`","]","which","contains","most","of","the","main","methods",".","Users","should","refer","to","this","superclass","for","more","information","regarding","those","methods",".","Args",":","special","(","`","List","[","str","]","`",",","*","optional","*",")",":","A","list","of","special","tokens","(","to","be","treated","by","the","original","implementation","of","this","tokenizer",")",".","min_freq","(","`","int","`",",","*","optional","*",",","defaults","to","0",")",":","The","minimum","number","of","times","a","token","has","to","be","present","in","order","to","be","kept","in","the","vocabulary","(","otherwise","it","will","be","mapped","to","`","unk_token","`",")",".","max_size","(","`","int","`",",","*","optional","*",")",":","The","maximum","size","of","the","vocabulary",".","If","left","unset",",","it","will","default","to","the","size","of","the","vocabulary","found","after","excluding","the","tokens","according","to","the","`","min_freq","`","rule",".","lower_case","(","`","bool","`",",","*","optional","*",",","defaults","to","`","False","`",")",":","Whether","or","not","to","lowercase","the","input","when","tokenizing",".","delimiter","(","`","str","`",",","*","optional","*",")",":","The","delimiter","used","between","tokens",".","vocab_file","(","`","str","`",",","*","optional","*",")",":","File","containing","the","vocabulary","(","from","the","original","implementation",")",".","pretrained_vocab_file","(","`","str","`",",","*","optional","*",")",":","File","containing","the","vocabulary","as","saved","with","the","`","save_pretrained","(",")","`","method",".","never_split","(","`","List","[","str","]","`",",","*","optional","*",")",":","List","of","tokens","that","should","never","be","split",".","If","no","list","is","specified",",","will","simply","use","the","existing","special","tokens",".","unk_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","unk",">","''","`",")",":","The","unknown","token",".","A","token","that","is","not","in","the","vocabulary","can","not","be","converted","to","an","ID","and","is","set","to","be","this","token","instead",".","eos_token","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","<","eos",">","''","`",")",":","The","end","of","sequence","token",".","additional_special_tokens","(","`","List","[","str","]","`",",","*","optional","*",",","defaults","to","`","[","``","<","formula",">","''","]","`",")",":","A","list","of","additional","special","tokens","(","for","the","HuggingFace","functionality",")",".","language","(","`","str","`",",","*","optional","*",",","defaults","to","`","``","en","''","`",")",":","The","language","of","this","tokenizer","(","used","for","mose","preprocessing",")","."]
code: def __init__(
      self,
      special=None,
      min_freq=0,
      max_size=None,
      lower_case=False,
      delimiter=None,
      vocab_file=None,
      pretrained_vocab_file: str = None,
      never_split=None,
      unk_token="<unk>",
      eos_token="<eos>",
      additional_special_tokens=["<formula>"],
      language="en",
      **kwargs,
  ):
      super().__init__(
          special=special,
          min_freq=min_freq,
          max_size=max_size,
          lower_case=lower_case,
          delimiter=delimiter,
          vocab_file=vocab_file,
          pretrained_vocab_file=pretrained_vocab_file,
          never_split=never_split,
          unk_token=unk_token,
          eos_token=eos_token,
          additional_special_tokens=additional_special_tokens,
          language=language,
          **kwargs,
      )
      requires_backends(self, "sacremoses")
  
      if never_split is None:
          never_split = self.all_special_tokens
      if special is None:
          special = []
      self.counter = Counter()
      self.special = special
      self.min_freq = min_freq
      self.max_size = max_size
      self.lower_case = lower_case
      self.delimiter = delimiter
      self.vocab_file = vocab_file
      self.never_split = never_split
      self.punctuation_symbols = '!"#$%&()*+,-./\\:;<=>?@[\\]^_`{|}~'
      self.punction_without_space_before_pattern = re.compile(
          rf"[^\s][{self.punctuation_symbols}]"
      )
      self.punctuation_with_space_around_pattern = (
          self._compile_space_around_punctuation_pattern()
      )
      self.language = language
      self.moses_punct_normalizer = sm.MosesPunctNormalizer(language)
      self.moses_tokenizer = sm.MosesTokenizer(language)
      self.moses_detokenizer = sm.MosesDetokenizer(language)
  
      # This try... catch... is not beautiful but honestly this tokenizer was not made to be used
      # in a library like ours, at all.
      try:
          vocab_dict = None
          if pretrained_vocab_file is not None:
              # Priority on pickle files (support PyTorch and TF)
              with open(pretrained_vocab_file, "rb") as f:
                  vocab_dict = pickle.load(f)
  
              # Loading a torch-saved transfo-xl vocab dict with pickle results in an integer
              # Entering this if statement means that we tried to load a torch-saved file with pickle, and we failed.
              # We therefore load it with torch, if it's available.
              if type(vocab_dict) == int:
                  if not is_torch_available():
                      raise ImportError(
                          "Not trying to load dict with PyTorch as you need to install pytorch to load "
                          "from a PyTorch pretrained vocabulary, "
                          "or activate it with environment variables USE_TORCH=1 and USE_TF=0."
                      )
                  vocab_dict = torch.load(pretrained_vocab_file)
  
          if vocab_dict is not None:
              for key, value in vocab_dict.items():
                  if key not in self.__dict__:
                      self.__dict__[key] = value
          elif vocab_file is not None:
              self.build_vocab()
  
      except Exception as e:
          raise ValueError(
              f"Unable to parse file {pretrained_vocab_file}. Unknown format. "
              "If you tried to load a model saved through TransfoXLTokenizerFast, "
              "please note they are not compatible."
          ) from e
  
      if vocab_file is not None:
          self.build_vocab()
  
imports: import glob
  import os
  import pickle
  import re
  from collections import Counter, OrderedDict
  from typing import List, Optional, Tuple
  import numpy
  from ...tokenization_utils import PreTrainedTokenizer
  from ...utils import cached_file, is_sacremoses_available, is_torch_available, logging, requires_backends, torch_only_method
label_desc: Use image node_id 1 to create a new TransfoXLTokenizer object from inherited base classes: PreTrainedTokenizer with example: obj = TransfoXLTokenizer(special, min_freq, max_size, lower_case, delimiter, vocab_file, pretrained_vocab_file, never_split, unk_token, eos_token, additional_special_tokens, language)