






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__A					=	logging.get_logger(__name__)

__A					=	{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__A					=	{
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

__A					=	{
    "distilbert-base-uncased": 5_12,
    "distilbert-base-uncased-distilled-squad": 5_12,
    "distilbert-base-cased": 5_12,
    "distilbert-base-cased-distilled-squad": 5_12,
    "distilbert-base-german-cased": 5_12,
    "distilbert-base-multilingual-cased": 5_12,
}


__A					=	{
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}



class 				_A			(     UpperCamelCase				):
    """simple docstring"""

    lowerCamelCase	:							List[str]				= VOCAB_FILES_NAMES
    lowerCamelCase	:							List[str]				= PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase	:							int				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase	:							str				= PRETRAINED_INIT_CONFIGURATION
    lowerCamelCase	:							Optional[int]				= ['input_ids', 'attention_mask']
    lowerCamelCase	:							Optional[int]				= DistilBertTokenizer



    def __init__(				self      :     int ,     __SCREAMING_SNAKE_CASE      :     List[Any]=None ,     __SCREAMING_SNAKE_CASE      :     Any=None ,     __SCREAMING_SNAKE_CASE      :     Union[str, Any]=True ,     __SCREAMING_SNAKE_CASE      :     List[Any]="[UNK]" ,     __SCREAMING_SNAKE_CASE      :     Optional[Any]="[SEP]" ,     __SCREAMING_SNAKE_CASE      :     Any="[PAD]" ,     __SCREAMING_SNAKE_CASE      :     Tuple="[CLS]" ,     __SCREAMING_SNAKE_CASE      :     List[Any]="[MASK]" ,     __SCREAMING_SNAKE_CASE      :     Tuple=True ,     __SCREAMING_SNAKE_CASE      :     List[str]=None ,     **__SCREAMING_SNAKE_CASE      :     Any ,     )      ->							str:
           super().__init__(
               __SCREAMING_SNAKE_CASE ,     tokenizer_file=__SCREAMING_SNAKE_CASE ,     do_lower_case=__SCREAMING_SNAKE_CASE ,     unk_token=__SCREAMING_SNAKE_CASE ,     sep_token=__SCREAMING_SNAKE_CASE ,     pad_token=__SCREAMING_SNAKE_CASE ,     cls_token=__SCREAMING_SNAKE_CASE ,     mask_token=__SCREAMING_SNAKE_CASE ,     tokenize_chinese_chars=__SCREAMING_SNAKE_CASE ,     strip_accents=__SCREAMING_SNAKE_CASE ,     **__SCREAMING_SNAKE_CASE ,     )

           __UpperCAmelCase     				=json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
           if (
               normalizer_state.get("""lowercase""" ,     __SCREAMING_SNAKE_CASE      ) != do_lower_case
               or normalizer_state.get("""strip_accents""" ,     __SCREAMING_SNAKE_CASE      ) != strip_accents
               or normalizer_state.get("""handle_chinese_chars""" ,     __SCREAMING_SNAKE_CASE      ) != tokenize_chinese_chars
           ):
                  __UpperCAmelCase     				=getattr(__SCREAMING_SNAKE_CASE ,     normalizer_state.pop("""type"""      )      )
                  __UpperCAmelCase     				=do_lower_case
                  __UpperCAmelCase     				=strip_accents
                  __UpperCAmelCase     				=tokenize_chinese_chars
                  __UpperCAmelCase     				=normalizer_class(**__SCREAMING_SNAKE_CASE      )

           __UpperCAmelCase     				=do_lower_case



    def _a							(				self      :     Union[str, Any] ,     __SCREAMING_SNAKE_CASE      :     Tuple ,     __SCREAMING_SNAKE_CASE      :     Any=None      )      ->							Optional[Any]:
           __UpperCAmelCase     				=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]

           return output



    def _a							(				self      :     Tuple ,     __SCREAMING_SNAKE_CASE      :     List[int] ,     __SCREAMING_SNAKE_CASE      :     Optional[List[int]] = None      )      ->							List[int]:
           __UpperCAmelCase     				=[self.sep_token_id]
           __UpperCAmelCase     				=[self.cls_token_id]
           if token_ids_a is None:
                  return len(cls + token_ids_a + sep      ) * [0]
           return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]



    def _a							(				self      :     Optional[int] ,     __SCREAMING_SNAKE_CASE      :     str ,     __SCREAMING_SNAKE_CASE      :     Optional[str] = None      )      ->							Tuple[str]:
           __UpperCAmelCase     				=self._tokenizer.model.save(__SCREAMING_SNAKE_CASE ,     name=__SCREAMING_SNAKE_CASE      )
           return tuple(__SCREAMING_SNAKE_CASE      )







import string


def 							__lowerCAmelCase     (							_UpperCamelCase		:	str						)		->				str:
 '''simple docstring'''


 SCREAMING_SNAKE_CASE		    =      ''
 for i in sequence:
  SCREAMING_SNAKE_CASE		    =      ord(_UpperCamelCase						)
  if 65 <= extract <= 90:
   output += chr(1_55 - extract						)
  elif 97 <= extract <= 1_22:
   output += chr(2_19 - extract						)
  else:
   output += i
 return output


def 							__lowerCAmelCase     (							_UpperCamelCase		:	str						)		->				str:
 '''simple docstring'''


 SCREAMING_SNAKE_CASE		    =      string.ascii_letters
 SCREAMING_SNAKE_CASE		    =      string.ascii_lowercase[::-1] + string.ascii_uppercase[::-1]
 return "".join(
     letters_reversed[letters.index(_UpperCamelCase						)] if c in letters else c for c in sequence						)


def 							__lowerCAmelCase     (							)		->				None:
 '''simple docstring'''


 from timeit import timeit

 print('Running performance benchmarks...'						)
 SCREAMING_SNAKE_CASE		    =      'from string import printable ; from __main__ import atbash, atbash_slow'
 print(f"""> atbash_slow(): {timeit('atbash_slow(printable)'		,				setup=_UpperCamelCase						)} seconds"""						)
 print(f""">      atbash(): {timeit('atbash(printable)'		,				setup=_UpperCamelCase						)} seconds"""						)


if __name__ == "__main__":
      for example in ("ABCDEFGH", "123GGjj", "testStringtest", "with space"):
            print(F"""{example} encrypted in atbash: {atbash(example)}""")
      benchmark()

