


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


a_ :Tuple  							=			logging.get_logger(__name__)

a_ :int  							=			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

a_ :Optional[Any]  							=			{
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

a_ :Union[str, Any]  							=			{
    'distilbert-base-uncased': 5_12,
    'distilbert-base-uncased-distilled-squad': 5_12,
    'distilbert-base-cased': 5_12,
    'distilbert-base-cased-distilled-squad': 5_12,
    'distilbert-base-german-cased': 5_12,
    'distilbert-base-multilingual-cased': 5_12,
}


a_ :Any  							=			{
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}

class 							lowercase  (						_UpperCAmelCase			):
     lowerCamelCase			:			Optional[Any]      = VOCAB_FILES_NAMES
     lowerCamelCase			:			List[str]      = PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase			:			Dict      = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCamelCase			:			Dict      = PRETRAINED_INIT_CONFIGURATION
     lowerCamelCase			:			int      = ['''input_ids''', '''attention_mask''']
     lowerCamelCase			:			Optional[int]      = DistilBertTokenizer







     def __init__(						self :						Union[str, Any]   ,   _lowercase :						List[Any]=None   ,   _lowercase :						int=None   ,   _lowercase :						Optional[int]=True   ,   _lowercase :						Optional[int]="[UNK]"   ,   _lowercase :						Dict="[SEP]"   ,   _lowercase :						Optional[int]="[PAD]"   ,   _lowercase :						List[Any]="[CLS]"   ,   _lowercase :						Union[str, Any]="[MASK]"   ,   _lowercase :						List[Any]=True   ,   _lowercase :						int=None   ,   **_lowercase :						List[Any]   ,   ):
           super().__init__(
               _lowercase   ,   tokenizer_file=_lowercase   ,   do_lower_case=_lowercase   ,   unk_token=_lowercase   ,   sep_token=_lowercase   ,   pad_token=_lowercase   ,   cls_token=_lowercase   ,   mask_token=_lowercase   ,   tokenize_chinese_chars=_lowercase   ,   strip_accents=_lowercase   ,   **_lowercase   ,   )

           SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
           if (
               normalizer_state.get('''lowercase'''   ,   _lowercase      ) != do_lower_case
               or normalizer_state.get('''strip_accents'''   ,   _lowercase      ) != strip_accents
               or normalizer_state.get('''handle_chinese_chars'''   ,   _lowercase      ) != tokenize_chinese_chars
           ):
                 SCREAMING_SNAKE_CASE__       :  Optional[Any]						     =		getattr(_lowercase   ,   normalizer_state.pop('''type'''      )      )
                 SCREAMING_SNAKE_CASE__       :  Tuple						     =		do_lower_case
                 SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		strip_accents
                 SCREAMING_SNAKE_CASE__       :  Dict						     =		tokenize_chinese_chars
                 SCREAMING_SNAKE_CASE__       :  List[Any]						     =		normalizer_class(**_lowercase      )

           SCREAMING_SNAKE_CASE__       :  List[str]						     =		do_lower_case







     def 							lowercase__       (						self :						List[str]   ,   _lowercase :						List[str]   ,   _lowercase :						Union[str, Any]=None      ):
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                 output += token_ids_a + [self.sep_token_id]

           return output







     def 							lowercase__       (						self :						Union[str, Any]   ,   _lowercase :						List[int]   ,   _lowercase :						Optional[List[int]] = None      ):
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		[self.sep_token_id]
           SCREAMING_SNAKE_CASE__       :  int						     =		[self.cls_token_id]
           if token_ids_a is None:
                 return len(cls + token_ids_a + sep      ) * [0]
           return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]







     def 							lowercase__       (						self :						Union[str, Any]   ,   _lowercase :						str   ,   _lowercase :						Optional[str] = None      ):
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		self._tokenizer.model.save(_lowercase   ,   name=_lowercase      )
           return tuple(_lowercase      )














"""simple docstring"""


from math import loga


def        UpperCAmelCase      (					a__ ):





						'''simple docstring'''






						if a < 0:
												raise ValueError('Input value must be a positive integer' )
						elif isinstance(a__     ,      a__ ):
												raise TypeError('Input value must be a \'int\' type' )
						return 0 if (a == 0) else int(loga(a & -a ) )


if __name__ == "__main__":
		import doctest

		doctest.testmod()