


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


A			:						Dict        =  logging.get_logger(__name__)

A			:						Union[str, Any]        =  {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

A			:						int        =  {
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

A			:						Any        =  {
    'distilbert-base-uncased': 5_1_2,
    'distilbert-base-uncased-distilled-squad': 5_1_2,
    'distilbert-base-cased': 5_1_2,
    'distilbert-base-cased-distilled-squad': 5_1_2,
    'distilbert-base-german-cased': 5_1_2,
    'distilbert-base-multilingual-cased': 5_1_2,
}


A			:						Optional[int]        =  {
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}



class 				A       (			UpperCAmelCase__				):
					'''simple docstring'''






					A__     =					VOCAB_FILES_NAMES
					A__     =					PRETRAINED_VOCAB_FILES_MAP
					A__     =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A__     =					PRETRAINED_INIT_CONFIGURATION
					A__     =					['''input_ids''', '''attention_mask''']
					A__     =					DistilBertTokenizer

					def __init__(self       :		List[Any] ,   _UpperCAmelCase       :		Optional[int]=None ,   _UpperCAmelCase       :		Optional[Any]=None ,   _UpperCAmelCase       :		Optional[Any]=True ,   _UpperCAmelCase       :		List[Any]="[UNK]" ,   _UpperCAmelCase       :		List[Any]="[SEP]" ,   _UpperCAmelCase       :		Any="[PAD]" ,   _UpperCAmelCase       :		List[Any]="[CLS]" ,   _UpperCAmelCase       :		str="[MASK]" ,   _UpperCAmelCase       :		int=True ,   _UpperCAmelCase       :		Optional[Any]=None ,   **_UpperCAmelCase       :		str ,   )		->  int:




							"""simple docstring"""


							super().__init__(
							    _UpperCAmelCase ,   tokenizer_file=_UpperCAmelCase ,   do_lower_case=_UpperCAmelCase ,   unk_token=_UpperCAmelCase ,   sep_token=_UpperCAmelCase ,   pad_token=_UpperCAmelCase ,   cls_token=_UpperCAmelCase ,   mask_token=_UpperCAmelCase ,   tokenize_chinese_chars=_UpperCAmelCase ,   strip_accents=_UpperCAmelCase ,   **_UpperCAmelCase ,   )

							lowercase__	     =						json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
							if (
							    normalizer_state.get("""lowercase""" ,   _UpperCAmelCase						) != do_lower_case
							    or normalizer_state.get("""strip_accents""" ,   _UpperCAmelCase						) != strip_accents
							    or normalizer_state.get("""handle_chinese_chars""" ,   _UpperCAmelCase						) != tokenize_chinese_chars
							):
									lowercase__	     =						getattr(_UpperCAmelCase ,   normalizer_state.pop("""type"""						)						)
									lowercase__	     =						do_lower_case
									lowercase__	     =						strip_accents
									lowercase__	     =						tokenize_chinese_chars
									lowercase__	     =						normalizer_class(**_UpperCAmelCase						)

							lowercase__	     =						do_lower_case

					def     lowerCamelCase__	(self       :		Optional[Any] ,   _UpperCAmelCase       :		int ,   _UpperCAmelCase       :		Dict=None						)		->  Any:




							"""simple docstring"""


							lowercase__	     =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

							return output

					def     lowerCamelCase__	(self       :		List[Any] ,   _UpperCAmelCase       :		List[int] ,   _UpperCAmelCase       :		Optional[List[int]] = None						)		->  List[int]:




							"""simple docstring"""


							lowercase__	     =						[self.sep_token_id]
							lowercase__	     =						[self.cls_token_id]
							if token_ids_a is None:
									return len(cls + token_ids_a + sep						) * [0]
							return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]


					def     lowerCamelCase__	(self       :		Dict ,   _UpperCAmelCase       :		str ,   _UpperCAmelCase       :		Optional[str] = None						)		->  Tuple[str]:




							"""simple docstring"""


							lowercase__	     =						self._tokenizer.model.save(_UpperCAmelCase ,   name=_UpperCAmelCase						)
							return tuple(_UpperCAmelCase						)





from __future__ import annotations






def 					UpperCamelCase   (      __magic_name__	:				list[int]	)  ->    list[int]:  # This function is recursive
		"""simple docstring"""




		lowercase__	     =						len(__magic_name__	)
		# If the array contains only one element, we return it (it's the stop condition of
		# recursion)
		if array_length <= 1:
				return array
				# Else
		lowercase__	     =						array[0]
		lowercase__	     =						False
		lowercase__	     =						1
		lowercase__	     =						[]
		while not is_found and i < array_length:
				if array[i] < pivot:
						lowercase__	     =						True
						lowercase__	     =						[element for element in array[i:] if element >= array[i]]
						lowercase__	     =						longest_subsequence(__magic_name__	)
						if len(__magic_name__	) > len(__magic_name__	):
								lowercase__	     =						temp_array
				else:
						i += 1

		lowercase__	     =						[element for element in array[1:] if element >= pivot]
		lowercase__	     =						[pivot, *longest_subsequence(__magic_name__	)]
		if len(__magic_name__	) > len(__magic_name__	):
				return temp_array
		else:
				return longest_subseq


if __name__ == "__main__":
	import doctest

	doctest.testmod()

