



'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


UpperCamelCase__			:      Tuple        =					logging.get_logger(__name__)

UpperCamelCase__			:      Tuple        =					{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase__			:      Dict        =					{
    '''vocab_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-german-cased''': '''https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt''',
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-german-cased''': (
            '''https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCamelCase__			:      str        =					{
    '''distilbert-base-uncased''': 5_12,
    '''distilbert-base-uncased-distilled-squad''': 5_12,
    '''distilbert-base-cased''': 5_12,
    '''distilbert-base-cased-distilled-squad''': 5_12,
    '''distilbert-base-german-cased''': 5_12,
    '''distilbert-base-multilingual-cased''': 5_12,
}


UpperCamelCase__			:      Any        =					{
    '''distilbert-base-uncased''': {'''do_lower_case''': True},
    '''distilbert-base-uncased-distilled-squad''': {'''do_lower_case''': True},
    '''distilbert-base-cased''': {'''do_lower_case''': False},
    '''distilbert-base-cased-distilled-squad''': {'''do_lower_case''': False},
    '''distilbert-base-german-cased''': {'''do_lower_case''': False},
    '''distilbert-base-multilingual-cased''': {'''do_lower_case''': False},
}






class 		_UpperCamelCase    (				__a       ):


			'''simple docstring'''

			_A :						Tuple            =						VOCAB_FILES_NAMES
			_A :						Optional[Any]            =						PRETRAINED_VOCAB_FILES_MAP
			_A :						List[Any]            =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_A :						int            =						PRETRAINED_INIT_CONFIGURATION
			_A :						List[Any]            =						['''input_ids''', '''attention_mask''']
			_A :						int            =						DistilBertTokenizer

			def __init__(				self	:					int				,	lowerCAmelCase__	:					Optional[Any]=None				,	lowerCAmelCase__	:					Union[str, Any]=None				,	lowerCAmelCase__	:					Optional[int]=True				,	lowerCAmelCase__	:					List[str]="[UNK]"				,	lowerCAmelCase__	:					int="[SEP]"				,	lowerCAmelCase__	:					Optional[int]="[PAD]"				,	lowerCAmelCase__	:					List[Any]="[CLS]"				,	lowerCAmelCase__	:					List[str]="[MASK]"				,	lowerCAmelCase__	:					Optional[int]=True				,	lowerCAmelCase__	:					Any=None				,	**lowerCAmelCase__	:					str				,	):





					"""simple docstring"""






					super().__init__(
					    _lowercase				,	tokenizer_file=_lowercase				,	do_lower_case=_lowercase				,	unk_token=_lowercase				,	sep_token=_lowercase				,	pad_token=_lowercase				,	cls_token=_lowercase				,	mask_token=_lowercase				,	tokenize_chinese_chars=_lowercase				,	strip_accents=_lowercase				,	**_lowercase				,	)

					__SCREAMING_SNAKE_CASE			:						Optional[int]       			=						json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
					if (
					    normalizer_state.get("""lowercase"""				,	_lowercase		) != do_lower_case
					    or normalizer_state.get("""strip_accents"""				,	_lowercase		) != strip_accents
					    or normalizer_state.get("""handle_chinese_chars"""				,	_lowercase		) != tokenize_chinese_chars
					):
							__SCREAMING_SNAKE_CASE			:						str       			=						getattr(_lowercase				,	normalizer_state.pop("""type"""		)		)
							__SCREAMING_SNAKE_CASE			:						Union[str, Any]       			=						do_lower_case
							__SCREAMING_SNAKE_CASE			:						int       			=						strip_accents
							__SCREAMING_SNAKE_CASE			:						Union[str, Any]       			=						tokenize_chinese_chars
							__SCREAMING_SNAKE_CASE			:						Optional[int]       			=						normalizer_class(**_lowercase		)

					__SCREAMING_SNAKE_CASE			:						Dict       			=						do_lower_case

			def UpperCamelCase__   (				self	:					Tuple				,	lowerCAmelCase__	:					Optional[int]				,	lowerCAmelCase__	:					Any=None		):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE			:						Dict       			=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

					return output

			def UpperCamelCase__   (				self	:					List[str]				,	lowerCAmelCase__	:					List[int]				,	lowerCAmelCase__	:					Optional[List[int]] = None		):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE			:						Dict       			=						[self.sep_token_id]
					__SCREAMING_SNAKE_CASE			:						Optional[int]       			=						[self.cls_token_id]
					if token_ids_a is None:
							return len(cls + token_ids_a + sep		) * [0]
					return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]



			def UpperCamelCase__   (				self	:					Any				,	lowerCAmelCase__	:					str				,	lowerCAmelCase__	:					Optional[str] = None		):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE			:						Any       			=						self._tokenizer.model.save(_lowercase				,	name=_lowercase		)
					return tuple(_lowercase		)
UpperCAmelCase_           =				'''
# Transformers installation
! pip install transformers datasets
# To install from source instead of the last release, comment the command above and uncomment the following one.
# ! pip install git+https://github.com/huggingface/transformers.git
'''

UpperCAmelCase_           =				[{'''type''': '''code''', '''content''': INSTALL_CONTENT}]
UpperCAmelCase_           =				{
    '''{processor_class}''': '''FakeProcessorClass''',
    '''{model_class}''': '''FakeModelClass''',
    '''{object_class}''': '''FakeObjectClass''',
}