






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


lowercase  :							Optional[Any]			  =    logging.get_logger(__name__)

lowercase  :							List[Any]			  =    {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase  :							int			  =    {
    '''vocab_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/vocab.txt''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/vocab.txt''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'''
        ),
        '''bert-base-multilingual-cased''': '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt''',
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'''
        ),
        '''bert-base-german-dbmdz-cased''': '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt''',
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json''',
        '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json''',
        '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json''',
        '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json''',
        '''bert-base-multilingual-uncased''': (
            '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-multilingual-cased''': (
            '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json''',
        '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json''',
        '''bert-large-uncased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'''
        ),
        '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-large-cased-whole-word-masking-finetuned-squad''': (
            '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'''
        ),
        '''bert-base-cased-finetuned-mrpc''': (
            '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-cased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'''
        ),
        '''bert-base-german-dbmdz-uncased''': (
            '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-cased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'''
        ),
        '''TurkuNLP/bert-base-finnish-uncased-v1''': (
            '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'''
        ),
        '''wietsedv/bert-base-dutch-cased''': (
            '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'''
        ),
    },
}

lowercase  :							Optional[int]			  =    {
    '''bert-base-uncased''': 5_12,
    '''bert-large-uncased''': 5_12,
    '''bert-base-cased''': 5_12,
    '''bert-large-cased''': 5_12,
    '''bert-base-multilingual-uncased''': 5_12,
    '''bert-base-multilingual-cased''': 5_12,
    '''bert-base-chinese''': 5_12,
    '''bert-base-german-cased''': 5_12,
    '''bert-large-uncased-whole-word-masking''': 5_12,
    '''bert-large-cased-whole-word-masking''': 5_12,
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': 5_12,
    '''bert-large-cased-whole-word-masking-finetuned-squad''': 5_12,
    '''bert-base-cased-finetuned-mrpc''': 5_12,
    '''bert-base-german-dbmdz-cased''': 5_12,
    '''bert-base-german-dbmdz-uncased''': 5_12,
    '''TurkuNLP/bert-base-finnish-cased-v1''': 5_12,
    '''TurkuNLP/bert-base-finnish-uncased-v1''': 5_12,
    '''wietsedv/bert-base-dutch-cased''': 5_12,
}

lowercase  :							Any			  =    {
    '''bert-base-uncased''': {'''do_lower_case''': True},
    '''bert-large-uncased''': {'''do_lower_case''': True},
    '''bert-base-cased''': {'''do_lower_case''': False},
    '''bert-large-cased''': {'''do_lower_case''': False},
    '''bert-base-multilingual-uncased''': {'''do_lower_case''': True},
    '''bert-base-multilingual-cased''': {'''do_lower_case''': False},
    '''bert-base-chinese''': {'''do_lower_case''': False},
    '''bert-base-german-cased''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking''': {'''do_lower_case''': False},
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': True},
    '''bert-large-cased-whole-word-masking-finetuned-squad''': {'''do_lower_case''': False},
    '''bert-base-cased-finetuned-mrpc''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-cased''': {'''do_lower_case''': False},
    '''bert-base-german-dbmdz-uncased''': {'''do_lower_case''': True},
    '''TurkuNLP/bert-base-finnish-cased-v1''': {'''do_lower_case''': False},
    '''TurkuNLP/bert-base-finnish-uncased-v1''': {'''do_lower_case''': True},
    '''wietsedv/bert-base-dutch-cased''': {'''do_lower_case''': False},
}




class 		UpperCAmelCase_						(	SCREAMING_SNAKE_CASE__					):


						'''simple docstring'''




						A     :   Dict						=					VOCAB_FILES_NAMES
						A     :   List[str]						=					PRETRAINED_VOCAB_FILES_MAP
						A     :   Optional[int]						=					PRETRAINED_INIT_CONFIGURATION
						A     :   Any						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						A     :   List[Any]						=					BertTokenizer






						def __init__(	self		,				_SCREAMING_SNAKE_CASE=None		,				_SCREAMING_SNAKE_CASE=None		,				_SCREAMING_SNAKE_CASE=True		,				_SCREAMING_SNAKE_CASE="[UNK]"		,				_SCREAMING_SNAKE_CASE="[SEP]"		,				_SCREAMING_SNAKE_CASE="[PAD]"		,				_SCREAMING_SNAKE_CASE="[CLS]"		,				_SCREAMING_SNAKE_CASE="[MASK]"		,				_SCREAMING_SNAKE_CASE=True		,				_SCREAMING_SNAKE_CASE=None		,				**_SCREAMING_SNAKE_CASE		,				) ->				List[str]:
								super().__init__(
								    _SCREAMING_SNAKE_CASE		,				tokenizer_file=_SCREAMING_SNAKE_CASE		,				do_lower_case=_SCREAMING_SNAKE_CASE		,				unk_token=_SCREAMING_SNAKE_CASE		,				sep_token=_SCREAMING_SNAKE_CASE		,				pad_token=_SCREAMING_SNAKE_CASE		,				cls_token=_SCREAMING_SNAKE_CASE		,				mask_token=_SCREAMING_SNAKE_CASE		,				tokenize_chinese_chars=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE		,				)

								snake_case_				:							Union[str, Any]						 =       json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
								if (
								    normalizer_state.get("lowercase"		,				_SCREAMING_SNAKE_CASE				) != do_lower_case
								    or normalizer_state.get("strip_accents"		,				_SCREAMING_SNAKE_CASE				) != strip_accents
								    or normalizer_state.get("handle_chinese_chars"		,				_SCREAMING_SNAKE_CASE				) != tokenize_chinese_chars
								):
										snake_case_				:							Dict						 =       getattr(_SCREAMING_SNAKE_CASE		,				normalizer_state.pop("type"				)				)
										snake_case_				:							Optional[int]						 =       do_lower_case
										snake_case_				:							Optional[Any]						 =       strip_accents
										snake_case_				:							Any						 =       tokenize_chinese_chars
										snake_case_				:							str						 =       normalizer_class(**_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Union[str, Any]						 =       do_lower_case






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE=None				) ->				Tuple:
								snake_case_				:							List[Any]						 =       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
										output += token_ids_a + [self.sep_token_id]

								return output






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE = None				) ->				List[int]:
								snake_case_				:							Optional[int]						 =       [self.sep_token_id]
								snake_case_				:							str						 =       [self.cls_token_id]
								if token_ids_a is None:
										return len(cls + token_ids_a + sep				) * [0]
								return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE = None				) ->				Tuple[str]:
								snake_case_				:							Dict						 =       self._tokenizer.model.save(_SCREAMING_SNAKE_CASE		,				name=_SCREAMING_SNAKE_CASE				)
								return tuple(_SCREAMING_SNAKE_CASE				)








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_speech_available, is_torch_available


lowercase  :							int			  =    {
    '''configuration_audio_spectrogram_transformer''': [
        '''AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''ASTConfig''',
    ]
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowercase  :							Optional[Any]			  =    [
							    '''AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''ASTForAudioClassification''',
							    '''ASTModel''',
							    '''ASTPreTrainedModel''',
							]

try:
							if not is_speech_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowercase  :							Optional[int]			  =    ['''ASTFeatureExtractor''']

if TYPE_CHECKING:
							from .configuration_audio_spectrogram_transformer import (
							    AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
							    ASTConfig,
							)

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_audio_spectrogram_transformer import (
														    AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
														    ASTForAudioClassification,
														    ASTModel,
														    ASTPreTrainedModel,
														)

							try:
														if not is_speech_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .feature_extraction_audio_spectrogram_transformer import ASTFeatureExtractor


else:
							import sys

							lowercase  :							Optional[int]			  =    _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
