




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__SCREAMING_SNAKE_CASE 						=logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE 						={'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__SCREAMING_SNAKE_CASE 						={
    '''vocab_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'''
        ),
        '''distilbert-base-german-cased''': '''https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt''',
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''distilbert-base-uncased''': '''https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json''',
        '''distilbert-base-uncased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-cased''': '''https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json''',
        '''distilbert-base-cased-distilled-squad''': (
            '''https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-german-cased''': (
            '''https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'''
        ),
        '''distilbert-base-multilingual-cased''': (
            '''https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'''
        ),
    },
}

__SCREAMING_SNAKE_CASE 						={
    '''distilbert-base-uncased''': 512,
    '''distilbert-base-uncased-distilled-squad''': 512,
    '''distilbert-base-cased''': 512,
    '''distilbert-base-cased-distilled-squad''': 512,
    '''distilbert-base-german-cased''': 512,
    '''distilbert-base-multilingual-cased''': 512,
}


__SCREAMING_SNAKE_CASE 						={
    '''distilbert-base-uncased''': {'''do_lower_case''': True},
    '''distilbert-base-uncased-distilled-squad''': {'''do_lower_case''': True},
    '''distilbert-base-cased''': {'''do_lower_case''': False},
    '''distilbert-base-cased-distilled-squad''': {'''do_lower_case''': False},
    '''distilbert-base-german-cased''': {'''do_lower_case''': False},
    '''distilbert-base-multilingual-cased''': {'''do_lower_case''': False},
}

class  __magic_name__						(  __UpperCAmelCase):
				'''simple docstring'''







				SCREAMING_SNAKE_CASE__					:							List[str]	      =					VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE__					:							int	      =					PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE__					:							Optional[Any]	      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE__					:							Union[str, Any]	      =					PRETRAINED_INIT_CONFIGURATION
				SCREAMING_SNAKE_CASE__					:							List[Any]	      =					["input_ids", "attention_mask"]
				SCREAMING_SNAKE_CASE__					:							Any	      =					DistilBertTokenizer


				def __init__(							self:   int		,					_lowerCamelCase:   Union[str, Any]=None		,					_lowerCamelCase:   Optional[Any]=None		,					_lowerCamelCase:   int=True		,					_lowerCamelCase:   Any="[UNK]"		,					_lowerCamelCase:   List[Any]="[SEP]"		,					_lowerCamelCase:   Optional[Any]="[PAD]"		,					_lowerCamelCase:   List[Any]="[CLS]"		,					_lowerCamelCase:   List[Any]="[MASK]"		,					_lowerCamelCase:   List[Any]=True		,					_lowerCamelCase:   Any=None		,					**_lowerCamelCase:   Optional[int]		,					):
											super().__init__(
											    UpperCamelCase_		,					tokenizer_file=UpperCamelCase_		,					do_lower_case=UpperCamelCase_		,					unk_token=UpperCamelCase_		,					sep_token=UpperCamelCase_		,					pad_token=UpperCamelCase_		,					cls_token=UpperCamelCase_		,					mask_token=UpperCamelCase_		,					tokenize_chinese_chars=UpperCamelCase_		,					strip_accents=UpperCamelCase_		,					**UpperCamelCase_		,					)

											SCREAMING_SNAKE_CASE_   	=    json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
											if (
											    normalizer_state.get('''lowercase'''		,					UpperCamelCase_		) != do_lower_case
											    or normalizer_state.get('''strip_accents'''		,					UpperCamelCase_		) != strip_accents
											    or normalizer_state.get('''handle_chinese_chars'''		,					UpperCamelCase_		) != tokenize_chinese_chars
											):
																		SCREAMING_SNAKE_CASE_   	=    getattr(UpperCamelCase_		,					normalizer_state.pop('''type'''		)		)
																		SCREAMING_SNAKE_CASE_   	=    do_lower_case
																		SCREAMING_SNAKE_CASE_   	=    strip_accents
																		SCREAMING_SNAKE_CASE_   	=    tokenize_chinese_chars
																		SCREAMING_SNAKE_CASE_   	=    normalizer_class(**UpperCamelCase_		)

											SCREAMING_SNAKE_CASE_   	=    do_lower_case


				def     _A							(							self:   int		,					_lowerCamelCase:   Any		,					_lowerCamelCase:   List[str]=None		):
											SCREAMING_SNAKE_CASE_   	=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

											if token_ids_a:
																		output += token_ids_a + [self.sep_token_id]

											return output


				def     _A							(							self:   Optional[Any]		,					_lowerCamelCase:   List[int]		,					_lowerCamelCase:   Optional[List[int]] = None		):
											SCREAMING_SNAKE_CASE_   	=    [self.sep_token_id]
											SCREAMING_SNAKE_CASE_   	=    [self.cls_token_id]
											if token_ids_a is None:
																		return len(cls + token_ids_a + sep		) * [0]
											return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]





				def     _A							(							self:   Tuple		,					_lowerCamelCase:   str		,					_lowerCamelCase:   Optional[str] = None		):
											SCREAMING_SNAKE_CASE_   	=    self._tokenizer.model.save(UpperCamelCase_		,					name=UpperCamelCase_		)
											return tuple(UpperCamelCase_		)








import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin
from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
from .tokenization_wavaveca import WavaVecaCTCTokenizer






class  __magic_name__						(  __UpperCAmelCase):
				'''simple docstring'''







				SCREAMING_SNAKE_CASE__					:							List[str]	      =					"Wav2Vec2FeatureExtractor"
				SCREAMING_SNAKE_CASE__					:							List[Any]	      =					"AutoTokenizer"


				def __init__(							self:   Tuple		,					_lowerCamelCase:   str		,					_lowerCamelCase:   Optional[Any]		):
											super().__init__(_lowerCamelCase		,					_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    self.feature_extractor
											SCREAMING_SNAKE_CASE_   	=    False


				@classmethod
				def     _A							(							cls:   List[Any]		,					_lowerCamelCase:   Tuple		,					**_lowerCamelCase:   List[str]		):
											try:
																		return super().from_pretrained(_lowerCamelCase		,					**_lowerCamelCase		)
											except OSError:
																		warnings.warn(
																		    f"Loading a tokenizer inside {cls.__name__} from a config that does not"
																		    ''' include a `tokenizer_class` attribute is deprecated and will be '''
																		    '''removed in v5. Please add `\'tokenizer_class\': \'Wav2Vec2CTCTokenizer\'`'''
																		    ''' attribute to either your `config.json` or `tokenizer_config.json` '''
																		    '''file to suppress this warning: '''		,					_lowerCamelCase		,					)

																		SCREAMING_SNAKE_CASE_   	=    WavaVecaFeatureExtractor.from_pretrained(_lowerCamelCase		,					**_lowerCamelCase		)
																		SCREAMING_SNAKE_CASE_   	=    WavaVecaCTCTokenizer.from_pretrained(_lowerCamelCase		,					**_lowerCamelCase		)

																		return cls(feature_extractor=_lowerCamelCase		,					tokenizer=_lowerCamelCase		)


				def __call__(							self:   Union[str, Any]		,					*_lowerCamelCase:   List[Any]		,					**_lowerCamelCase:   str		):
											# For backward compatibility
											if self._in_target_context_manager:
																		return self.current_processor(*_lowerCamelCase		,					**_lowerCamelCase		)

											if "raw_speech" in kwargs:
																		warnings.warn('''Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'''		)
																		SCREAMING_SNAKE_CASE_   	=    kwargs.pop('''raw_speech'''		)
											else:
																		SCREAMING_SNAKE_CASE_   	=    kwargs.pop('''audio'''		,					_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    kwargs.pop('''sampling_rate'''		,					_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    kwargs.pop('''text'''		,					_lowerCamelCase		)
											if len(_lowerCamelCase		) > 0:
																		SCREAMING_SNAKE_CASE_   	=    args[0]
																		SCREAMING_SNAKE_CASE_   	=    args[1:]

											if audio is None and text is None:
																		raise ValueError('''You need to specify either an `audio` or `text` input to process.'''		)

											if audio is not None:
																		SCREAMING_SNAKE_CASE_   	=    self.feature_extractor(_lowerCamelCase		,					*_lowerCamelCase		,					sampling_rate=_lowerCamelCase		,					**_lowerCamelCase		)
											if text is not None:
																		SCREAMING_SNAKE_CASE_   	=    self.tokenizer(_lowerCamelCase		,					**_lowerCamelCase		)

											if text is None:
																		return inputs
											elif audio is None:
																		return encodings
											else:
																		SCREAMING_SNAKE_CASE_   	=    encodings['''input_ids''']
																		return inputs


				def     _A							(							self:   Optional[int]		,					*_lowerCamelCase:   str		,					**_lowerCamelCase:   List[Any]		):
											# For backward compatibility
											if self._in_target_context_manager:
																		return self.current_processor.pad(*_lowerCamelCase		,					**_lowerCamelCase		)

											SCREAMING_SNAKE_CASE_   	=    kwargs.pop('''input_features'''		,					_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    kwargs.pop('''labels'''		,					_lowerCamelCase		)
											if len(_lowerCamelCase		) > 0:
																		SCREAMING_SNAKE_CASE_   	=    args[0]
																		SCREAMING_SNAKE_CASE_   	=    args[1:]

											if input_features is not None:
																		SCREAMING_SNAKE_CASE_   	=    self.feature_extractor.pad(_lowerCamelCase		,					*_lowerCamelCase		,					**_lowerCamelCase		)
											if labels is not None:
																		SCREAMING_SNAKE_CASE_   	=    self.tokenizer.pad(_lowerCamelCase		,					**_lowerCamelCase		)

											if labels is None:
																		return input_features
											elif input_features is None:
																		return labels
											else:
																		SCREAMING_SNAKE_CASE_   	=    labels['''input_ids''']
																		return input_features


				def     _A							(							self:   str		,					*_lowerCamelCase:   Dict		,					**_lowerCamelCase:   Dict		):
											return self.tokenizer.batch_decode(*_lowerCamelCase		,					**_lowerCamelCase		)


				def     _A							(							self:   Optional[int]		,					*_lowerCamelCase:   Optional[int]		,					**_lowerCamelCase:   Tuple		):
											return self.tokenizer.decode(*_lowerCamelCase		,					**_lowerCamelCase		)





				@contextmanager
				def     _A							(							self:   Tuple		):
											warnings.warn(
											    '''`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '''
											    '''labels by using the argument `text` of the regular `__call__` method (either in the same call as '''
											    '''your audio inputs, or in a separate call.'''		)
											SCREAMING_SNAKE_CASE_   	=    True
											SCREAMING_SNAKE_CASE_   	=    self.tokenizer
											yield
											SCREAMING_SNAKE_CASE_   	=    self.feature_extractor
											SCREAMING_SNAKE_CASE_   	=    False


