




import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


UpperCamelCase__ :   str						=							logging.get_logger(__name__)


UpperCamelCase__ :   Optional[Any]						=							{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

# See all MVP models at https://huggingface.co/models?filter=mvp
UpperCamelCase__ :   Optional[int]						=							{
    '''vocab_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json''',
    },
    '''added_tokens.json''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json''',
    },
    '''merges_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json''',
    },
}

UpperCamelCase__ :   Tuple						=							{
    '''RUCAIBox/mvp''': 10_24,
}







class 				lowerCAmelCase_       (					lowerCamelCase_							):
				__a     :       Optional[int]						=     VOCAB_FILES_NAMES
				__a     :       Tuple						=     PRETRAINED_VOCAB_FILES_MAP
				__a     :       str						=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__a     :       List[Any]						=     ["input_ids", "attention_mask"]
				__a     :       Dict						=     MvpTokenizer






				def __init__(			self      ,snake_case__=None      ,snake_case__=None      ,snake_case__=None      ,snake_case__="replace"      ,snake_case__="<s>"      ,snake_case__="</s>"      ,snake_case__="</s>"      ,snake_case__="<s>"      ,snake_case__="<unk>"      ,snake_case__="<pad>"      ,snake_case__="<mask>"      ,snake_case__=False      ,snake_case__=True      ,**snake_case__      ,):
							super().__init__(
							    snake_case__      ,snake_case__      ,tokenizer_file=snake_case__      ,errors=snake_case__      ,bos_token=snake_case__      ,eos_token=snake_case__      ,sep_token=snake_case__      ,cls_token=snake_case__      ,unk_token=snake_case__      ,pad_token=snake_case__      ,mask_token=snake_case__      ,add_prefix_space=snake_case__      ,trim_offsets=snake_case__      ,**snake_case__      ,)

							SCREAMING_SNAKE_CASE_   : str					  =      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__() )
							if pre_tok_state.get('add_prefix_space'      ,snake_case__ ) != add_prefix_space:
										SCREAMING_SNAKE_CASE_   : Tuple					  =      getattr(snake_case__      ,pre_tok_state.pop('type' ) )
										SCREAMING_SNAKE_CASE_   : Tuple					  =      add_prefix_space
										SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      pre_tok_class(**snake_case__ )

							SCREAMING_SNAKE_CASE_   : int					  =      add_prefix_space

							# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      'post_processor'
							SCREAMING_SNAKE_CASE_   : str					  =      getattr(self.backend_tokenizer      ,snake_case__      ,snake_case__ )
							if tokenizer_component_instance:
										SCREAMING_SNAKE_CASE_   : Tuple					  =      json.loads(tokenizer_component_instance.__getstate__() )

										# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
										if "sep" in state:
													SCREAMING_SNAKE_CASE_   : List[str]					  =      tuple(state['sep'] )
										if "cls" in state:
													SCREAMING_SNAKE_CASE_   : Optional[int]					  =      tuple(state['cls'] )

										SCREAMING_SNAKE_CASE_   : List[Any]					  =      False

										if state.get('add_prefix_space'      ,snake_case__ ) != add_prefix_space:
													SCREAMING_SNAKE_CASE_   : Any					  =      add_prefix_space
													SCREAMING_SNAKE_CASE_   : str					  =      True

										if state.get('trim_offsets'      ,snake_case__ ) != trim_offsets:
													SCREAMING_SNAKE_CASE_   : Optional[int]					  =      trim_offsets
													SCREAMING_SNAKE_CASE_   : str					  =      True

										if changes_to_apply:
													SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      getattr(snake_case__      ,state.pop('type' ) )
													SCREAMING_SNAKE_CASE_   : Optional[int]					  =      component_class(**snake_case__ )
													setattr(self.backend_tokenizer      ,snake_case__      ,snake_case__ )






				@property
				def 		snake_case	(			self ):
							if self._mask_token is None:
										if self.verbose:
													logger.error('Using mask_token, but it is not set yet.' )
										return None
							return str(self._mask_token )






				@mask_token.setter
				def 		snake_case	(			self      ,snake_case__ ):
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      AddedToken(snake_case__      ,lstrip=snake_case__      ,rstrip=snake_case__ ) if isinstance(snake_case__      ,snake_case__ ) else value
							SCREAMING_SNAKE_CASE_   : Tuple					  =      value






				def 		snake_case	(			self      ,*snake_case__      ,**snake_case__ ):
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      kwargs.get('is_split_into_words'      ,snake_case__ )

							if is_split_into_words and not self.add_prefix_space:
										raise ValueError(
										    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
										    'to use it with pretokenized inputs.' )

							return super()._batch_encode_plus(*snake_case__      ,**snake_case__ )






				def 		snake_case	(			self      ,*snake_case__      ,**snake_case__ ):
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      kwargs.get('is_split_into_words'      ,snake_case__ )

							if is_split_into_words and not self.add_prefix_space:
										raise ValueError(
										    F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
										    'to use it with pretokenized inputs.' )

							return super()._encode_plus(*snake_case__      ,**snake_case__ )






				def 		snake_case	(			self      ,snake_case__      ,snake_case__ = None ):
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      self._tokenizer.model.save(snake_case__      ,name=snake_case__ )
							return tuple(snake_case__ )






				def 		snake_case	(			self      ,snake_case__      ,snake_case__=None ):
							SCREAMING_SNAKE_CASE_   : int					  =      [self.bos_token_id] + token_ids_a + [self.eos_token_id]
							if token_ids_a is None:
										return output

							return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






				def 		snake_case	(			self      ,snake_case__      ,snake_case__ = None ):
							SCREAMING_SNAKE_CASE_   : str					  =      [self.sep_token_id]
							SCREAMING_SNAKE_CASE_   : Any					  =      [self.cls_token_id]

							if token_ids_a is None:
										return len(cls + token_ids_a + sep ) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase__ :   Optional[Any]						=							{
    '''configuration_whisper''': ['''WHISPER_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''WhisperConfig''', '''WhisperOnnxConfig'''],
    '''feature_extraction_whisper''': ['''WhisperFeatureExtractor'''],
    '''processing_whisper''': ['''WhisperProcessor'''],
    '''tokenization_whisper''': ['''WhisperTokenizer'''],
}

try:
						if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__ :   str						=							['''WhisperTokenizerFast''']

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__ :   List[Any]						=							[
						    '''WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''WhisperForConditionalGeneration''',
						    '''WhisperModel''',
						    '''WhisperPreTrainedModel''',
						    '''WhisperForAudioClassification''',
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__ :   Tuple						=							[
						    '''TF_WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''TFWhisperForConditionalGeneration''',
						    '''TFWhisperModel''',
						    '''TFWhisperPreTrainedModel''',
						]

try:
						if not is_flax_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						UpperCamelCase__ :   List[str]						=							[
						    '''FlaxWhisperForConditionalGeneration''',
						    '''FlaxWhisperModel''',
						    '''FlaxWhisperPreTrainedModel''',
						    '''FlaxWhisperForAudioClassification''',
						]


if TYPE_CHECKING:
						from .configuration_whisper import WHISPER_PRETRAINED_CONFIG_ARCHIVE_MAP, WhisperConfig, WhisperOnnxConfig
						from .feature_extraction_whisper import WhisperFeatureExtractor
						from .processing_whisper import WhisperProcessor
						from .tokenization_whisper import WhisperTokenizer

						try:
												if not is_tokenizers_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_whisper_fast import WhisperTokenizerFast

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_whisper import (
												    WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST,
												    WhisperForAudioClassification,
												    WhisperForConditionalGeneration,
												    WhisperModel,
												    WhisperPreTrainedModel,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_whisper import (
												    TF_WHISPER_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFWhisperForConditionalGeneration,
												    TFWhisperModel,
												    TFWhisperPreTrainedModel,
												)

						try:
												if not is_flax_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_flax_whisper import (
												    FlaxWhisperForAudioClassification,
												    FlaxWhisperForConditionalGeneration,
												    FlaxWhisperModel,
												    FlaxWhisperPreTrainedModel,
												)

else:
						import sys

						UpperCamelCase__ :   Optional[Any]						=							_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



