





"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


A			      =	logging.get_logger(__name__)


A			      =	{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

# See all MVP models at https://huggingface.co/models?filter=mvp
A			      =	{
    '''vocab_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json''',
    },
    '''added_tokens.json''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json''',
    },
    '''merges_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json''',
    },
}

A			      =	{
    '''RUCAIBox/mvp''': 1_024,
}




class 					__lowercase      (      _UpperCamelCase				):







					'''simple docstring'''




					__lowerCAmelCase	=	VOCAB_FILES_NAMES
					__lowerCAmelCase	=	PRETRAINED_VOCAB_FILES_MAP
					__lowerCAmelCase	=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__lowerCAmelCase	=	['''input_ids''', '''attention_mask''']
					__lowerCAmelCase	=	MvpTokenizer







					def __init__(    self						,      _UpperCAmelCase=None						,      _UpperCAmelCase=None						,      _UpperCAmelCase=None						,      _UpperCAmelCase="replace"						,      _UpperCAmelCase="<s>"						,      _UpperCAmelCase="</s>"						,      _UpperCAmelCase="</s>"						,      _UpperCAmelCase="<s>"						,      _UpperCAmelCase="<unk>"						,      _UpperCAmelCase="<pad>"						,      _UpperCAmelCase="<mask>"						,      _UpperCAmelCase=False						,      _UpperCAmelCase=True						,      **_UpperCAmelCase						,      ):
									super().__init__(
									    _UpperCAmelCase						,      _UpperCAmelCase						,      tokenizer_file=_UpperCAmelCase						,      errors=_UpperCAmelCase						,      bos_token=_UpperCAmelCase						,      eos_token=_UpperCAmelCase						,      sep_token=_UpperCAmelCase						,      cls_token=_UpperCAmelCase						,      unk_token=_UpperCAmelCase						,      pad_token=_UpperCAmelCase						,      mask_token=_UpperCAmelCase						,      add_prefix_space=_UpperCAmelCase						,      trim_offsets=_UpperCAmelCase						,      **_UpperCAmelCase						,      )

									__a			:    str      							=	json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()  )
									if pre_tok_state.get('''add_prefix_space'''						,      _UpperCAmelCase  ) != add_prefix_space:
													__a			:    Union[str, Any]      							=	getattr(_UpperCAmelCase						,      pre_tok_state.pop('''type'''  )  )
													__a			:    str      							=	add_prefix_space
													__a			:    Optional[Any]      							=	pre_tok_class(**_UpperCAmelCase  )

									__a			:    List[str]      							=	add_prefix_space

									# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
									__a			:    Tuple      							=	'''post_processor'''
									__a			:    Dict      							=	getattr(self.backend_tokenizer						,      _UpperCAmelCase						,      _UpperCAmelCase  )
									if tokenizer_component_instance:
													__a			:    List[str]      							=	json.loads(tokenizer_component_instance.__getstate__()  )

													# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
													if "sep" in state:
																	__a			:    Any      							=	tuple(state['''sep''']  )
													if "cls" in state:
																	__a			:    Optional[int]      							=	tuple(state['''cls''']  )

													__a			:    Any      							=	False

													if state.get('''add_prefix_space'''						,      _UpperCAmelCase  ) != add_prefix_space:
																	__a			:    Union[str, Any]      							=	add_prefix_space
																	__a			:    str      							=	True

													if state.get('''trim_offsets'''						,      _UpperCAmelCase  ) != trim_offsets:
																	__a			:    List[Any]      							=	trim_offsets
																	__a			:    Optional[Any]      							=	True

													if changes_to_apply:
																	__a			:    List[str]      							=	getattr(_UpperCAmelCase						,      state.pop('''type'''  )  )
																	__a			:    List[str]      							=	component_class(**_UpperCAmelCase  )
																	setattr(self.backend_tokenizer						,      _UpperCAmelCase						,      _UpperCAmelCase  )







					@property
					def 			_lowerCamelCase		(    self  ):
									if self._mask_token is None:
													if self.verbose:
																	logger.error('''Using mask_token, but it is not set yet.'''  )
													return None
									return str(self._mask_token  )







					@mask_token.setter
					def 			_lowerCamelCase		(    self						,      _UpperCAmelCase  ):
									__a			:    str      							=	AddedToken(_UpperCAmelCase						,      lstrip=_UpperCAmelCase						,      rstrip=_UpperCAmelCase  ) if isinstance(_UpperCAmelCase						,      _UpperCAmelCase  ) else value
									__a			:    Tuple      							=	value







					def 			_lowerCamelCase		(    self						,      *_UpperCAmelCase						,      **_UpperCAmelCase  ):
									__a			:    List[Any]      							=	kwargs.get('''is_split_into_words'''						,      _UpperCAmelCase  )

									if is_split_into_words and not self.add_prefix_space:
													raise ValueError(
													    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
													    '''to use it with pretokenized inputs.'''  )

									return super()._batch_encode_plus(*_UpperCAmelCase						,      **_UpperCAmelCase  )







					def 			_lowerCamelCase		(    self						,      *_UpperCAmelCase						,      **_UpperCAmelCase  ):
									__a			:    List[str]      							=	kwargs.get('''is_split_into_words'''						,      _UpperCAmelCase  )

									if is_split_into_words and not self.add_prefix_space:
													raise ValueError(
													    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
													    '''to use it with pretokenized inputs.'''  )

									return super()._encode_plus(*_UpperCAmelCase						,      **_UpperCAmelCase  )







					def 			_lowerCamelCase		(    self						,      _UpperCAmelCase						,      _UpperCAmelCase = None  ):
									__a			:    Any      							=	self._tokenizer.model.save(_UpperCAmelCase						,      name=_UpperCAmelCase  )
									return tuple(_UpperCAmelCase  )







					def 			_lowerCamelCase		(    self						,      _UpperCAmelCase						,      _UpperCAmelCase=None  ):
									__a			:    str      							=	[self.bos_token_id] + token_ids_a + [self.eos_token_id]
									if token_ids_a is None:
													return output

									return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







					def 			_lowerCamelCase		(    self						,      _UpperCAmelCase						,      _UpperCAmelCase = None  ):
									__a			:    str      							=	[self.sep_token_id]
									__a			:    Any      							=	[self.cls_token_id]

									if token_ids_a is None:
													return len(cls + token_ids_a + sep  ) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]

"""simple docstring"""






import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin
from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
from .tokenization_wavaveca import WavaVecaCTCTokenizer

class 					__a     ( __a       ):



							'''simple docstring'''







							_lowerCamelCase					:			List[Any]          =   """Wav2Vec2FeatureExtractor"""
							_lowerCamelCase					:			Any          =   """AutoTokenizer"""
							def __init__(     self  ,    _lowerCamelCase  ,    _lowerCamelCase       )			->							List[str]:




										'''simple docstring'''
										super().__init__(_lowerCamelCase  ,    _lowerCamelCase       )
										__lowercase       =       self.feature_extractor
										__lowercase       =       False
							@classmethod
							def   SCREAMING_SNAKE_CASE						(     cls  ,    _lowerCamelCase  ,    **_lowerCamelCase       )			->							Any:




										'''simple docstring'''
										try:
													return super().from_pretrained(_lowerCamelCase  ,    **_lowerCamelCase       )
										except OSError:
													warnings.warn(
													    f'''Loading a tokenizer inside {cls.__name__} from a config that does not'''
													    " include a `tokenizer_class` attribute is deprecated and will be "
													    "removed in v5. Please add `'tokenizer_class': 'Wav2Vec2CTCTokenizer'`"
													    " attribute to either your `config.json` or `tokenizer_config.json` "
													    "file to suppress this warning: "  ,    _lowerCamelCase  ,    )

													__lowercase       =       WavaVecaFeatureExtractor.from_pretrained(_lowerCamelCase  ,    **_lowerCamelCase       )
													__lowercase       =       WavaVecaCTCTokenizer.from_pretrained(_lowerCamelCase  ,    **_lowerCamelCase       )

													return cls(feature_extractor=_lowerCamelCase  ,    tokenizer=_lowerCamelCase       )
							def __call__(     self  ,    *_lowerCamelCase  ,    **_lowerCamelCase       )			->							List[Any]:




										'''simple docstring'''
										# For backward compatibility
										if self._in_target_context_manager:
													return self.current_processor(*_lowerCamelCase  ,    **_lowerCamelCase       )

										if "raw_speech" in kwargs:
													warnings.warn("Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead."       )
													__lowercase       =       kwargs.pop("raw_speech"       )
										else:
													__lowercase       =       kwargs.pop("audio"  ,    _lowerCamelCase       )
										__lowercase       =       kwargs.pop("sampling_rate"  ,    _lowerCamelCase       )
										__lowercase       =       kwargs.pop("text"  ,    _lowerCamelCase       )
										if len(_lowerCamelCase       ) > 0:
													__lowercase       =       args[0]
													__lowercase       =       args[1:]

										if audio is None and text is None:
													raise ValueError("You need to specify either an `audio` or `text` input to process."       )

										if audio is not None:
													__lowercase       =       self.feature_extractor(_lowerCamelCase  ,    *_lowerCamelCase  ,    sampling_rate=_lowerCamelCase  ,    **_lowerCamelCase       )
										if text is not None:
													__lowercase       =       self.tokenizer(_lowerCamelCase  ,    **_lowerCamelCase       )

										if text is None:
													return inputs
										elif audio is None:
													return encodings
										else:
													__lowercase       =       encodings["input_ids"]
													return inputs
							def   SCREAMING_SNAKE_CASE						(     self  ,    *_lowerCamelCase  ,    **_lowerCamelCase       )			->							List[str]:




										'''simple docstring'''
										# For backward compatibility
										if self._in_target_context_manager:
													return self.current_processor.pad(*_lowerCamelCase  ,    **_lowerCamelCase       )

										__lowercase       =       kwargs.pop("input_features"  ,    _lowerCamelCase       )
										__lowercase       =       kwargs.pop("labels"  ,    _lowerCamelCase       )
										if len(_lowerCamelCase       ) > 0:
													__lowercase       =       args[0]
													__lowercase       =       args[1:]

										if input_features is not None:
													__lowercase       =       self.feature_extractor.pad(_lowerCamelCase  ,    *_lowerCamelCase  ,    **_lowerCamelCase       )
										if labels is not None:
													__lowercase       =       self.tokenizer.pad(_lowerCamelCase  ,    **_lowerCamelCase       )

										if labels is None:
													return input_features
										elif input_features is None:
													return labels
										else:
													__lowercase       =       labels["input_ids"]
													return input_features
							def   SCREAMING_SNAKE_CASE						(     self  ,    *_lowerCamelCase  ,    **_lowerCamelCase       )			->							Optional[int]:




										'''simple docstring'''
										return self.tokenizer.batch_decode(*_lowerCamelCase  ,    **_lowerCamelCase       )
							def   SCREAMING_SNAKE_CASE						(     self  ,    *_lowerCamelCase  ,    **_lowerCamelCase       )			->							Dict:




										'''simple docstring'''
										return self.tokenizer.decode(*_lowerCamelCase  ,    **_lowerCamelCase       )





							@contextmanager
							def   SCREAMING_SNAKE_CASE						(     self       )			->							str:




										'''simple docstring'''
										warnings.warn(
										    "`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your "
										    "labels by using the argument `text` of the regular `__call__` method (either in the same call as "
										    "your audio inputs, or in a separate call."       )
										__lowercase       =       True
										__lowercase       =       self.tokenizer
										yield
										__lowercase       =       self.feature_extractor
										__lowercase       =       False





