


import warnings
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__			:      List[Any]								=logging.get_logger(__name__)







class 	__lowercase    (__lowerCAmelCase  ):





							"""simple docstring"""







							_UpperCAmelCase      =						['''input_ids''', '''attention_mask''']






							def __init__(      self			,						lowerCAmelCase__="</s>"			,						lowerCAmelCase__="<unk>"			,						lowerCAmelCase__="<pad>"			,						lowerCAmelCase__=1_2_5			,						lowerCAmelCase__=None			,						**lowerCAmelCase__			,						):





										"""simple docstring"""




										if extra_ids > 0 and additional_special_tokens is None:
													SCREAMING_SNAKE_CASE_		:     List[Any]					=					[F'''<extra_id_{i}>''' for i in range(_lowerCamelCase       )]
										elif extra_ids > 0 and additional_special_tokens is not None:
													# Check that we have the right number of extra_id special tokens
													SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					len(set(filter(lambda lowerCAmelCase__       : bool('extra_id' in str(_lowerCamelCase       )       )			,						_lowerCamelCase       )       )       )
													if extra_tokens != extra_ids:
																raise ValueError(
																    F'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
																    ' provided to ByT5Tokenizer. In this case the additional_special_tokens must include the'
																    ' extra_ids tokens'       )

										SCREAMING_SNAKE_CASE_		:     Any					=					AddedToken(_lowerCamelCase			,						lstrip=_lowerCamelCase			,						rstrip=_lowerCamelCase       ) if isinstance(_lowerCamelCase			,						_lowerCamelCase       ) else pad_token
										SCREAMING_SNAKE_CASE_		:     List[str]					=					AddedToken(_lowerCamelCase			,						lstrip=_lowerCamelCase			,						rstrip=_lowerCamelCase       ) if isinstance(_lowerCamelCase			,						_lowerCamelCase       ) else eos_token
										SCREAMING_SNAKE_CASE_		:     int					=					AddedToken(_lowerCamelCase			,						lstrip=_lowerCamelCase			,						rstrip=_lowerCamelCase       ) if isinstance(_lowerCamelCase			,						_lowerCamelCase       ) else unk_token

										super().__init__(
										    eos_token=_lowerCamelCase			,						unk_token=_lowerCamelCase			,						pad_token=_lowerCamelCase			,						extra_ids=_lowerCamelCase			,						additional_special_tokens=_lowerCamelCase			,						**_lowerCamelCase			,						)

										SCREAMING_SNAKE_CASE_		:     Tuple					=					extra_ids

										SCREAMING_SNAKE_CASE_		:     Any					=					2**8  # utf is 8 bits

										# define special tokens dict
										SCREAMING_SNAKE_CASE_		:     Any					=					{
										    self.pad_token: 0,
										    self.eos_token: 1,
										    self.unk_token: 2,
										}
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					len(self.special_tokens_encoder       )
										SCREAMING_SNAKE_CASE_		:     Tuple					=					len(_lowerCamelCase       )
										for i, token in enumerate(_lowerCamelCase       ):
													SCREAMING_SNAKE_CASE_		:     int					=					self.vocab_size + i - n
										SCREAMING_SNAKE_CASE_		:     Tuple					=					{v: k for k, v in self.special_tokens_encoder.items()}






							@property
							def 					UpperCamelCase__     (      self       ):





										"""simple docstring"""




										return self._utf_vocab_size + self._num_special_tokens + self._extra_ids






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__			,						lowerCAmelCase__ = None			,						lowerCAmelCase__ = False       ):





										"""simple docstring"""




										if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=_lowerCamelCase			,						token_ids_a=_lowerCamelCase			,						already_has_special_tokens=_lowerCamelCase       )

										# normal case: some special tokens
										if token_ids_a is None:
													return ([0] * len(_lowerCamelCase       )) + [1]
										return ([0] * len(_lowerCamelCase       )) + [1] + ([0] * len(_lowerCamelCase       )) + [1]






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__       ):





										"""simple docstring"""




										if len(_lowerCamelCase       ) > 0 and token_ids[-1] == self.eos_token_id:
													warnings.warn(
													    F'''This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'''
													    ' eos tokens being added.'       )
													return token_ids
										else:
													return token_ids + [self.eos_token_id]






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__			,						lowerCAmelCase__ = None       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     List[Any]					=					[self.eos_token_id]

										if token_ids_a is None:
													return len(token_ids_a + eos       ) * [0]
										return len(token_ids_a + eos + token_ids_a + eos       ) * [0]






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__			,						lowerCAmelCase__ = None       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					self._add_eos_if_not_present(_lowerCamelCase       )
										if token_ids_a is None:
													return token_ids_a
										else:
													SCREAMING_SNAKE_CASE_		:     Tuple					=					self._add_eos_if_not_present(_lowerCamelCase       )
													return token_ids_a + token_ids_a






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Optional[int]					=					[chr(_lowerCamelCase       ) for i in text.encode('utf-8'       )]
										return tokens






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__       ):





										"""simple docstring"""




										if token in self.special_tokens_encoder:
													SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					self.special_tokens_encoder[token]
										elif token in self.added_tokens_encoder:
													SCREAMING_SNAKE_CASE_		:     int					=					self.added_tokens_encoder[token]
										elif len(_lowerCamelCase       ) != 1:
													SCREAMING_SNAKE_CASE_		:     Dict					=					self.unk_token_id
										else:
													SCREAMING_SNAKE_CASE_		:     Dict					=					ord(_lowerCamelCase       ) + self._num_special_tokens
										return token_id






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__       ):





										"""simple docstring"""




										if index in self.special_tokens_decoder:
													SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					self.special_tokens_decoder[index]
										else:
													SCREAMING_SNAKE_CASE_		:     Any					=					chr(index - self._num_special_tokens       )
										return token






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     List[Any]					=					b''
										for token in tokens:
													if token in self.special_tokens_decoder:
																SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					self.special_tokens_decoder[token].encode('utf-8'       )
													elif token in self.added_tokens_decoder:
																SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					self.special_tokens_decoder[token].encode('utf-8'       )
													elif token in self.special_tokens_encoder:
																SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					token.encode('utf-8'       )
													elif token in self.added_tokens_encoder:
																SCREAMING_SNAKE_CASE_		:     Any					=					token.encode('utf-8'       )
													else:
																SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					bytes([ord(_lowerCamelCase       )]       )
													bstring += tok_string
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					bstring.decode('utf-8'			,						errors='ignore'       )
										return string






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__			,						lowerCAmelCase__ = None       ):





										"""simple docstring"""




										return ()






"""simple docstring"""


import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin
from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
from .tokenization_wavaveca import WavaVecaCTCTokenizer






class   __UpperCAmelCase		(      __lowerCAmelCase      ):
				A__ :  List[Any]    				=			'''Wav2Vec2FeatureExtractor'''
				A__ :  Any    				=			'''AutoTokenizer'''

				def __init__( self      ,					_lowerCamelCase      ,					_lowerCamelCase ):
										super().__init__(_lowerCamelCase      ,					_lowerCamelCase )
										lowerCamelCase__   =self.feature_extractor
										lowerCamelCase__   =False

				@classmethod
				def 						_a      ( cls      ,					_lowerCamelCase      ,					**_lowerCamelCase ):
										try:
																return super().from_pretrained(_lowerCamelCase      ,					**_lowerCamelCase )
										except OSError:
																warnings.warn(
																    F'''Loading a tokenizer inside {cls.__name__} from a config that does not'''
																    " include a `tokenizer_class` attribute is deprecated and will be "
																    "removed in v5. Please add `'tokenizer_class': 'Wav2Vec2CTCTokenizer'`"
																    " attribute to either your `config.json` or `tokenizer_config.json` "
																    "file to suppress this warning: "      ,					_lowerCamelCase      ,					)

																lowerCamelCase__   =WavaVecaFeatureExtractor.from_pretrained(_lowerCamelCase      ,					**_lowerCamelCase )
																lowerCamelCase__   =WavaVecaCTCTokenizer.from_pretrained(_lowerCamelCase      ,					**_lowerCamelCase )

																return cls(feature_extractor=_lowerCamelCase      ,					tokenizer=_lowerCamelCase )

				def __call__( self      ,					*_lowerCamelCase      ,					**_lowerCamelCase ):
										# For backward compatibility
										if self._in_target_context_manager:
																return self.current_processor(*_lowerCamelCase      ,					**_lowerCamelCase )

										if "raw_speech" in kwargs:
																warnings.warn("Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead." )
																lowerCamelCase__   =kwargs.pop("raw_speech" )
										else:
																lowerCamelCase__   =kwargs.pop("audio"      ,					_lowerCamelCase )
										lowerCamelCase__   =kwargs.pop("sampling_rate"      ,					_lowerCamelCase )
										lowerCamelCase__   =kwargs.pop("text"      ,					_lowerCamelCase )
										if len(_lowerCamelCase ) > 0:
																lowerCamelCase__   =args[0]
																lowerCamelCase__   =args[1:]

										if audio is None and text is None:
																raise ValueError("You need to specify either an `audio` or `text` input to process." )

										if audio is not None:
																lowerCamelCase__   =self.feature_extractor(_lowerCamelCase      ,					*_lowerCamelCase      ,					sampling_rate=_lowerCamelCase      ,					**_lowerCamelCase )
										if text is not None:
																lowerCamelCase__   =self.tokenizer(_lowerCamelCase      ,					**_lowerCamelCase )

										if text is None:
																return inputs
										elif audio is None:
																return encodings
										else:
																lowerCamelCase__   =encodings["input_ids"]
																return inputs

				def 						_a      ( self      ,					*_lowerCamelCase      ,					**_lowerCamelCase ):
										# For backward compatibility
										if self._in_target_context_manager:
																return self.current_processor.pad(*_lowerCamelCase      ,					**_lowerCamelCase )

										lowerCamelCase__   =kwargs.pop("input_features"      ,					_lowerCamelCase )
										lowerCamelCase__   =kwargs.pop("labels"      ,					_lowerCamelCase )
										if len(_lowerCamelCase ) > 0:
																lowerCamelCase__   =args[0]
																lowerCamelCase__   =args[1:]

										if input_features is not None:
																lowerCamelCase__   =self.feature_extractor.pad(_lowerCamelCase      ,					*_lowerCamelCase      ,					**_lowerCamelCase )
										if labels is not None:
																lowerCamelCase__   =self.tokenizer.pad(_lowerCamelCase      ,					**_lowerCamelCase )

										if labels is None:
																return input_features
										elif input_features is None:
																return labels
										else:
																lowerCamelCase__   =labels["input_ids"]
																return input_features

				def 						_a      ( self      ,					*_lowerCamelCase      ,					**_lowerCamelCase ):
										return self.tokenizer.batch_decode(*_lowerCamelCase      ,					**_lowerCamelCase )

				def 						_a      ( self      ,					*_lowerCamelCase      ,					**_lowerCamelCase ):
										return self.tokenizer.decode(*_lowerCamelCase      ,					**_lowerCamelCase )






				@contextmanager
				def 						_a      ( self ):
										warnings.warn(
										    "`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your "
										    "labels by using the argument `text` of the regular `__call__` method (either in the same call as "
										    "your audio inputs, or in a separate call." )
										lowerCamelCase__   =True
										lowerCamelCase__   =self.tokenizer
										yield
										lowerCamelCase__   =self.feature_extractor
										lowerCamelCase__   =False


