


'''simple docstring'''
import json
import os
import unittest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class      lowercase			(							A__			,	unittest.TestCase      ):

		"""simple docstring"""

		_a									=       MgpstrTokenizer
		_a									=       False
		_a									=       {}
		_a									=       False




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							super().setUp()

							# fmt: off
							UpperCamelCase__     :Tuple							      =				['''[GO]''', '''[s]''', '''0''', '''1''', '''2''', '''3''', '''4''', '''5''', '''6''', '''7''', '''8''', '''9''', '''a''', '''b''', '''c''', '''d''', '''e''', '''f''', '''g''', '''h''', '''i''', '''j''', '''k''', '''l''', '''m''', '''n''', '''o''', '''p''', '''q''', '''r''', '''s''', '''t''', '''u''', '''v''', '''w''', '''x''', '''y''', '''z''']
							# fmt: on
							UpperCamelCase__     :str							      =				dict(zip(UpperCamelCase_					,     range(len(UpperCamelCase_							)							)							)							)

							UpperCamelCase__     :List[Any]							      =				os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''vocab_file''']							)
							with open(self.vocab_file					,     '''w'''					,     encoding='''utf-8'''							) as fp:
												fp.write(json.dumps(UpperCamelCase_							) + '''\n'''							)




		def 					lowerCAmelCase__       (	self					,     **UpperCamelCase_							):

							'''simple docstring'''


							return MgpstrTokenizer.from_pretrained(self.tmpdirname					,     **UpperCamelCase_							)




		def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__     :Tuple							      =				'''tester'''
							UpperCamelCase__     :Dict							      =				'''tester'''
							return input_text, output_text




		@unittest.skip('''MGP-STR always lower cases letters.'''							)
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							pass




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[str]							      =				self.get_tokenizers(do_lower_case=UpperCamelCase_							)
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
																	UpperCamelCase__     :Optional[Any]							      =				'''[SPECIAL_TOKEN]'''

																	tokenizer.add_special_tokens({'''cls_token''': special_token}							)
																	UpperCamelCase__     :Union[str, Any]							      =				tokenizer.encode([special_token]					,     add_special_tokens=UpperCamelCase_							)
																	self.assertEqual(len(UpperCamelCase_							)					,     1							)

																	UpperCamelCase__     :Dict							      =				tokenizer.decode(UpperCamelCase_					,     skip_special_tokens=UpperCamelCase_							)
																	self.assertTrue(special_token not in decoded							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Optional[Any]							      =				self.get_tokenizers()
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
																	UpperCamelCase__      ,       UpperCamelCase__     :Optional[Any]							      =				self.get_input_output_texts(UpperCamelCase_							)

																	UpperCamelCase__     :List[str]							      =				tokenizer.tokenize(UpperCamelCase_							)
																	UpperCamelCase__     :Union[str, Any]							      =				tokenizer.convert_tokens_to_ids(UpperCamelCase_							)
																	UpperCamelCase__     :List[Any]							      =				tokenizer.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)

																	UpperCamelCase__     :int							      =				tokenizer.convert_ids_to_tokens(UpperCamelCase_							)
																	self.assertNotEqual(len(UpperCamelCase_							)					,     0							)
																	UpperCamelCase__     :List[str]							      =				tokenizer.decode(UpperCamelCase_							)
																	self.assertIsInstance(UpperCamelCase_					,     UpperCamelCase_							)

																	self.assertEqual(text_a.replace(''' '''					,     ''''''							)					,     UpperCamelCase_							)




		@unittest.skip('''MGP-STR tokenizer only handles one sequence.'''							)
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							pass




		@unittest.skip('''inputs cannot be pretokenized in MgpstrTokenizer'''							)
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							pass



'''simple docstring'''
import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin
class      lowercase			(							A__      ):

		"""simple docstring"""

		_a									=       'Speech2TextFeatureExtractor'
		_a									=       'Speech2TextTokenizer'




		def __init__(	self					,     UpperCamelCase_					,     UpperCamelCase_							):

							'''simple docstring'''


							super().__init__(UpperCamelCase_					,     UpperCamelCase_							)
							UpperCamelCase__     :List[str]							      =				self.feature_extractor
							UpperCamelCase__     :Optional[Any]							      =				False




		def __call__(	self					,     *UpperCamelCase_					,     **UpperCamelCase_							):

							'''simple docstring'''


							if self._in_target_context_manager:
												return self.current_processor(*UpperCamelCase_					,     **UpperCamelCase_							)

							if "raw_speech" in kwargs:
												warnings.warn('''Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'''							)
												UpperCamelCase__     :Optional[Any]							      =				kwargs.pop('''raw_speech'''							)
							else:
												UpperCamelCase__     :List[Any]							      =				kwargs.pop('''audio'''					,     UpperCamelCase_							)
							UpperCamelCase__     :List[Any]							      =				kwargs.pop('''sampling_rate'''					,     UpperCamelCase_							)
							UpperCamelCase__     :Dict							      =				kwargs.pop('''text'''					,     UpperCamelCase_							)
							if len(UpperCamelCase_							) > 0:
												UpperCamelCase__     :Tuple							      =				args[0]
												UpperCamelCase__     :Optional[int]							      =				args[1:]

							if audio is None and text is None:
												raise ValueError('''You need to specify either an `audio` or `text` input to process.'''							)

							if audio is not None:
												UpperCamelCase__     :Any							      =				self.feature_extractor(UpperCamelCase_					,     *UpperCamelCase_					,     sampling_rate=UpperCamelCase_					,     **UpperCamelCase_							)
							if text is not None:
												UpperCamelCase__     :Dict							      =				self.tokenizer(UpperCamelCase_					,     **UpperCamelCase_							)

							if text is None:
												return inputs
							elif audio is None:
												return encodings
							else:
												UpperCamelCase__     :Dict							      =				encodings['''input_ids''']
												return inputs




		def 					lowerCAmelCase__       (	self					,     *UpperCamelCase_					,     **UpperCamelCase_							):

							'''simple docstring'''


							return self.tokenizer.batch_decode(*UpperCamelCase_					,     **UpperCamelCase_							)




		def 					lowerCAmelCase__       (	self					,     *UpperCamelCase_					,     **UpperCamelCase_							):

							'''simple docstring'''


							return self.tokenizer.decode(*UpperCamelCase_					,     **UpperCamelCase_							)




		@contextmanager
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							warnings.warn(
							    '''`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '''
							    '''labels by using the argument `text` of the regular `__call__` method (either in the same call as '''
							    '''your audio inputs, or in a separate call.'''							)
							UpperCamelCase__     :Optional[Any]							      =				True
							UpperCamelCase__     :Any							      =				self.tokenizer
							yield
							UpperCamelCase__     :Any							      =				self.feature_extractor
							UpperCamelCase__     :Any							      =				False