






"""simple docstring"""






import os
import unittest

from transformers import FunnelTokenizer, FunnelTokenizerFast
from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class       snake_case_       (    _lowerCamelCase    ,							unittest.TestCase       ):



	"""simple docstring"""

	SCREAMING_SNAKE_CASE_:							Optional[Any]   			=					FunnelTokenizer
	SCREAMING_SNAKE_CASE_:							Union[str, Any]   			=					FunnelTokenizerFast
	SCREAMING_SNAKE_CASE_:							List[Any]   			=					True
	SCREAMING_SNAKE_CASE_:							Any   			=					True


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					super().setUp()

					A__							 =       [
					    '<unk>',
					    '<cls>',
					    '<sep>',
					    'want',
					    '##want',
					    '##ed',
					    'wa',
					    'un',
					    'runn',
					    '##ing',
					    ',',
					    'low',
					    'lowest',
					]
					A__							 =       os.path.join(self.tmpdirname  ,					VOCAB_FILES_NAMES['vocab_file']						)
					with open(self.vocab_file  ,					'w'  ,					encoding='utf-8'						) as vocab_writer:
									vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)


	def       _UpperCAmelCase						(      self  ,					**__a						):


					"""simple docstring"""

					return FunnelTokenizer.from_pretrained(self.tmpdirname  ,					**__a						)


	def       _UpperCAmelCase						(      self  ,					**__a						):


					"""simple docstring"""

					return FunnelTokenizerFast.from_pretrained(self.tmpdirname  ,					**__a						)


	def       _UpperCAmelCase						(      self  ,					__a						):


					"""simple docstring"""

					A__							 =       'UNwant\u00E9d,running'
					A__							 =       'unwanted, running'
					return input_text, output_text


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       self.tokenizer_class(self.vocab_file						)

					A__							 =       tokenizer.tokenize('UNwant\u00E9d,running'						)
					self.assertListEqual(__a  ,					['un', '##want', '##ed', ',', 'runn', '##ing']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(__a						)  ,					[7, 4, 5, 10, 8, 9]						)






	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       self.get_tokenizers(do_lower_case=__a						)
					for tokenizer in tokenizers:
									A__							 =       tokenizer('UNwant\u00E9d,running'						)
									A__							 =       len(inputs['input_ids']						) - 1
									self.assertListEqual(inputs['token_type_ids']  ,					[2] + [0] * sentence_len						)

									A__							 =       tokenizer('UNwant\u00E9d,running'  ,					'UNwant\u00E9d,running'						)
									self.assertListEqual(inputs['token_type_ids']  ,					[2] + [0] * sentence_len + [1] * sentence_len						)












"""simple docstring"""






from ...processing_utils import ProcessorMixin





class       snake_case_       (    _lowerCamelCase       ):



	"""simple docstring"""

	SCREAMING_SNAKE_CASE_:							List[str]   			=					"""WhisperFeatureExtractor"""
	SCREAMING_SNAKE_CASE_:							Union[str, Any]   			=					"""WhisperTokenizer"""


	def __init__(      self  ,					__a  ,					__a						):


					"""simple docstring"""

					super().__init__(__a  ,					__a						)
					A__							 =       self.feature_extractor
					A__							 =       False


	def       _UpperCAmelCase						(      self  ,					__a=None  ,					__a=None  ,					__a=True						):


					"""simple docstring"""

					return self.tokenizer.get_decoder_prompt_ids(task=__a  ,					language=__a  ,					no_timestamps=__a						)


	def __call__(      self  ,					*__a  ,					**__a						):


					"""simple docstring"""

					if self._in_target_context_manager:
									return self.current_processor(*__a  ,					**__a						)

					A__							 =       kwargs.pop('audio'  ,					__a						)
					A__							 =       kwargs.pop('sampling_rate'  ,					__a						)
					A__							 =       kwargs.pop('text'  ,					__a						)
					if len(__a						) > 0:
									A__							 =       args[0]
									A__							 =       args[1:]

					if audio is None and text is None:
									raise ValueError('You need to specify either an `audio` or `text` input to process.'						)

					if audio is not None:
									A__							 =       self.feature_extractor(__a  ,					*__a  ,					sampling_rate=__a  ,					**__a						)
					if text is not None:
									A__							 =       self.tokenizer(__a  ,					**__a						)

					if text is None:
									return inputs

					elif audio is None:
									return encodings
					else:
									A__							 =       encodings['input_ids']
									return inputs


	def       _UpperCAmelCase						(      self  ,					*__a  ,					**__a						):


					"""simple docstring"""

					return self.tokenizer.batch_decode(*__a  ,					**__a						)


	def       _UpperCAmelCase						(      self  ,					*__a  ,					**__a						):


					"""simple docstring"""

					return self.tokenizer.decode(*__a  ,					**__a						)






	def       _UpperCAmelCase						(      self  ,					__a  ,					__a="np"						):


					"""simple docstring"""

					return self.tokenizer.get_prompt_ids(__a  ,					return_tensors=__a						)


