





"""simple docstring"""


import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__       :					List[str]						=			get_tests_dir("""fixtures/test_sentencepiece_bpe_char.model""")






@require_sentencepiece
@require_tokenizers
class 							lowercase__			(   __SCREAMING_SNAKE_CASE					,     unittest.TestCase	):

			"""simple docstring"""
			__lowerCAmelCase     :				Tuple										=  SpeechTaTokenizer
			__lowerCAmelCase     :				str										=  False
			__lowerCAmelCase     :				List[str]										=  True







			def   _a      (      self    ):






									'''simple docstring'''







									super().setUp()

									# We have a SentencePiece fixture for testing
									UpperCamelCase				:		List[str]				   =			SpeechTaTokenizer(_A    )

									UpperCamelCase				:		Tuple				   =			AddedToken("""<mask>"""						,   lstrip=_A						,   rstrip=_A    )
									UpperCamelCase				:		Optional[Any]				   =			mask_token
									tokenizer.add_special_tokens({"""mask_token""": mask_token}    )
									tokenizer.add_tokens(["""<ctc_blank>"""]    )

									tokenizer.save_pretrained(self.tmpdirname    )







			def   _a      (      self						,   _A    ):






									'''simple docstring'''







									UpperCamelCase				:		Any				   =			"""this is a test"""
									UpperCamelCase				:		str				   =			"""this is a test"""
									return input_text, output_text







			def   _a      (      self						,   _A						,   _A=False						,   _A=2_0						,   _A=5    ):






									'''simple docstring'''







									UpperCamelCase   ,						UpperCamelCase				:		List[str]				   =			self.get_input_output_texts(_A    )
									UpperCamelCase				:		Dict				   =			tokenizer.encode(_A						,   add_special_tokens=_A    )
									UpperCamelCase				:		Union[str, Any]				   =			tokenizer.decode(_A						,   clean_up_tokenization_spaces=_A    )
									return text, ids







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Optional[Any]				   =			"""<pad>"""
									UpperCamelCase				:		List[Any]				   =			1

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A    )						,   _A    )
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A    )						,   _A    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Union[str, Any]				   =			list(self.get_tokenizer().get_vocab().keys()    )

									self.assertEqual(vocab_keys[0]						,   """<s>"""    )
									self.assertEqual(vocab_keys[1]						,   """<pad>"""    )
									self.assertEqual(vocab_keys[-4]						,   """œ"""    )
									self.assertEqual(vocab_keys[-2]						,   """<mask>"""    )
									self.assertEqual(vocab_keys[-1]						,   """<ctc_blank>"""    )
									self.assertEqual(len(_A    )						,   8_1    )







			def   _a      (      self    ):






									'''simple docstring'''







									self.assertEqual(self.get_tokenizer().vocab_size						,   7_9    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		List[str]				   =			self.get_tokenizers(do_lower_case=_A    )
									for tokenizer in tokenizers:
															with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
																					UpperCamelCase				:		Optional[int]				   =			tokenizer.vocab_size
																					UpperCamelCase				:		Union[str, Any]				   =			len(_A    )

																					self.assertNotEqual(_A						,   0    )

																					# We usually have added tokens from the start in tests because our vocab fixtures are
																					# smaller than the original vocabs - let's not assert this
																					# self.assertEqual(vocab_size, all_size)

																					UpperCamelCase				:		Dict				   =			["""aaaaa bbbbbb""", """cccccccccdddddddd"""]
																					UpperCamelCase				:		Any				   =			tokenizer.add_tokens(_A    )
																					UpperCamelCase				:		Dict				   =			tokenizer.vocab_size
																					UpperCamelCase				:		int				   =			len(_A    )

																					self.assertNotEqual(_A						,   0    )
																					self.assertEqual(_A						,   _A    )
																					self.assertEqual(_A						,   len(_A    )    )
																					self.assertEqual(_A						,   all_size + len(_A    )    )

																					UpperCamelCase				:		Optional[Any]				   =			tokenizer.encode("""aaaaa bbbbbb low cccccccccdddddddd l"""						,   add_special_tokens=_A    )

																					self.assertGreaterEqual(len(_A    )						,   4    )
																					self.assertGreater(tokens[0]						,   tokenizer.vocab_size - 1    )
																					self.assertGreater(tokens[-3]						,   tokenizer.vocab_size - 1    )

																					UpperCamelCase				:		int				   =			{"""eos_token""": """>>>>|||<||<<|<<""", """pad_token""": """<<<<<|||>|>>>>|>"""}
																					UpperCamelCase				:		List[Any]				   =			tokenizer.add_special_tokens(_A    )
																					UpperCamelCase				:		str				   =			tokenizer.vocab_size
																					UpperCamelCase				:		Optional[int]				   =			len(_A    )

																					self.assertNotEqual(_A						,   0    )
																					self.assertEqual(_A						,   _A    )
																					self.assertEqual(_A						,   len(_A    )    )
																					self.assertEqual(_A						,   all_size_a + len(_A    )    )

																					UpperCamelCase				:		List[Any]				   =			tokenizer.encode(
																					    """>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l"""						,   add_special_tokens=_A    )

																					self.assertGreaterEqual(len(_A    )						,   6    )
																					self.assertGreater(tokens[0]						,   tokenizer.vocab_size - 1    )
																					self.assertGreater(tokens[0]						,   tokens[1]    )
																					self.assertGreater(tokens[-3]						,   tokenizer.vocab_size - 1    )
																					self.assertGreater(tokens[-3]						,   tokens[-4]    )
																					self.assertEqual(tokens[0]						,   tokenizer.eos_token_id    )
																					self.assertEqual(tokens[-3]						,   tokenizer.pad_token_id    )







			def   _a      (      self    ):






									'''simple docstring'''







									pass







			def   _a      (      self    ):






									'''simple docstring'''







									pass







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		List[Any]				   =			self.get_tokenizer()

									UpperCamelCase				:		Dict				   =			tokenizer.tokenize("""This is a test"""    )
									# fmt: off
									self.assertListEqual(_A						,   [SPIECE_UNDERLINE, """T""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """a""", SPIECE_UNDERLINE, """t""", """e""", """s""", """t"""]    )
									# fmt: on

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(_A    )						,   [4, 3_2, 1_1, 1_0, 1_2, 4, 1_0, 1_2, 4, 7, 4, 6, 5, 1_2, 6]						,   )

									UpperCamelCase				:		Tuple				   =			tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
									self.assertListEqual(
									    _A						,   [SPIECE_UNDERLINE, """I""", SPIECE_UNDERLINE, """w""", """a""", """s""", SPIECE_UNDERLINE, """b""", """o""", """r""", """n""", SPIECE_UNDERLINE, """i""", """n""", SPIECE_UNDERLINE, """92000""", """,""", SPIECE_UNDERLINE, """a""", """n""", """d""", SPIECE_UNDERLINE, """t""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """f""", """a""", """l""", """s""", """é""", """."""]    )

									UpperCamelCase				:		Optional[Any]				   =			tokenizer.convert_tokens_to_ids(_A    )
									# fmt: off
									self.assertListEqual(_A						,   [4, 3_0, 4, 2_0, 7, 1_2, 4, 2_5, 8, 1_3, 9, 4, 1_0, 9, 4, 3, 2_3, 4, 7, 9, 1_4, 4, 6, 1_1, 1_0, 1_2, 4, 1_0, 1_2, 4, 1_9, 7, 1_5, 1_2, 7_3, 2_6]    )
									# fmt: on

									UpperCamelCase				:		List[Any]				   =			tokenizer.convert_ids_to_tokens(_A    )
									self.assertListEqual(
									    _A						,   [SPIECE_UNDERLINE, """I""", SPIECE_UNDERLINE, """w""", """a""", """s""", SPIECE_UNDERLINE, """b""", """o""", """r""", """n""", SPIECE_UNDERLINE, """i""", """n""", SPIECE_UNDERLINE, """<unk>""", """,""", SPIECE_UNDERLINE, """a""", """n""", """d""", SPIECE_UNDERLINE, """t""", """h""", """i""", """s""", SPIECE_UNDERLINE, """i""", """s""", SPIECE_UNDERLINE, """f""", """a""", """l""", """s""", """é""", """."""]    )







			@slow
			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Optional[Any]				   =			[
									    """Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides """
									    """general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural """
									    """Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained """
									    """models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.""",
									    """BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly """
									    """conditioning on both left and right context in all layers.""",
									    """The quick brown fox jumps over the lazy dog.""",
									]

									# fmt: off
									UpperCamelCase				:		Optional[Any]				   =			{
									    """input_ids""": [
									        [4, 3_2, 1_3, 7, 9, 1_2, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_2, 4, 6_4, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_5, 2_2, 4, 2_8, 9, 8, 2_0, 9, 4, 7, 1_2, 4, 2_4, 2_2, 6, 8, 1_3, 1_7, 1_1, 3_9, 6, 1_3, 7, 9, 1_2, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_2, 4, 7, 9, 1_4, 4, 2_4, 2_2, 6, 8, 1_3, 1_7, 1_1, 3_9, 2_4, 1_3, 5, 6, 1_3, 7, 1_0, 9, 5, 1_4, 3_9, 2_5, 5, 1_3, 6, 6_3, 4, 2_4, 1_3, 8, 2_7, 1_0, 1_4, 5, 1_2, 4, 2_1, 5, 9, 5, 1_3, 7, 1_5, 3_9, 2_4, 1_6, 1_3, 2_4, 8, 1_2, 5, 4, 7, 1_3, 1_7, 1_1, 1_0, 6, 5, 1_7, 6, 1_6, 1_3, 5, 1_2, 4, 6_4, 4_0, 4_7, 5_4, 3_2, 2_3, 4, 5_3, 4_9, 3_2, 2_3, 4, 5_4, 8, 4_0, 4_7, 5_4, 3_2, 7, 2_3, 4, 6_9, 5_2, 4_3, 2_3, 4, 5_1, 1_0, 1_2, 6, 1_0, 1_5, 4_0, 5, 1_3, 6, 2_3, 4, 6_9, 5_2, 4_8, 5, 6, 2_6, 2_6, 2_6, 6_3, 4, 1_9, 8, 1_3, 4, 4_8, 7, 6, 1_6, 1_3, 7, 1_5, 4, 5_2, 7, 9, 2_1, 1_6, 7, 2_1, 5, 4, 6_1, 9, 1_4, 5, 1_3, 1_2, 6, 7, 9, 1_4, 1_0, 9, 2_1, 4, 6_4, 4_8, 5_2, 6_1, 6_3, 4, 7, 9, 1_4, 4, 4_8, 7, 6, 1_6, 1_3, 7, 1_5, 4, 5_2, 7, 9, 2_1, 1_6, 7, 2_1, 5, 4, 5_3, 5, 9, 5, 1_3, 7, 6, 1_0, 8, 9, 4, 6_4, 4_8, 5_2, 5_3, 6_3, 4, 2_0, 1_0, 6, 1_1, 4, 8, 2_7, 5, 1_3, 4, 6, 1_1, 1_0, 1_3, 6, 2_2, 3_9, 6, 2_0, 8, 4, 2_4, 1_3, 5, 6, 1_3, 7, 1_0, 9, 5, 1_4, 4, 1_8, 8, 1_4, 5, 1_5, 1_2, 4, 1_0, 9, 4, 8, 9, 5, 4, 1_1, 1_6, 9, 1_4, 1_3, 5, 1_4, 4, 2_4, 1_5, 1_6, 1_2, 4, 1_5, 7, 9, 2_1, 1_6, 7, 2_1, 5, 1_2, 4, 7, 9, 1_4, 4, 1_4, 5, 5, 2_4, 4, 1_0, 9, 6, 5, 1_3, 8, 2_4, 5, 1_3, 7, 2_5, 1_0, 1_5, 1_0, 6, 2_2, 4, 2_5, 5, 6, 2_0, 5, 5, 9, 4, 5_8, 7, 3_7, 2_3, 4, 4_9, 2_2, 3_2, 8, 1_3, 1_7, 1_1, 4, 7, 9, 1_4, 4, 3_2, 5, 9, 1_2, 8, 1_3, 5_5, 1_5, 8, 2_0, 2_6, 2],
									        [4, 4_0, 4_7, 5_4, 3_2, 4, 1_0, 1_2, 4, 1_4, 5, 1_2, 1_0, 2_1, 9, 5, 1_4, 4, 6, 8, 4, 2_4, 1_3, 5, 3_9, 6, 1_3, 7, 1_0, 9, 4, 1_4, 5, 5, 2_4, 4, 2_5, 1_0, 1_4, 1_0, 1_3, 5, 1_7, 6, 1_0, 8, 9, 7, 1_5, 4, 1_3, 5, 2_4, 1_3, 5, 1_2, 5, 9, 6, 7, 6, 1_0, 8, 9, 1_2, 4, 1_9, 1_3, 8, 1_8, 4, 1_6, 9, 1_5, 7, 2_5, 5, 1_5, 5, 1_4, 4, 6, 5, 3_7, 6, 4, 2_5, 2_2, 4, 4_6, 8, 1_0, 9, 6, 1_5, 2_2, 4, 1_7, 8, 9, 1_4, 1_0, 6, 1_0, 8, 9, 1_0, 9, 2_1, 4, 8, 9, 4, 2_5, 8, 6, 1_1, 4, 1_5, 5, 1_9, 6, 4, 7, 9, 1_4, 4, 1_3, 1_0, 2_1, 1_1, 6, 4, 1_7, 8, 9, 6, 5, 3_7, 6, 4, 1_0, 9, 4, 7, 1_5, 1_5, 4, 1_5, 7, 2_2, 5, 1_3, 1_2, 2_6, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
									        [4, 3_2, 1_1, 5, 4, 4_5, 1_6, 1_0, 1_7, 2_8, 4, 2_5, 1_3, 8, 2_0, 9, 4, 1_9, 8, 3_7, 4, 4_6, 1_6, 1_8, 2_4, 1_2, 4, 8, 2_7, 5, 1_3, 4, 6, 1_1, 5, 4, 1_5, 7, 5_7, 2_2, 4, 1_4, 8, 2_1, 2_6, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
									    ],
									    """attention_mask""": [
									        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
									        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
									    ]
									}
									# fmt: on

									self.tokenizer_integration_test_util(
									    expected_encoding=_A						,   model_name="""microsoft/speecht5_asr"""						,   revision="""c5ef64c71905caeccde0e4462ef3f9077224c524"""						,   sequences=_A						,   )








"""simple docstring"""

import unittest

import numpy as np
from datasets import load_dataset

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
							import torch

if is_vision_available():
							from PIL import Image

							from transformers import BeitImageProcessor
class     lowerCAmelCase__						(	unittest.TestCase ):


	'''simple docstring'''







	def __init__(      self				:       Union[str, Any]     ,       _SCREAMING_SNAKE_CASE				:       Any     ,       _SCREAMING_SNAKE_CASE				:       Union[str, Any]=7     ,       _SCREAMING_SNAKE_CASE				:       Union[str, Any]=3     ,       _SCREAMING_SNAKE_CASE				:       Optional[int]=18     ,       _SCREAMING_SNAKE_CASE				:       List[Any]=30     ,       _SCREAMING_SNAKE_CASE				:       Optional[int]=400     ,       _SCREAMING_SNAKE_CASE				:       Dict=True     ,       _SCREAMING_SNAKE_CASE				:       str=None     ,       _SCREAMING_SNAKE_CASE				:       Any=True     ,       _SCREAMING_SNAKE_CASE				:       Union[str, Any]=None     ,       _SCREAMING_SNAKE_CASE				:       Dict=True     ,       _SCREAMING_SNAKE_CASE				:       List[str]=[0.5, 0.5, 0.5]     ,       _SCREAMING_SNAKE_CASE				:       Optional[int]=[0.5, 0.5, 0.5]     ,       _SCREAMING_SNAKE_CASE				:       List[Any]=False     ,       )						->	int:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							List[str]							=     size if size is not None else {'height': 20, 'width': 20}
					SCREAMING_SNAKE_CASE						:							int							=     crop_size if crop_size is not None else {'height': 18, 'width': 18}
					SCREAMING_SNAKE_CASE						:							Optional[int]							=     parent
					SCREAMING_SNAKE_CASE						:							Dict							=     batch_size
					SCREAMING_SNAKE_CASE						:							Optional[Any]							=     num_channels
					SCREAMING_SNAKE_CASE						:							Tuple							=     image_size
					SCREAMING_SNAKE_CASE						:							Optional[Any]							=     min_resolution
					SCREAMING_SNAKE_CASE						:							List[str]							=     max_resolution
					SCREAMING_SNAKE_CASE						:							Tuple							=     do_resize
					SCREAMING_SNAKE_CASE						:							str							=     size
					SCREAMING_SNAKE_CASE						:							Dict							=     do_center_crop
					SCREAMING_SNAKE_CASE						:							List[Any]							=     crop_size
					SCREAMING_SNAKE_CASE						:							Optional[int]							=     do_normalize
					SCREAMING_SNAKE_CASE						:							Optional[Any]							=     image_mean
					SCREAMING_SNAKE_CASE						:							Dict							=     image_std
					SCREAMING_SNAKE_CASE						:							Any							=     do_reduce_labels







	def 			_lowerCAmelCase     (      self				:       Dict    )						->	str:





					"""simple docstring"""


					return {
					    "do_resize": self.do_resize,
					    "size": self.size,
					    "do_center_crop": self.do_center_crop,
					    "crop_size": self.crop_size,
					    "do_normalize": self.do_normalize,
					    "image_mean": self.image_mean,
					    "image_std": self.image_std,
					    "do_reduce_labels": self.do_reduce_labels,
					}




def  __snake_case							(	)     ->       List[str]:






				'''simple docstring'''
				SCREAMING_SNAKE_CASE						:							Dict							=     load_dataset('hf-internal-testing/fixtures_ade20k'							,		split='test'     )

				SCREAMING_SNAKE_CASE						:							Any							=     Image.open(dataset[0]['file']     )
				SCREAMING_SNAKE_CASE						:							int							=     Image.open(dataset[1]['file']     )

				return image, map
def  __snake_case							(	)     ->       Optional[Any]:






				'''simple docstring'''
				SCREAMING_SNAKE_CASE						:							int							=     load_dataset('hf-internal-testing/fixtures_ade20k'							,		split='test'     )

				SCREAMING_SNAKE_CASE						:							Optional[int]							=     Image.open(ds[0]['file']     )
				SCREAMING_SNAKE_CASE						:							Optional[int]							=     Image.open(ds[1]['file']     )
				SCREAMING_SNAKE_CASE						:							Tuple							=     Image.open(ds[2]['file']     )
				SCREAMING_SNAKE_CASE						:							Tuple							=     Image.open(ds[3]['file']     )

				return [imagea, imagea], [mapa, mapa]







@require_torch
@require_vision
class     lowerCAmelCase__						(	_lowerCamelCase							,							unittest.TestCase ):


	'''simple docstring'''







	_SCREAMING_SNAKE_CASE	:		Tuple  						= BeitImageProcessor if is_vision_available() else None





	def 			_lowerCAmelCase     (      self				:       Optional[Any]    )						->	Dict:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							List[Any]							=     BeitImageProcessingTester(self    )





	@property
	def 			_lowerCAmelCase     (      self				:       Optional[int]    )						->	int:





					"""simple docstring"""


					return self.image_processor_tester.prepare_image_processor_dict()





	def 			_lowerCAmelCase     (      self				:       int    )						->	Optional[int]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Any							=     self.image_processing_class(**self.image_processor_dict    )
					self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE     ,       'do_resize'    )    )
					self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE     ,       'size'    )    )
					self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE     ,       'do_center_crop'    )    )
					self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE     ,       'center_crop'    )    )
					self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE     ,       'do_normalize'    )    )
					self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE     ,       'image_mean'    )    )
					self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE     ,       'image_std'    )    )





	def 			_lowerCAmelCase     (      self				:       Optional[Any]    )						->	Union[str, Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							int							=     self.image_processing_class.from_dict(self.image_processor_dict    )
					self.assertEqual(image_processor.size     ,       {'height': 20, 'width': 20}    )
					self.assertEqual(image_processor.crop_size     ,       {'height': 18, 'width': 18}    )
					self.assertEqual(image_processor.do_reduce_labels     ,       _SCREAMING_SNAKE_CASE    )

					SCREAMING_SNAKE_CASE						:							List[Any]							=     self.image_processing_class.from_dict(
					    self.image_processor_dict     ,       size=42     ,       crop_size=84     ,       reduce_labels=_SCREAMING_SNAKE_CASE    )
					self.assertEqual(image_processor.size     ,       {'height': 42, 'width': 42}    )
					self.assertEqual(image_processor.crop_size     ,       {'height': 84, 'width': 84}    )
					self.assertEqual(image_processor.do_reduce_labels     ,       _SCREAMING_SNAKE_CASE    )





	def 			_lowerCAmelCase     (      self				:       str    )						->	Dict:





					"""simple docstring"""


					pass





	def 			_lowerCAmelCase     (      self				:       Any    )						->	List[Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Optional[Any]							=     self.image_processing_class(**self.image_processor_dict    )
					# create random PIL images
					SCREAMING_SNAKE_CASE						:							List[str]							=     prepare_image_inputs(self.image_processor_tester     ,       equal_resolution=_SCREAMING_SNAKE_CASE    )
					for image in image_inputs:
									self.assertIsInstance(_SCREAMING_SNAKE_CASE     ,       Image.Image    )

					# Test not batched input
					SCREAMING_SNAKE_CASE						:							Any							=     image_processing(image_inputs[0]     ,       return_tensors='pt'    ).pixel_values
					self.assertEqual(
					    encoded_images.shape     ,       (
					        1,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )     ,       )

					# Test batched
					SCREAMING_SNAKE_CASE						:							str							=     image_processing(_SCREAMING_SNAKE_CASE     ,       return_tensors='pt'    ).pixel_values
					self.assertEqual(
					    encoded_images.shape     ,       (
					        self.image_processor_tester.batch_size,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )     ,       )





	def 			_lowerCAmelCase     (      self				:       Union[str, Any]    )						->	Any:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							List[str]							=     self.image_processing_class(**self.image_processor_dict    )
					# create random numpy tensors
					SCREAMING_SNAKE_CASE						:							Dict							=     prepare_image_inputs(self.image_processor_tester     ,       equal_resolution=_SCREAMING_SNAKE_CASE     ,       numpify=_SCREAMING_SNAKE_CASE    )
					for image in image_inputs:
									self.assertIsInstance(_SCREAMING_SNAKE_CASE     ,       np.ndarray    )

					# Test not batched input
					SCREAMING_SNAKE_CASE						:							Dict							=     image_processing(image_inputs[0]     ,       return_tensors='pt'    ).pixel_values
					self.assertEqual(
					    encoded_images.shape     ,       (
					        1,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )     ,       )

					# Test batched
					SCREAMING_SNAKE_CASE						:							str							=     image_processing(_SCREAMING_SNAKE_CASE     ,       return_tensors='pt'    ).pixel_values
					self.assertEqual(
					    encoded_images.shape     ,       (
					        self.image_processor_tester.batch_size,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )     ,       )





	def 			_lowerCAmelCase     (      self				:       str    )						->	List[Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Any							=     self.image_processing_class(**self.image_processor_dict    )
					# create random PyTorch tensors
					SCREAMING_SNAKE_CASE						:							Optional[Any]							=     prepare_image_inputs(self.image_processor_tester     ,       equal_resolution=_SCREAMING_SNAKE_CASE     ,       torchify=_SCREAMING_SNAKE_CASE    )
					for image in image_inputs:
									self.assertIsInstance(_SCREAMING_SNAKE_CASE     ,       torch.Tensor    )

					# Test not batched input
					SCREAMING_SNAKE_CASE						:							List[str]							=     image_processing(image_inputs[0]     ,       return_tensors='pt'    ).pixel_values
					self.assertEqual(
					    encoded_images.shape     ,       (
					        1,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )     ,       )

					# Test batched
					SCREAMING_SNAKE_CASE						:							str							=     image_processing(_SCREAMING_SNAKE_CASE     ,       return_tensors='pt'    ).pixel_values
					self.assertEqual(
					    encoded_images.shape     ,       (
					        self.image_processor_tester.batch_size,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )     ,       )





	def 			_lowerCAmelCase     (      self				:       int    )						->	Optional[Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Dict							=     self.image_processing_class(**self.image_processor_dict    )
					# create random PyTorch tensors
					SCREAMING_SNAKE_CASE						:							str							=     prepare_image_inputs(self.image_processor_tester     ,       equal_resolution=_SCREAMING_SNAKE_CASE     ,       torchify=_SCREAMING_SNAKE_CASE    )
					SCREAMING_SNAKE_CASE						:							Tuple							=     []
					for image in image_inputs:
									self.assertIsInstance(_SCREAMING_SNAKE_CASE     ,       torch.Tensor    )
									maps.append(torch.zeros(image.shape[-2:]    ).long()    )

					# Test not batched input
					SCREAMING_SNAKE_CASE						:							int							=     image_processing(image_inputs[0]     ,       maps[0]     ,       return_tensors='pt'    )
					self.assertEqual(
					    encoding['pixel_values'].shape     ,       (
					        1,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )     ,       )
					self.assertEqual(
					    encoding['labels'].shape     ,       (
					        1,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )     ,       )
					self.assertEqual(encoding['labels'].dtype     ,       torch.long    )
					self.assertTrue(encoding['labels'].min().item() >= 0    )
					self.assertTrue(encoding['labels'].max().item() <= 255    )

					# Test batched
					SCREAMING_SNAKE_CASE						:							List[str]							=     image_processing(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE     ,       return_tensors='pt'    )
					self.assertEqual(
					    encoding['pixel_values'].shape     ,       (
					        self.image_processor_tester.batch_size,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )     ,       )
					self.assertEqual(
					    encoding['labels'].shape     ,       (
					        self.image_processor_tester.batch_size,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )     ,       )
					self.assertEqual(encoding['labels'].dtype     ,       torch.long    )
					self.assertTrue(encoding['labels'].min().item() >= 0    )
					self.assertTrue(encoding['labels'].max().item() <= 255    )

					# Test not batched input (PIL images)
					SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE						:							str							=     prepare_semantic_single_inputs()

					SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     image_processing(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE     ,       return_tensors='pt'    )
					self.assertEqual(
					    encoding['pixel_values'].shape     ,       (
					        1,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )     ,       )
					self.assertEqual(
					    encoding['labels'].shape     ,       (
					        1,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )     ,       )
					self.assertEqual(encoding['labels'].dtype     ,       torch.long    )
					self.assertTrue(encoding['labels'].min().item() >= 0    )
					self.assertTrue(encoding['labels'].max().item() <= 255    )

					# Test batched input (PIL images)
					SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE						:							int							=     prepare_semantic_batch_inputs()

					SCREAMING_SNAKE_CASE						:							Any							=     image_processing(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE     ,       return_tensors='pt'    )
					self.assertEqual(
					    encoding['pixel_values'].shape     ,       (
					        2,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )     ,       )
					self.assertEqual(
					    encoding['labels'].shape     ,       (
					        2,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )     ,       )
					self.assertEqual(encoding['labels'].dtype     ,       torch.long    )
					self.assertTrue(encoding['labels'].min().item() >= 0    )
					self.assertTrue(encoding['labels'].max().item() <= 255    )







	def 			_lowerCAmelCase     (      self				:       Optional[int]    )						->	Union[str, Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE						:							Optional[Any]							=     self.image_processing_class(**self.image_processor_dict    )

					# ADE20k has 150 classes, and the background is included, so labels should be between 0 and 150
					SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE						:							Any							=     prepare_semantic_single_inputs()
					SCREAMING_SNAKE_CASE						:							int							=     image_processing(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE     ,       return_tensors='pt'    )
					self.assertTrue(encoding['labels'].min().item() >= 0    )
					self.assertTrue(encoding['labels'].max().item() <= 150    )

					SCREAMING_SNAKE_CASE						:							Optional[Any]							=     True
					SCREAMING_SNAKE_CASE						:							Any							=     image_processing(_SCREAMING_SNAKE_CASE     ,       _SCREAMING_SNAKE_CASE     ,       return_tensors='pt'    )
					self.assertTrue(encoding['labels'].min().item() >= 0    )
					self.assertTrue(encoding['labels'].max().item() <= 255    )





