






'''simple docstring'''
import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__		:     Any					=    get_tests_dir('''fixtures/test_sentencepiece_bpe_char.model''')







@require_sentencepiece
@require_tokenizers
class 	UpperCamelCase__					(     UpperCamelCase_      ,     unittest.TestCase			):

		"""simple docstring"""
		UpperCAmelCase__     			=						SpeechTaTokenizer
		UpperCAmelCase__     			=						False
		UpperCAmelCase__     			=						True




		def       snake_case     ( self						:						Union[str, Any]    ):







							"""simple docstring"""
							super().setUp()

							# We have a SentencePiece fixture for testing
							_lowercase			= SpeechTaTokenizer(a__    )

							_lowercase			= AddedToken("<mask>"							,    lstrip=a__							,    rstrip=a__    )
							_lowercase			= mask_token
							tokenizer.add_special_tokens({"mask_token": mask_token}    )
							tokenizer.add_tokens(["<ctc_blank>"]    )

							tokenizer.save_pretrained(self.tmpdirname    )




		def       snake_case     ( self						:						Dict							,    __A						:						str    ):







							"""simple docstring"""
							_lowercase			= "this is a test"
							_lowercase			= "this is a test"
							return input_text, output_text




		def       snake_case     ( self						:						Union[str, Any]							,    __A						:						Any							,    __A						:						Union[str, Any]=False							,    __A						:						str=2_0							,    __A						:						Optional[Any]=5    ):







							"""simple docstring"""
							_lowercase			= self.get_input_output_texts(a__    )
							_lowercase			= tokenizer.encode(a__							,    add_special_tokens=a__    )
							_lowercase			= tokenizer.decode(a__							,    clean_up_tokenization_spaces=a__    )
							return text, ids




		def       snake_case     ( self						:						List[str]    ):







							"""simple docstring"""
							_lowercase			= "<pad>"
							_lowercase			= 1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(a__    )							,    a__    )
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(a__    )							,    a__    )




		def       snake_case     ( self						:						str    ):







							"""simple docstring"""
							_lowercase			= list(self.get_tokenizer().get_vocab().keys()    )

							self.assertEqual(vocab_keys[0]							,    "<s>"    )
							self.assertEqual(vocab_keys[1]							,    "<pad>"    )
							self.assertEqual(vocab_keys[-4]							,    "œ"    )
							self.assertEqual(vocab_keys[-2]							,    "<mask>"    )
							self.assertEqual(vocab_keys[-1]							,    "<ctc_blank>"    )
							self.assertEqual(len(a__    )							,    8_1    )




		def       snake_case     ( self						:						List[Any]    ):







							"""simple docstring"""
							self.assertEqual(self.get_tokenizer().vocab_size							,    7_9    )




		def       snake_case     ( self						:						List[str]    ):







							"""simple docstring"""
							_lowercase			= self.get_tokenizers(do_lower_case=a__    )
							for tokenizer in tokenizers:
												with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
																	_lowercase			= tokenizer.vocab_size
																	_lowercase			= len(a__    )

																	self.assertNotEqual(a__							,    0    )

																	# We usually have added tokens from the start in tests because our vocab fixtures are
																	# smaller than the original vocabs - let's not assert this
																	# self.assertEqual(vocab_size, all_size)

																	_lowercase			= ["aaaaa bbbbbb", "cccccccccdddddddd"]
																	_lowercase			= tokenizer.add_tokens(a__    )
																	_lowercase			= tokenizer.vocab_size
																	_lowercase			= len(a__    )

																	self.assertNotEqual(a__							,    0    )
																	self.assertEqual(a__							,    a__    )
																	self.assertEqual(a__							,    len(a__    )    )
																	self.assertEqual(a__							,    all_size + len(a__    )    )

																	_lowercase			= tokenizer.encode("aaaaa bbbbbb low cccccccccdddddddd l"							,    add_special_tokens=a__    )

																	self.assertGreaterEqual(len(a__    )							,    4    )
																	self.assertGreater(tokens[0]							,    tokenizer.vocab_size - 1    )
																	self.assertGreater(tokens[-3]							,    tokenizer.vocab_size - 1    )

																	_lowercase			= {"eos_token": ">>>>|||<||<<|<<", "pad_token": "<<<<<|||>|>>>>|>"}
																	_lowercase			= tokenizer.add_special_tokens(a__    )
																	_lowercase			= tokenizer.vocab_size
																	_lowercase			= len(a__    )

																	self.assertNotEqual(a__							,    0    )
																	self.assertEqual(a__							,    a__    )
																	self.assertEqual(a__							,    len(a__    )    )
																	self.assertEqual(a__							,    all_size_a + len(a__    )    )

																	_lowercase			= tokenizer.encode(
																	    ">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l"							,    add_special_tokens=a__    )

																	self.assertGreaterEqual(len(a__    )							,    6    )
																	self.assertGreater(tokens[0]							,    tokenizer.vocab_size - 1    )
																	self.assertGreater(tokens[0]							,    tokens[1]    )
																	self.assertGreater(tokens[-3]							,    tokenizer.vocab_size - 1    )
																	self.assertGreater(tokens[-3]							,    tokens[-4]    )
																	self.assertEqual(tokens[0]							,    tokenizer.eos_token_id    )
																	self.assertEqual(tokens[-3]							,    tokenizer.pad_token_id    )




		def       snake_case     ( self						:						Any    ):







							"""simple docstring"""
							pass




		def       snake_case     ( self						:						List[str]    ):







							"""simple docstring"""
							pass




		def       snake_case     ( self						:						Optional[Any]    ):







							"""simple docstring"""
							_lowercase			= self.get_tokenizer()

							_lowercase			= tokenizer.tokenize("This is a test"    )
							# fmt: off
							self.assertListEqual(a__							,    [SPIECE_UNDERLINE, "T", "h", "i", "s", SPIECE_UNDERLINE, "i", "s", SPIECE_UNDERLINE, "a", SPIECE_UNDERLINE, "t", "e", "s", "t"]    )
							# fmt: on

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(a__    )							,    [4, 3_2, 1_1, 1_0, 1_2, 4, 1_0, 1_2, 4, 7, 4, 6, 5, 1_2, 6]							,    )

							_lowercase			= tokenizer.tokenize("I was born in 92000, and this is falsé."    )
							self.assertListEqual(
							    a__							,    [SPIECE_UNDERLINE, "I", SPIECE_UNDERLINE, "w", "a", "s", SPIECE_UNDERLINE, "b", "o", "r", "n", SPIECE_UNDERLINE, "i", "n", SPIECE_UNDERLINE, "92000", ",", SPIECE_UNDERLINE, "a", "n", "d", SPIECE_UNDERLINE, "t", "h", "i", "s", SPIECE_UNDERLINE, "i", "s", SPIECE_UNDERLINE, "f", "a", "l", "s", "é", "."]    )

							_lowercase			= tokenizer.convert_tokens_to_ids(a__    )
							# fmt: off
							self.assertListEqual(a__							,    [4, 3_0, 4, 2_0, 7, 1_2, 4, 2_5, 8, 1_3, 9, 4, 1_0, 9, 4, 3, 2_3, 4, 7, 9, 1_4, 4, 6, 1_1, 1_0, 1_2, 4, 1_0, 1_2, 4, 1_9, 7, 1_5, 1_2, 7_3, 2_6]    )
							# fmt: on

							_lowercase			= tokenizer.convert_ids_to_tokens(a__    )
							self.assertListEqual(
							    a__							,    [SPIECE_UNDERLINE, "I", SPIECE_UNDERLINE, "w", "a", "s", SPIECE_UNDERLINE, "b", "o", "r", "n", SPIECE_UNDERLINE, "i", "n", SPIECE_UNDERLINE, "<unk>", ",", SPIECE_UNDERLINE, "a", "n", "d", SPIECE_UNDERLINE, "t", "h", "i", "s", SPIECE_UNDERLINE, "i", "s", SPIECE_UNDERLINE, "f", "a", "l", "s", "é", "."]    )




		@slow
		def       snake_case     ( self						:						Optional[int]    ):







							"""simple docstring"""
							_lowercase			= [
							    "Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides "
							    "general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural "
							    "Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained "
							    "models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.",
							    "BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly "
							    "conditioning on both left and right context in all layers.",
							    "The quick brown fox jumps over the lazy dog.",
							]

							# fmt: off
							_lowercase			= {
							    "input_ids": [
							        [4, 3_2, 1_3, 7, 9, 1_2, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_2, 4, 6_4, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_5, 2_2, 4, 2_8, 9, 8, 2_0, 9, 4, 7, 1_2, 4, 2_4, 2_2, 6, 8, 1_3, 1_7, 1_1, 3_9, 6, 1_3, 7, 9, 1_2, 1_9, 8, 1_3, 1_8, 5, 1_3, 1_2, 4, 7, 9, 1_4, 4, 2_4, 2_2, 6, 8, 1_3, 1_7, 1_1, 3_9, 2_4, 1_3, 5, 6, 1_3, 7, 1_0, 9, 5, 1_4, 3_9, 2_5, 5, 1_3, 6, 6_3, 4, 2_4, 1_3, 8, 2_7, 1_0, 1_4, 5, 1_2, 4, 2_1, 5, 9, 5, 1_3, 7, 1_5, 3_9, 2_4, 1_6, 1_3, 2_4, 8, 1_2, 5, 4, 7, 1_3, 1_7, 1_1, 1_0, 6, 5, 1_7, 6, 1_6, 1_3, 5, 1_2, 4, 6_4, 4_0, 4_7, 5_4, 3_2, 2_3, 4, 5_3, 4_9, 3_2, 2_3, 4, 5_4, 8, 4_0, 4_7, 5_4, 3_2, 7, 2_3, 4, 6_9, 5_2, 4_3, 2_3, 4, 5_1, 1_0, 1_2, 6, 1_0, 1_5, 4_0, 5, 1_3, 6, 2_3, 4, 6_9, 5_2, 4_8, 5, 6, 2_6, 2_6, 2_6, 6_3, 4, 1_9, 8, 1_3, 4, 4_8, 7, 6, 1_6, 1_3, 7, 1_5, 4, 5_2, 7, 9, 2_1, 1_6, 7, 2_1, 5, 4, 6_1, 9, 1_4, 5, 1_3, 1_2, 6, 7, 9, 1_4, 1_0, 9, 2_1, 4, 6_4, 4_8, 5_2, 6_1, 6_3, 4, 7, 9, 1_4, 4, 4_8, 7, 6, 1_6, 1_3, 7, 1_5, 4, 5_2, 7, 9, 2_1, 1_6, 7, 2_1, 5, 4, 5_3, 5, 9, 5, 1_3, 7, 6, 1_0, 8, 9, 4, 6_4, 4_8, 5_2, 5_3, 6_3, 4, 2_0, 1_0, 6, 1_1, 4, 8, 2_7, 5, 1_3, 4, 6, 1_1, 1_0, 1_3, 6, 2_2, 3_9, 6, 2_0, 8, 4, 2_4, 1_3, 5, 6, 1_3, 7, 1_0, 9, 5, 1_4, 4, 1_8, 8, 1_4, 5, 1_5, 1_2, 4, 1_0, 9, 4, 8, 9, 5, 4, 1_1, 1_6, 9, 1_4, 1_3, 5, 1_4, 4, 2_4, 1_5, 1_6, 1_2, 4, 1_5, 7, 9, 2_1, 1_6, 7, 2_1, 5, 1_2, 4, 7, 9, 1_4, 4, 1_4, 5, 5, 2_4, 4, 1_0, 9, 6, 5, 1_3, 8, 2_4, 5, 1_3, 7, 2_5, 1_0, 1_5, 1_0, 6, 2_2, 4, 2_5, 5, 6, 2_0, 5, 5, 9, 4, 5_8, 7, 3_7, 2_3, 4, 4_9, 2_2, 3_2, 8, 1_3, 1_7, 1_1, 4, 7, 9, 1_4, 4, 3_2, 5, 9, 1_2, 8, 1_3, 5_5, 1_5, 8, 2_0, 2_6, 2],
							        [4, 4_0, 4_7, 5_4, 3_2, 4, 1_0, 1_2, 4, 1_4, 5, 1_2, 1_0, 2_1, 9, 5, 1_4, 4, 6, 8, 4, 2_4, 1_3, 5, 3_9, 6, 1_3, 7, 1_0, 9, 4, 1_4, 5, 5, 2_4, 4, 2_5, 1_0, 1_4, 1_0, 1_3, 5, 1_7, 6, 1_0, 8, 9, 7, 1_5, 4, 1_3, 5, 2_4, 1_3, 5, 1_2, 5, 9, 6, 7, 6, 1_0, 8, 9, 1_2, 4, 1_9, 1_3, 8, 1_8, 4, 1_6, 9, 1_5, 7, 2_5, 5, 1_5, 5, 1_4, 4, 6, 5, 3_7, 6, 4, 2_5, 2_2, 4, 4_6, 8, 1_0, 9, 6, 1_5, 2_2, 4, 1_7, 8, 9, 1_4, 1_0, 6, 1_0, 8, 9, 1_0, 9, 2_1, 4, 8, 9, 4, 2_5, 8, 6, 1_1, 4, 1_5, 5, 1_9, 6, 4, 7, 9, 1_4, 4, 1_3, 1_0, 2_1, 1_1, 6, 4, 1_7, 8, 9, 6, 5, 3_7, 6, 4, 1_0, 9, 4, 7, 1_5, 1_5, 4, 1_5, 7, 2_2, 5, 1_3, 1_2, 2_6, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
							        [4, 3_2, 1_1, 5, 4, 4_5, 1_6, 1_0, 1_7, 2_8, 4, 2_5, 1_3, 8, 2_0, 9, 4, 1_9, 8, 3_7, 4, 4_6, 1_6, 1_8, 2_4, 1_2, 4, 8, 2_7, 5, 1_3, 4, 6, 1_1, 5, 4, 1_5, 7, 5_7, 2_2, 4, 1_4, 8, 2_1, 2_6, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
							    ],
							    "attention_mask": [
							        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
							        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
							        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
							    ]
							}
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=a__							,    model_name="microsoft/speecht5_asr"							,    revision="c5ef64c71905caeccde0e4462ef3f9077224c524"							,    sequences=a__							,    )



'''simple docstring'''

import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMInverseScheduler,
    DDIMScheduler,
    DPMSolverMultistepInverseScheduler,
    DPMSolverMultistepScheduler,
    StableDiffusionDiffEditPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import load_image, slow
from diffusers.utils.testing_utils import enable_full_determinism, floats_tensor, require_torch_gpu, torch_device

from ..pipeline_params import TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS, TEXT_GUIDED_IMAGE_INPAINTING_PARAMS
from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()




class    lowerCAmelCase       (     UpperCamelCase_    ,  UpperCamelCase_    ,  unittest.TestCase  ):
 A_				:				int	=			StableDiffusionDiffEditPipeline
 A_				:				List[Any]	=			TEXT_GUIDED_IMAGE_INPAINTING_PARAMS - {"""height""", """width""", """image"""} | {"""image_latents"""}
 A_				:				str	=			TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS - {"""image"""} | {"""image_latents"""}
 A_				:				Dict	=			frozenset(
     []  )  # TO-DO: update image_params once pipeline is refactored with VaeImageProcessor.preprocess
 A_				:				Union[str, Any]	=			frozenset([]  )


 def    _A	(    self  : Tuple    ):
  '''simple docstring'''

  torch.manual_seed(0    )
  lowerCAmelCase__     :   str     		=	UNetaDConditionModel(
      block_out_channels=(32, 64)			,							layers_per_block=2			,							sample_size=32			,							in_channels=4			,							out_channels=4			,							down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")			,							up_block_types=("CrossAttnUpBlock2D", "UpBlock2D")			,							cross_attention_dim=32			,							attention_head_dim=(2, 4)			,							use_linear_projection=a__			,							)
  lowerCAmelCase__     :   str     		=	DDIMScheduler(
      beta_start=0.00085			,							beta_end=0.012			,							beta_schedule="scaled_linear"			,							clip_sample=a__			,							set_alpha_to_one=a__			,							)
  lowerCAmelCase__     :   str     		=	DDIMInverseScheduler(
      beta_start=0.00085			,							beta_end=0.012			,							beta_schedule="scaled_linear"			,							clip_sample=a__			,							set_alpha_to_zero=a__			,							)
  torch.manual_seed(0    )
  lowerCAmelCase__     :   int     		=	AutoencoderKL(
      block_out_channels=[32, 64]			,							in_channels=3			,							out_channels=3			,							down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"]			,							up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"]			,							latent_channels=4			,							sample_size=128			,							)
  torch.manual_seed(0    )
  lowerCAmelCase__     :   Dict     		=	CLIPTextConfig(
      bos_token_id=0			,							eos_token_id=2			,							hidden_size=32			,							intermediate_size=37			,							layer_norm_eps=1e-05			,							num_attention_heads=4			,							num_hidden_layers=5			,							pad_token_id=1			,							vocab_size=1000			,							hidden_act="gelu"			,							projection_dim=512			,							)
  lowerCAmelCase__     :   Tuple     		=	CLIPTextModel(a__    )
  lowerCAmelCase__     :   int     		=	CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"    )

  lowerCAmelCase__     :   List[Any]     		=	{
      "unet": unet,
      "scheduler": scheduler,
      "inverse_scheduler": inverse_scheduler,
      "vae": vae,
      "text_encoder": text_encoder,
      "tokenizer": tokenizer,
      "safety_checker": None,
      "feature_extractor": None,
  }

  return components


 def    _A	(    self  : Dict			,							a__  : List[Any]			,							a__  : str=0    ):
  '''simple docstring'''

  lowerCAmelCase__     :   Union[str, Any]     		=	floats_tensor((1, 16, 16)			,							rng=random.Random(a__    )    ).to(a__    )
  lowerCAmelCase__     :   Optional[int]     		=	floats_tensor((1, 2, 4, 16, 16)			,							rng=random.Random(a__    )    ).to(a__    )
  if str(a__    ).startswith("mps"    ):
   lowerCAmelCase__     :   Any     		=	torch.manual_seed(a__    )
  else:
   lowerCAmelCase__     :   int     		=	torch.Generator(device=a__    ).manual_seed(a__    )
  lowerCAmelCase__     :   Optional[Any]     		=	{
      "prompt": "a dog and a newt",
      "mask_image": mask,
      "image_latents": latents,
      "generator": generator,
      "num_inference_steps": 2,
      "inpaint_strength": 1.0,
      "guidance_scale": 6.0,
      "output_type": "numpy",
  }

  return inputs


 def    _A	(    self  : Union[str, Any]			,							a__  : Union[str, Any]			,							a__  : Tuple=0    ):
  '''simple docstring'''

  lowerCAmelCase__     :   Any     		=	floats_tensor((1, 3, 32, 32)			,							rng=random.Random(a__    )    ).to(a__    )
  lowerCAmelCase__     :   Optional[Any]     		=	image.cpu().permute(0			,							2			,							3			,							1    )[0]
  lowerCAmelCase__     :   Tuple     		=	Image.fromarray(np.uinta(a__    )    ).convert("RGB"    )
  if str(a__    ).startswith("mps"    ):
   lowerCAmelCase__     :   str     		=	torch.manual_seed(a__    )
  else:
   lowerCAmelCase__     :   List[str]     		=	torch.Generator(device=a__    ).manual_seed(a__    )
  lowerCAmelCase__     :   Dict     		=	{
      "image": image,
      "source_prompt": "a cat and a frog",
      "target_prompt": "a dog and a newt",
      "generator": generator,
      "num_inference_steps": 2,
      "num_maps_per_mask": 2,
      "mask_encode_strength": 1.0,
      "guidance_scale": 6.0,
      "output_type": "numpy",
  }

  return inputs


 def    _A	(    self  : Tuple			,							a__  : Any			,							a__  : Optional[Any]=0    ):
  '''simple docstring'''

  lowerCAmelCase__     :   List[Any]     		=	floats_tensor((1, 3, 32, 32)			,							rng=random.Random(a__    )    ).to(a__    )
  lowerCAmelCase__     :   List[str]     		=	image.cpu().permute(0			,							2			,							3			,							1    )[0]
  lowerCAmelCase__     :   Tuple     		=	Image.fromarray(np.uinta(a__    )    ).convert("RGB"    )
  if str(a__    ).startswith("mps"    ):
   lowerCAmelCase__     :   Dict     		=	torch.manual_seed(a__    )
  else:
   lowerCAmelCase__     :   Tuple     		=	torch.Generator(device=a__    ).manual_seed(a__    )
  lowerCAmelCase__     :   Dict     		=	{
      "image": image,
      "prompt": "a cat and a frog",
      "generator": generator,
      "num_inference_steps": 2,
      "inpaint_strength": 1.0,
      "guidance_scale": 6.0,
      "decode_latents": True,
      "output_type": "numpy",
  }
  return inputs


 def    _A	(    self  : List[str]    ):
  '''simple docstring'''

  if not hasattr(self.pipeline_class			,							"_optional_components"    ):
   return

  lowerCAmelCase__     :   int     		=	self.get_dummy_components()
  lowerCAmelCase__     :   Optional[Any]     		=	self.pipeline_class(**a__    )
  pipe.to(a__    )
  pipe.set_progress_bar_config(disable=a__    )

  # set all optional components to None and update pipeline config accordingly
  for optional_component in pipe._optional_components:
   setattr(a__			,							a__			,							a__    )
  pipe.register_modules(**{optional_component: None for optional_component in pipe._optional_components}    )

  lowerCAmelCase__     :   str     		=	self.get_dummy_inputs(a__    )
  lowerCAmelCase__     :   List[Any]     		=	pipe(**a__    )[0]

  with tempfile.TemporaryDirectory() as tmpdir:
   pipe.save_pretrained(a__    )
   lowerCAmelCase__     :   Tuple     		=	self.pipeline_class.from_pretrained(a__    )
   pipe_loaded.to(a__    )
   pipe_loaded.set_progress_bar_config(disable=a__    )

  for optional_component in pipe._optional_components:
   self.assertTrue(
       getattr(a__			,							a__    ) is None			,							F'''`{optional_component}` did not stay set to None after loading.'''			,							)

  lowerCAmelCase__     :   Union[str, Any]     		=	self.get_dummy_inputs(a__    )
  lowerCAmelCase__     :   int     		=	pipe_loaded(**a__    )[0]

  lowerCAmelCase__     :   Dict     		=	np.abs(output - output_loaded    ).max()
  self.assertLess(a__			,							1e-4    )


 def    _A	(    self  : str    ):
  '''simple docstring'''

  lowerCAmelCase__     :   Union[str, Any]     		=	"cpu"

  lowerCAmelCase__     :   Any     		=	self.get_dummy_components()
  lowerCAmelCase__     :   List[Any]     		=	self.pipeline_class(**a__    )
  pipe.to(a__    )
  pipe.set_progress_bar_config(disable=a__    )

  lowerCAmelCase__     :   Union[str, Any]     		=	self.get_dummy_mask_inputs(a__    )
  lowerCAmelCase__     :   List[str]     		=	pipe.generate_mask(**a__    )
  lowerCAmelCase__     :   Optional[Any]     		=	mask[0, -3:, -3:]

  self.assertEqual(mask.shape			,							(1, 16, 16)    )
  lowerCAmelCase__     :   str     		=	np.array([0] * 9    )
  lowerCAmelCase__     :   Dict     		=	np.abs(mask_slice.flatten() - expected_slice    ).max()
  self.assertLessEqual(a__			,							1e-3    )
  self.assertEqual(mask[0, -3, -4]			,							0    )


 def    _A	(    self  : List[str]    ):
  '''simple docstring'''

  lowerCAmelCase__     :   int     		=	"cpu"

  lowerCAmelCase__     :   Union[str, Any]     		=	self.get_dummy_components()
  lowerCAmelCase__     :   List[Any]     		=	self.pipeline_class(**a__    )
  pipe.to(a__    )
  pipe.set_progress_bar_config(disable=a__    )

  lowerCAmelCase__     :   List[str]     		=	self.get_dummy_inversion_inputs(a__    )
  lowerCAmelCase__     :   Optional[int]     		=	pipe.invert(**a__    ).images
  lowerCAmelCase__     :   Dict     		=	image[0, -1, -3:, -3:]

  self.assertEqual(image.shape			,							(2, 32, 32, 3)    )
  lowerCAmelCase__     :   Tuple     		=	np.array(
      [0.5150, 0.5134, 0.5043, 0.5376, 0.4694, 0.51050, 0.5015, 0.4407, 0.4799]			,							)
  lowerCAmelCase__     :   Union[str, Any]     		=	np.abs(image_slice.flatten() - expected_slice    ).max()
  self.assertLessEqual(a__			,							1e-3    )


 def    _A	(    self  : str    ):
  '''simple docstring'''

  super().test_inference_batch_single_identical(expected_max_diff=5e-3    )


 def    _A	(    self  : Optional[int]    ):
  '''simple docstring'''

  lowerCAmelCase__     :   Optional[int]     		=	"cpu"

  lowerCAmelCase__     :   Optional[int]     		=	self.get_dummy_components()

  lowerCAmelCase__     :   List[Any]     		=	{"beta_start": 0.00085, "beta_end": 0.012, "beta_schedule": "scaled_linear"}
  lowerCAmelCase__     :   List[Any]     		=	DPMSolverMultistepScheduler(**a__    )
  lowerCAmelCase__     :   List[Any]     		=	DPMSolverMultistepInverseScheduler(**a__    )

  lowerCAmelCase__     :   Any     		=	self.pipeline_class(**a__    )
  pipe.to(a__    )
  pipe.set_progress_bar_config(disable=a__    )

  lowerCAmelCase__     :   Dict     		=	self.get_dummy_inversion_inputs(a__    )
  lowerCAmelCase__     :   Optional[Any]     		=	pipe.invert(**a__    ).images
  lowerCAmelCase__     :   Any     		=	image[0, -1, -3:, -3:]

  self.assertEqual(image.shape			,							(2, 32, 32, 3)    )
  lowerCAmelCase__     :   Dict     		=	np.array(
      [0.5150, 0.5134, 0.5043, 0.5376, 0.4694, 0.51050, 0.5015, 0.4407, 0.4799]			,							)
  lowerCAmelCase__     :   int     		=	np.abs(image_slice.flatten() - expected_slice    ).max()
  self.assertLessEqual(a__			,							1e-3    )




@require_torch_gpu
@slow
class    lowerCAmelCase       (     unittest.TestCase  ):


 def    _A	(    self  : int    ):
  '''simple docstring'''

  super().tearDown()
  gc.collect()
  torch.cuda.empty_cache()


 @classmethod
 def    _A	(    cls  : Dict    ):
  '''simple docstring'''

  lowerCAmelCase__     :   Any     		=	load_image(
      "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/diffedit/fruit.png"    )

  lowerCAmelCase__     :   Dict     		=	raw_image.convert("RGB"    ).resize((768, 768)    )

  lowerCAmelCase__     :   str     		=	raw_image


 def    _A	(    self  : Dict    ):
  '''simple docstring'''

  lowerCAmelCase__     :   List[str]     		=	torch.manual_seed(0    )

  lowerCAmelCase__     :   str     		=	StableDiffusionDiffEditPipeline.from_pretrained(
      "stabilityai/stable-diffusion-2-1"			,							safety_checker=a__			,							torch_dtype=torch.floataa    )
  lowerCAmelCase__     :   Optional[Any]     		=	DDIMScheduler.from_config(pipe.scheduler.config    )
  lowerCAmelCase__     :   int     		=	DDIMInverseScheduler.from_config(pipe.scheduler.config    )
  pipe.enable_model_cpu_offload()
  pipe.set_progress_bar_config(disable=a__    )

  lowerCAmelCase__     :   Optional[Any]     		=	"a bowl of fruit"
  lowerCAmelCase__     :   Union[str, Any]     		=	"a bowl of pears"

  lowerCAmelCase__     :   Optional[Any]     		=	pipe.generate_mask(
      image=self.raw_image			,							source_prompt=a__			,							target_prompt=a__			,							generator=a__			,							)

  lowerCAmelCase__     :   str     		=	pipe.invert(
      prompt=a__			,							image=self.raw_image			,							inpaint_strength=0.7			,							generator=a__    ).latents

  lowerCAmelCase__     :   Tuple     		=	pipe(
      prompt=a__			,							mask_image=a__			,							image_latents=a__			,							generator=a__			,							negative_prompt=a__			,							inpaint_strength=0.7			,							output_type="numpy"			,							).images[0]

  lowerCAmelCase__     :   List[str]     		=	(
      np.array(
          load_image(
              "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
              "/diffedit/pears.png"    ).resize((768, 768)    )    )
      / 255
  )
  assert np.abs((expected_image - image).max()    ) < 5e-1


 def    _A	(    self  : Optional[Any]    ):
  '''simple docstring'''

  lowerCAmelCase__     :   Tuple     		=	torch.manual_seed(0    )

  lowerCAmelCase__     :   str     		=	StableDiffusionDiffEditPipeline.from_pretrained(
      "stabilityai/stable-diffusion-2-1"			,							safety_checker=a__			,							torch_dtype=torch.floataa    )
  lowerCAmelCase__     :   int     		=	DPMSolverMultistepScheduler.from_config(pipe.scheduler.config    )
  lowerCAmelCase__     :   Any     		=	DPMSolverMultistepInverseScheduler.from_config(pipe.scheduler.config    )
  pipe.enable_model_cpu_offload()
  pipe.set_progress_bar_config(disable=a__    )

  lowerCAmelCase__     :   Any     		=	"a bowl of fruit"
  lowerCAmelCase__     :   Optional[int]     		=	"a bowl of pears"

  lowerCAmelCase__     :   Optional[Any]     		=	pipe.generate_mask(
      image=self.raw_image			,							source_prompt=a__			,							target_prompt=a__			,							generator=a__			,							)

  lowerCAmelCase__     :   Optional[int]     		=	pipe.invert(
      prompt=a__			,							image=self.raw_image			,							inpaint_strength=0.7			,							generator=a__			,							num_inference_steps=25			,							).latents

  lowerCAmelCase__     :   Any     		=	pipe(
      prompt=a__			,							mask_image=a__			,							image_latents=a__			,							generator=a__			,							negative_prompt=a__			,							inpaint_strength=0.7			,							num_inference_steps=25			,							output_type="numpy"			,							).images[0]

  lowerCAmelCase__     :   Optional[int]     		=	(
      np.array(
          load_image(
              "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
              "/diffedit/pears.png"    ).resize((768, 768)    )    )
      / 255
  )
  assert np.abs((expected_image - image).max()    ) < 5e-1





