




'''simple docstring'''





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase_		:     Union[str, Any]          =		get_tests_dir('''fixtures/test_sentencepiece.model''')

@require_sentencepiece
@require_tokenizers
class   _a							(		SCREAMING_SNAKE_CASE_   ,   unittest.TestCase				):
     SCREAMING_SNAKE_CASE_				:       Union[str, Any]       							=		XLNetTokenizer
     SCREAMING_SNAKE_CASE_				:       Any       							=		XLNetTokenizerFast
     SCREAMING_SNAKE_CASE_				:       List[str]       							=		True
     SCREAMING_SNAKE_CASE_				:       int       							=		True






     def 							_lowercase      (					self       )				->     Union[str, Any]:
          super().setUp()

          # We have a SentencePiece fixture for testing
          _snake_case				= XLNetTokenizer(_SCREAMING_SNAKE_CASE   ,keep_accents=_SCREAMING_SNAKE_CASE       )
          tokenizer.sanitize_special_tokens()
          tokenizer.save_pretrained(self.tmpdirname       )






     def 							_lowercase      (					self       )				->     str:
          _snake_case				= '<s>'
          _snake_case				= 1

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE       )   ,_SCREAMING_SNAKE_CASE       )
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE       )   ,_SCREAMING_SNAKE_CASE       )






     def 							_lowercase      (					self       )				->     Optional[int]:
          _snake_case				= list(self.get_tokenizer().get_vocab().keys()       )

          self.assertEqual(vocab_keys[0]   ,"<unk>"       )
          self.assertEqual(vocab_keys[1]   ,"<s>"       )
          self.assertEqual(vocab_keys[-1]   ,"<eod>"       )
          self.assertEqual(len(_SCREAMING_SNAKE_CASE       )   ,1_006       )






     def 							_lowercase      (					self       )				->     Any:
          self.assertEqual(self.get_tokenizer().vocab_size   ,1_000       )






     def 							_lowercase      (					self       )				->     List[Any]:
          _snake_case				= XLNetTokenizer(_SCREAMING_SNAKE_CASE   ,keep_accents=_SCREAMING_SNAKE_CASE       )

          _snake_case				= tokenizer.tokenize("This is a test"       )
          self.assertListEqual(_SCREAMING_SNAKE_CASE   ,["▁This", "▁is", "▁a", "▁t", "est"]       )

          self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE       )   ,[285, 46, 10, 170, 382]       )

          _snake_case				= tokenizer.tokenize("I was born in 92000, and this is falsé."       )
          self.assertListEqual(
              _SCREAMING_SNAKE_CASE   ,[
                  SPIECE_UNDERLINE + "I",
                  SPIECE_UNDERLINE + "was",
                  SPIECE_UNDERLINE + "b",
                  "or",
                  "n",
                  SPIECE_UNDERLINE + "in",
                  SPIECE_UNDERLINE + "",
                  "9",
                  "2",
                  "0",
                  "0",
                  "0",
                  ",",
                  SPIECE_UNDERLINE + "and",
                  SPIECE_UNDERLINE + "this",
                  SPIECE_UNDERLINE + "is",
                  SPIECE_UNDERLINE + "f",
                  "al",
                  "s",
                  "é",
                  ".",
              ]   ,)
          _snake_case				= tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE       )
          self.assertListEqual(_SCREAMING_SNAKE_CASE   ,[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]       )

          _snake_case				= tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE       )
          self.assertListEqual(
              _SCREAMING_SNAKE_CASE   ,[
                  SPIECE_UNDERLINE + "I",
                  SPIECE_UNDERLINE + "was",
                  SPIECE_UNDERLINE + "b",
                  "or",
                  "n",
                  SPIECE_UNDERLINE + "in",
                  SPIECE_UNDERLINE + "",
                  "<unk>",
                  "2",
                  "0",
                  "0",
                  "0",
                  ",",
                  SPIECE_UNDERLINE + "and",
                  SPIECE_UNDERLINE + "this",
                  SPIECE_UNDERLINE + "is",
                  SPIECE_UNDERLINE + "f",
                  "al",
                  "s",
                  "<unk>",
                  ".",
              ]   ,)






     def 							_lowercase      (					self       )				->     Optional[int]:
          _snake_case				= XLNetTokenizer(_SCREAMING_SNAKE_CASE   ,do_lower_case=_SCREAMING_SNAKE_CASE       )
          _snake_case				= tokenizer.tokenize("I was born in 92000, and this is falsé."       )
          self.assertListEqual(
              _SCREAMING_SNAKE_CASE   ,[
                  SPIECE_UNDERLINE + "",
                  "i",
                  SPIECE_UNDERLINE + "was",
                  SPIECE_UNDERLINE + "b",
                  "or",
                  "n",
                  SPIECE_UNDERLINE + "in",
                  SPIECE_UNDERLINE + "",
                  "9",
                  "2",
                  "0",
                  "0",
                  "0",
                  ",",
                  SPIECE_UNDERLINE + "and",
                  SPIECE_UNDERLINE + "this",
                  SPIECE_UNDERLINE + "is",
                  SPIECE_UNDERLINE + "f",
                  "al",
                  "se",
                  ".",
              ]   ,)
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )   ,["▁he", "ll", "o"]       )






     def 							_lowercase      (					self       )				->     Optional[int]:
          _snake_case				= XLNetTokenizer(_SCREAMING_SNAKE_CASE   ,do_lower_case=_SCREAMING_SNAKE_CASE       )
          _snake_case				= tokenizer.tokenize("I was born in 92000, and this is falsé."       )
          self.assertListEqual(
              _SCREAMING_SNAKE_CASE   ,[
                  SPIECE_UNDERLINE + "I",
                  SPIECE_UNDERLINE + "was",
                  SPIECE_UNDERLINE + "b",
                  "or",
                  "n",
                  SPIECE_UNDERLINE + "in",
                  SPIECE_UNDERLINE + "",
                  "9",
                  "2",
                  "0",
                  "0",
                  "0",
                  ",",
                  SPIECE_UNDERLINE + "and",
                  SPIECE_UNDERLINE + "this",
                  SPIECE_UNDERLINE + "is",
                  SPIECE_UNDERLINE + "f",
                  "al",
                  "se",
                  ".",
              ]   ,)






     @slow
     def 							_lowercase      (					self       )				->     Any:
          _snake_case				= XLNetTokenizer.from_pretrained("xlnet-base-cased"       )

          _snake_case				= tokenizer.encode("sequence builders"   ,add_special_tokens=_SCREAMING_SNAKE_CASE       )
          _snake_case				= tokenizer.encode("multi-sequence build"   ,add_special_tokens=_SCREAMING_SNAKE_CASE       )

          _snake_case				= tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE       )
          _snake_case				= tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       )

          assert encoded_sentence == text + [4, 3]
          assert encoded_pair == text + [4] + text_a + [4, 3]






     @slow
     def 							_lowercase      (					self       )				->     int:
          _snake_case				= {'input_ids': [[17, 21_442, 270, 17, 10, 14_645, 318, 34, 17, 4_546, 3_145, 787, 13, 7_752, 22_018, 23, 21, 17, 4_546, 3_145, 787, 13, 3_352, 14_431, 13, 5_500, 11, 1_176, 580, 13, 16_819, 4_797, 23, 17, 10, 17_135, 658, 19, 457, 7_932, 13, 184, 19, 3_154, 17_135, 6_468, 19, 1_404, 12_269, 19, 4_229, 5_356, 16_264, 46, 19, 17, 20_545, 10_395, 9, 9, 9, 11, 28, 6_421, 9_531, 20_729, 17, 10, 353, 17_022, 11, 21, 6_421, 9_531, 16_949, 17, 10, 11_509, 753, 11, 33, 95, 2_421, 7_385, 956, 14_431, 2_626, 25, 842, 7_385, 4_836, 21, 1_429, 2_272, 9_855, 3_120, 161, 24_738, 19, 13_203, 658, 218, 787, 21, 430, 18_482, 847, 2_637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22_178, 27, 1_064, 22, 956, 13, 11_101, 1_429, 5_854, 24_313, 18_953, 40, 422, 24_366, 68, 1_758, 37, 10_483, 14_257, 31, 207, 263, 21, 203, 3_773, 25, 71, 9_735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2_049, 3_442, 17, 13_894, 3_380, 23, 95, 18, 17_634, 2_288, 9, 4, 3]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=_SCREAMING_SNAKE_CASE   ,model_name="xlnet-base-cased"   ,revision="c841166438c31ec7ca9a106dee7bb312b73ae511"   ,)







import gc
import tempfile
import unittest

import numpy as np
import torch

from diffusers import VersatileDiffusionPipeline
from diffusers.utils.testing_utils import load_image, nightly, require_torch_gpu, torch_device


_lowerCamelCase    :		Any		      =    False


class 						lowercase    (							unittest.TestCase):
						'''simple docstring'''


						pass


@nightly
@require_torch_gpu
class 						lowercase    (							unittest.TestCase):
						'''simple docstring'''


						def       lowerCamelCase_				(  self    :				List[Any]	):



											'''simple docstring'''


											super().tearDown()
											gc.collect()
											torch.cuda.empty_cache()







						def       lowerCamelCase_				(  self    :				List[str]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Dict						=				VersatileDiffusionPipeline.from_pretrained('shi-labs/versatile-diffusion'       ,	torch_dtype=torch.floataa	)
											pipe.to(snake_case	)
											pipe.set_progress_bar_config(disable=snake_case	)

											SCREAMING_SNAKE_CASE							:     Dict						=				load_image(
											    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/versatile_diffusion/benz.jpg'	)

											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				torch.manual_seed(0	)
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				pipe.dual_guided(
											    prompt='first prompt'       ,	image=snake_case       ,	text_to_image_strength=0.75       ,	generator=snake_case       ,	guidance_scale=7.5       ,	num_inference_steps=2       ,	output_type='numpy'       ,	).images

											with tempfile.TemporaryDirectory() as tmpdirname:
																pipe.save_pretrained(snake_case	)
																SCREAMING_SNAKE_CASE							:     Any						=				VersatileDiffusionPipeline.from_pretrained(snake_case       ,	torch_dtype=torch.floataa	)
											pipe.to(snake_case	)
											pipe.set_progress_bar_config(disable=snake_case	)

											SCREAMING_SNAKE_CASE							:     List[str]						=				generator.manual_seed(0	)
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				pipe.dual_guided(
											    prompt='first prompt'       ,	image=snake_case       ,	text_to_image_strength=0.75       ,	generator=snake_case       ,	guidance_scale=7.5       ,	num_inference_steps=2       ,	output_type='numpy'       ,	).images

											assert np.abs(image - new_image	).sum() < 1E-5, "Models don't have the same forward pass"







						def       lowerCamelCase_				(  self    :				Tuple	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Dict						=				VersatileDiffusionPipeline.from_pretrained('shi-labs/versatile-diffusion'       ,	torch_dtype=torch.floataa	)
											pipe.to(snake_case	)
											pipe.set_progress_bar_config(disable=snake_case	)

											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				'cyberpunk 2077'
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				load_image(
											    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/versatile_diffusion/benz.jpg'	)
											SCREAMING_SNAKE_CASE							:     List[Any]						=				torch.manual_seed(0	)
											SCREAMING_SNAKE_CASE							:     Any						=				pipe.dual_guided(
											    prompt=snake_case       ,	image=snake_case       ,	text_to_image_strength=0.75       ,	generator=snake_case       ,	guidance_scale=7.5       ,	num_inference_steps=50       ,	output_type='numpy'       ,	).images

											SCREAMING_SNAKE_CASE							:     List[Any]						=				image[0, 253:256, 253:256, -1]

											assert image.shape == (1, 512, 512, 3)
											SCREAMING_SNAKE_CASE							:     Tuple						=				np.array([0.1448, 0.1619, 0.1741, 0.1086, 0.1147, 0.1128, 0.1199, 0.1165, 0.1001]	)

											assert np.abs(image_slice.flatten() - expected_slice	).max() < 1E-1

											SCREAMING_SNAKE_CASE							:     Any						=				'A painting of a squirrel eating a burger '
											SCREAMING_SNAKE_CASE							:     Any						=				torch.manual_seed(0	)
											SCREAMING_SNAKE_CASE							:     Tuple						=				pipe.text_to_image(
											    prompt=snake_case       ,	generator=snake_case       ,	guidance_scale=7.5       ,	num_inference_steps=50       ,	output_type='numpy'	).images

											SCREAMING_SNAKE_CASE							:     Dict						=				image[0, 253:256, 253:256, -1]

											assert image.shape == (1, 512, 512, 3)
											SCREAMING_SNAKE_CASE							:     List[Any]						=				np.array([0.3367, 0.3169, 0.2656, 0.3870, 0.4790, 0.3796, 0.4009, 0.4878, 0.4778]	)

											assert np.abs(image_slice.flatten() - expected_slice	).max() < 1E-1

											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				pipe.image_variation(snake_case       ,	generator=snake_case       ,	output_type='numpy'	).images

											SCREAMING_SNAKE_CASE							:     Optional[int]						=				image[0, 253:256, 253:256, -1]

											assert image.shape == (1, 512, 512, 3)
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				np.array([0.3076, 0.3123, 0.3284, 0.3782, 0.3770, 0.3894, 0.4297, 0.4331, 0.4456]	)

											assert np.abs(image_slice.flatten() - expected_slice	).max() < 1E-1