






"""simple docstring"""





import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCamelCase							=  get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model')



@require_sentencepiece
@require_tokenizers
class       __A		(   SCREAMING_SNAKE_CASE_     ,unittest.TestCase			):
					UpperCAmelCase__								=				GPTSwaTokenizer
					UpperCAmelCase__								=				False
					UpperCAmelCase__								=				True
					UpperCAmelCase__								=				False

					def   lowerCamelCase__   (			self  :      int					)			->							List[Any]:
											super().setUp()

											# We have a SentencePiece fixture for testing
											__magic_name__:	Any							=       GPTSwaTokenizer(__snake_case     ,		eos_token="""<unk>"""     ,		bos_token="""<unk>"""     ,		pad_token="""<unk>"""					)

											tokenizer.save_pretrained(self.tmpdirname					)

					def   lowerCamelCase__   (			self  :      List[Any]     ,		__snake_case  :      List[Any]					)			->							Optional[int]:
											__magic_name__:	Tuple							=       """This is a test"""
											__magic_name__:	Dict							=       """This is a test"""
											return input_text, output_text

					def   lowerCamelCase__   (			self  :      Dict					)			->							str:
											__magic_name__:	Union[str, Any]							=       """<s>"""
											__magic_name__:	Any							=       1

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case					)     ,		__snake_case					)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case					)     ,		__snake_case					)

					def   lowerCamelCase__   (			self  :      Tuple					)			->							Optional[int]:
											__magic_name__:	Any							=       list(self.get_tokenizer().get_vocab().keys()					)

											self.assertEqual(vocab_keys[0]     ,		"""<unk>"""					)
											self.assertEqual(vocab_keys[1]     ,		"""<s>"""					)
											self.assertEqual(vocab_keys[-1]     ,		"""j"""					)
											self.assertEqual(len(__snake_case					)     ,		2_0_0_0					)

					def   lowerCamelCase__   (			self  :      Any					)			->							Union[str, Any]:
											self.assertEqual(self.get_tokenizer().vocab_size     ,		2_0_0_0					)

					def   lowerCamelCase__   (			self  :      List[str]					)			->							List[str]:
											__magic_name__:	List[Any]							=       GPTSwaTokenizer(__snake_case					)

											__magic_name__:	str							=       tokenizer.tokenize("""This is a test"""					)
											self.assertListEqual(__snake_case     ,		["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]					)

											self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case					)     ,		[4_6_5, 2_8_7, 2_6_5, 6_3_1, 8_4_2]					)

											__magic_name__:	Optional[Any]							=       tokenizer.tokenize("""I was born in 92000, and this is falsé."""					)
											# fmt: off
											self.assertListEqual(
											    __snake_case     ,		["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]     ,		)
											# fmt: on

											__magic_name__:	List[Any]							=       tokenizer.convert_tokens_to_ids(__snake_case					)
											self.assertListEqual(
											    __snake_case     ,		[2_6_2, 2_7_2, 1_5_2_5, 2_8_6, 2_7_1, 2_6_8, 6_0, 9_1_6, 6_3_3, 6_3_3, 6_3_3, 2_5_9, 2_6_6, 3_0_1, 2_8_7, 3_8_4, 3_6_7, 2_6_3, 1_9_8, 1_7_2, 2_6_0]     ,		)

											__magic_name__:	Tuple							=       tokenizer.convert_ids_to_tokens(__snake_case					)
											# fmt: off
											self.assertListEqual(
											    __snake_case     ,		["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]					)
											# fmt: on

					def   lowerCamelCase__   (			self  :      Tuple					)			->							Union[str, Any]:
											__magic_name__:	Tuple							=       GPTSwaTokenizer(__snake_case					)
											__magic_name__:	Any							=       ["""This is a test""", """I was born in 92000, and this is falsé."""]
											__magic_name__:	Dict							=       [
											    [4_6_5, 2_8_7, 2_6_5, 6_3_1, 8_4_2],
											    [2_6_2, 2_7_2, 1_5_2_5, 2_8_6, 2_7_1, 2_6_8, 6_0, 9_1_6, 6_3_3, 6_3_3, 6_3_3, 2_5_9, 2_6_6, 3_0_1, 2_8_7, 3_8_4, 3_6_7, 2_6_3, 1_9_8, 1_7_2, 2_6_0],
											]

											# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
											for text, expected_ids in zip(__snake_case     ,		__snake_case					):
																	self.assertListEqual(tokenizer.encode_fast(__snake_case					)     ,		__snake_case					)

											# Test that decode_fast returns the input text
											for text, token_ids in zip(__snake_case     ,		__snake_case					):
																	self.assertEqual(tokenizer.decode_fast(__snake_case					)     ,		__snake_case					)






					@slow
					def   lowerCamelCase__   (			self  :      List[str]					)			->							Optional[Any]:
											__magic_name__:	Tuple							=       [
											    """<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')""",
											    """Hey there, how are you doing this fine day?""",
											    """This is a text with a trailing spaces followed by a dot     .""",
											    """Häj sväjs lillebrör! =)""",
											    """Det är inget fel på Mr. Cool""",
											]

											# fmt: off
											__magic_name__:	str							=       {"""input_ids""": [[6_3_4_2_3, 5, 6_8_1_1, 1_4_9_5_4, 2_8_2, 8_1_6, 3_8_2_1, 6_3_4_6_6, 6_3_4_2_5, 6_3_4_6_2, 1_8, 6_3_9_7_8, 6_7_8, 3_0_1, 1_3_2_0, 6_3_4_2_3, 6_3_4_5_5, 6_3_4_5_8, 1_8, 6_3_9_8_2, 4_2_4_6, 3_9_4_0, 1_9_0_1, 4_7_7_8_9, 5_5_4_7, 1_8_9_9_4], [1_9_6_3_0, 1_1_0_0, 6_3_4_4_6, 1_3_4_2, 6_3_3, 5_4_4, 4_4_8_8, 5_9_3, 5_1_0_2, 2_4_1_6, 6_3_4_9_5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_6_5_2, 4_2_8, 2_6_8, 1_9_3_6, 5_1_5, 2_6_8, 5_8_5_9_3, 2_2_4_1_3, 9_1_0_6, 5_4_6, 2_6_8, 3_3_2_1_3, 6_3_9_7_9, 6_9_8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5_1_3_0, 6_3_4_5_0, 9_2_4, 6_3_4_4_9, 2_2_4_9, 4_0_6_2, 1_5_5_8, 3_1_8, 6_3_5_0_4, 2_1_4_9_8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_0_9, 3_7_7, 2_8_2_7, 2_5_5_9, 3_3_2, 6_5_7_5, 6_3_4_4_3, 2_6_8_0_1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
											# fmt: on
											self.tokenizer_integration_test_util(
											    expected_encoding=__snake_case     ,		model_name="""AI-Sweden/gpt-sw3-126m"""     ,		sequences=__snake_case     ,		)









'''simple docstring'''







import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, DDPMScheduler, StableDiffusionUpscalePipeline, UNetaDConditionModel
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu


enable_full_determinism()

class 	_snake_case(			unittest.TestCase      ):






 def 						_UpperCamelCase (self       :  List[str]    )       ->   str:






    """simple docstring"""


    super().tearDown()
    gc.collect()
    torch.cuda.empty_cache()






 @property
 def 						_UpperCamelCase (self       :  int    )       ->   Any:






    """simple docstring"""


    A__       =     1
    A__       =     3
    A__       =     (32, 32)

    A__       =     floats_tensor((batch_size, num_channels) + sizes    ,		rng=random.Random(0    )    ).to(a    )
    return image






 @property
 def 						_UpperCamelCase (self       :  Union[str, Any]    )       ->   Tuple:






    """simple docstring"""


    torch.manual_seed(0    )
    A__       =     UNetaDConditionModel(
        block_out_channels=(32, 32, 64)    ,		layers_per_block=2    ,		sample_size=32    ,		in_channels=7    ,		out_channels=4    ,		down_block_types=('DownBlock2D', 'CrossAttnDownBlock2D', 'CrossAttnDownBlock2D')    ,		up_block_types=('CrossAttnUpBlock2D', 'CrossAttnUpBlock2D', 'UpBlock2D')    ,		cross_attention_dim=32    ,		attention_head_dim=8    ,		use_linear_projection=a    ,		only_cross_attention=(True, True, False)    ,		num_class_embeds=1_00    ,		)
    return model






 @property
 def 						_UpperCamelCase (self       :  Tuple    )       ->   Optional[Any]:






    """simple docstring"""


    torch.manual_seed(0    )
    A__       =     AutoencoderKL(
        block_out_channels=[32, 32, 64]    ,		in_channels=3    ,		out_channels=3    ,		down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D', 'DownEncoderBlock2D']    ,		up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D', 'UpDecoderBlock2D']    ,		latent_channels=4    ,		)
    return model






 @property
 def 						_UpperCamelCase (self       :  str    )       ->   List[str]:






    """simple docstring"""


    torch.manual_seed(0    )
    A__       =     CLIPTextConfig(
        bos_token_id=0    ,		eos_token_id=2    ,		hidden_size=32    ,		intermediate_size=37    ,		layer_norm_eps=1e-05    ,		num_attention_heads=4    ,		num_hidden_layers=5    ,		pad_token_id=1    ,		vocab_size=10_00    ,		hidden_act='gelu'    ,		projection_dim=5_12    ,		)
    return CLIPTextModel(a    )






 def 						_UpperCamelCase (self       :  str    )       ->   List[str]:






    """simple docstring"""


    A__       =     'cpu'  # ensure determinism for the device-dependent torch.Generator
    A__       =     self.dummy_cond_unet_upscale
    A__       =     DDPMScheduler()
    A__       =     DDIMScheduler(prediction_type='v_prediction'    )
    A__       =     self.dummy_vae
    A__       =     self.dummy_text_encoder
    A__       =     CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'    )

    A__       =     self.dummy_image.cpu().permute(0    ,		2    ,		3    ,		1    )[0]
    A__       =     Image.fromarray(np.uinta(a    )    ).convert('RGB'    ).resize((64, 64)    )

    # make sure here that pndm scheduler skips prk
    A__       =     StableDiffusionUpscalePipeline(
        unet=a    ,		low_res_scheduler=a    ,		scheduler=a    ,		vae=a    ,		text_encoder=a    ,		tokenizer=a    ,		max_noise_level=3_50    ,		)
    A__       =     sd_pipe.to(a    )
    sd_pipe.set_progress_bar_config(disable=a    )

    A__       =     'A painting of a squirrel eating a burger'
    A__       =     torch.Generator(device=a    ).manual_seed(0    )
    A__       =     sd_pipe(
        [prompt]    ,		image=a    ,		generator=a    ,		guidance_scale=6.0    ,		noise_level=20    ,		num_inference_steps=2    ,		output_type='np'    ,		)

    A__       =     output.images

    A__       =     torch.Generator(device=a    ).manual_seed(0    )
    A__       =     sd_pipe(
        [prompt]    ,		image=a    ,		generator=a    ,		guidance_scale=6.0    ,		noise_level=20    ,		num_inference_steps=2    ,		output_type='np'    ,		return_dict=a    ,		)[0]

    A__       =     image[0, -3:, -3:, -1]
    A__       =     image_from_tuple[0, -3:, -3:, -1]

    A__       =     low_res_image.size[0] * 4
    assert image.shape == (1, expected_height_width, expected_height_width, 3)
    A__       =     np.array([0.3113, 0.3910, 0.4272, 0.4859, 0.5061, 0.4652, 0.5362, 0.5715, 0.5661]    )

    assert np.abs(image_slice.flatten() - expected_slice    ).max() < 1e-2
    assert np.abs(image_from_tuple_slice.flatten() - expected_slice    ).max() < 1e-2






 def 						_UpperCamelCase (self       :  Optional[int]    )       ->   Optional[int]:






    """simple docstring"""


    A__       =     'cpu'  # ensure determinism for the device-dependent torch.Generator
    A__       =     self.dummy_cond_unet_upscale
    A__       =     DDPMScheduler()
    A__       =     DDIMScheduler(prediction_type='v_prediction'    )
    A__       =     self.dummy_vae
    A__       =     self.dummy_text_encoder
    A__       =     CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'    )

    A__       =     self.dummy_image.cpu().permute(0    ,		2    ,		3    ,		1    )[0]
    A__       =     Image.fromarray(np.uinta(a    )    ).convert('RGB'    ).resize((64, 64)    )

    # make sure here that pndm scheduler skips prk
    A__       =     StableDiffusionUpscalePipeline(
        unet=a    ,		low_res_scheduler=a    ,		scheduler=a    ,		vae=a    ,		text_encoder=a    ,		tokenizer=a    ,		max_noise_level=3_50    ,		)
    A__       =     sd_pipe.to(a    )
    sd_pipe.set_progress_bar_config(disable=a    )

    A__       =     'A painting of a squirrel eating a burger'
    A__       =     sd_pipe(
        2 * [prompt]    ,		image=2 * [low_res_image]    ,		guidance_scale=6.0    ,		noise_level=20    ,		num_inference_steps=2    ,		output_type='np'    ,		)
    A__       =     output.images
    assert image.shape[0] == 2

    A__       =     torch.Generator(device=a    ).manual_seed(0    )
    A__       =     sd_pipe(
        [prompt]    ,		image=a    ,		generator=a    ,		num_images_per_prompt=2    ,		guidance_scale=6.0    ,		noise_level=20    ,		num_inference_steps=2    ,		output_type='np'    ,		)
    A__       =     output.images
    assert image.shape[0] == 2






 @unittest.skipIf(torch_device != 'cuda'    ,		'This test requires a GPU'    )
 def 						_UpperCamelCase (self       :  List[Any]    )       ->   Union[str, Any]:






    """simple docstring"""


    A__       =     self.dummy_cond_unet_upscale
    A__       =     DDPMScheduler()
    A__       =     DDIMScheduler(prediction_type='v_prediction'    )
    A__       =     self.dummy_vae
    A__       =     self.dummy_text_encoder
    A__       =     CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'    )

    A__       =     self.dummy_image.cpu().permute(0    ,		2    ,		3    ,		1    )[0]
    A__       =     Image.fromarray(np.uinta(a    )    ).convert('RGB'    ).resize((64, 64)    )

    # put models in fp16, except vae as it overflows in fp16
    A__       =     unet.half()
    A__       =     text_encoder.half()

    # make sure here that pndm scheduler skips prk
    A__       =     StableDiffusionUpscalePipeline(
        unet=a    ,		low_res_scheduler=a    ,		scheduler=a    ,		vae=a    ,		text_encoder=a    ,		tokenizer=a    ,		max_noise_level=3_50    ,		)
    A__       =     sd_pipe.to(a    )
    sd_pipe.set_progress_bar_config(disable=a    )

    A__       =     'A painting of a squirrel eating a burger'
    A__       =     torch.manual_seed(0    )
    A__       =     sd_pipe(
        [prompt]    ,		image=a    ,		generator=a    ,		num_inference_steps=2    ,		output_type='np'    ,		).images

    A__       =     low_res_image.size[0] * 4
    assert image.shape == (1, expected_height_width, expected_height_width, 3)




@slow
@require_torch_gpu
class 	_snake_case(			unittest.TestCase      ):






 def 						_UpperCamelCase (self       :  Union[str, Any]    )       ->   Dict:






    """simple docstring"""


    super().tearDown()
    gc.collect()
    torch.cuda.empty_cache()






 def 						_UpperCamelCase (self       :  Optional[Any]    )       ->   Optional[int]:






    """simple docstring"""


    A__       =     load_image(
        'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
        '/sd2-upscale/low_res_cat.png'    )
    A__       =     load_numpy(
        'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale'
        '/upsampled_cat.npy'    )

    A__       =     'stabilityai/stable-diffusion-x4-upscaler'
    A__       =     StableDiffusionUpscalePipeline.from_pretrained(a    )
    pipe.to(a    )
    pipe.set_progress_bar_config(disable=a    )
    pipe.enable_attention_slicing()

    A__       =     'a cat sitting on a park bench'

    A__       =     torch.manual_seed(0    )
    A__       =     pipe(
        prompt=a    ,		image=a    ,		generator=a    ,		output_type='np'    ,		)
    A__       =     output.images[0]

    assert image.shape == (5_12, 5_12, 3)
    assert np.abs(expected_image - image    ).max() < 1e-3






 def 						_UpperCamelCase (self       :  Any    )       ->   str:






    """simple docstring"""


    A__       =     load_image(
        'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
        '/sd2-upscale/low_res_cat.png'    )
    A__       =     load_numpy(
        'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale'
        '/upsampled_cat_fp16.npy'    )

    A__       =     'stabilityai/stable-diffusion-x4-upscaler'
    A__       =     StableDiffusionUpscalePipeline.from_pretrained(
        a    ,		torch_dtype=torch.floataa    ,		)
    pipe.to(a    )
    pipe.set_progress_bar_config(disable=a    )
    pipe.enable_attention_slicing()

    A__       =     'a cat sitting on a park bench'

    A__       =     torch.manual_seed(0    )
    A__       =     pipe(
        prompt=a    ,		image=a    ,		generator=a    ,		output_type='np'    ,		)
    A__       =     output.images[0]

    assert image.shape == (5_12, 5_12, 3)
    assert np.abs(expected_image - image    ).max() < 5e-1






 def 						_UpperCamelCase (self       :  Dict    )       ->   List[str]:






    """simple docstring"""


    torch.cuda.empty_cache()
    torch.cuda.reset_max_memory_allocated()
    torch.cuda.reset_peak_memory_stats()

    A__       =     load_image(
        'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
        '/sd2-upscale/low_res_cat.png'    )

    A__       =     'stabilityai/stable-diffusion-x4-upscaler'
    A__       =     StableDiffusionUpscalePipeline.from_pretrained(
        a    ,		torch_dtype=torch.floataa    ,		)
    pipe.to(a    )
    pipe.set_progress_bar_config(disable=a    )
    pipe.enable_attention_slicing(1    )
    pipe.enable_sequential_cpu_offload()

    A__       =     'a cat sitting on a park bench'

    A__       =     torch.manual_seed(0    )
    A__       =     pipe(
        prompt=a    ,		image=a    ,		generator=a    ,		num_inference_steps=5    ,		output_type='np'    ,		)

    A__       =     torch.cuda.max_memory_allocated()
    # make sure that less than 2.9 GB is allocated
    assert mem_bytes < 2.9 * 10**9


