




"""simple docstring"""

import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    ControlNetModel,
    DDIMScheduler,
    StableDiffusionControlNetImgaImgPipeline,
    UNetaDConditionModel,
)
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_controlnet import MultiControlNetModel
from diffusers.utils import floats_tensor, load_image, load_numpy, randn_tensor, slow, torch_device
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import (
    IMAGE_TO_IMAGE_IMAGE_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_PARAMS,
)
from ..test_pipelines_common import (
    PipelineKarrasSchedulerTesterMixin,
    PipelineLatentTesterMixin,
    PipelineTesterMixin,
)


enable_full_determinism()
class       lowerCamelCase__     (	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	unittest.TestCase      ):
			a__			:   str             =  StableDiffusionControlNetImgaImgPipeline
			a__			:   Optional[Any]             =  TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {'height', 'width'}
			a__			:   Any             =  TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS
			a__			:   Dict             =  IMAGE_TO_IMAGE_IMAGE_PARAMS.union({"""control_image"""}      )
			a__			:   Optional[Any]             =  IMAGE_TO_IMAGE_IMAGE_PARAMS


			def 				lowerCamelCase_					(  self				):



					"""simple docstring"""







					torch.manual_seed(0				)
					snake_case		:    List[str]             =					UNetaDConditionModel(
					    block_out_channels=(32, 64)					,			layers_per_block=2					,			sample_size=32					,			in_channels=4					,			out_channels=4					,			down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")					,			up_block_types=("CrossAttnUpBlock2D", "UpBlock2D")					,			cross_attention_dim=32					,			)
					torch.manual_seed(0				)
					snake_case		:    str             =					ControlNetModel(
					    block_out_channels=(32, 64)					,			layers_per_block=2					,			in_channels=4					,			down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")					,			cross_attention_dim=32					,			conditioning_embedding_out_channels=(16, 32)					,			)
					torch.manual_seed(0				)
					snake_case		:    Any             =					DDIMScheduler(
					    beta_start=0.0_00_85					,			beta_end=0.0_12					,			beta_schedule="scaled_linear"					,			clip_sample=SCREAMING_SNAKE_CASE					,			set_alpha_to_one=SCREAMING_SNAKE_CASE					,			)
					torch.manual_seed(0				)
					snake_case		:    Tuple             =					AutoencoderKL(
					    block_out_channels=[32, 64]					,			in_channels=3					,			out_channels=3					,			down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"]					,			up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"]					,			latent_channels=4					,			)
					torch.manual_seed(0				)
					snake_case		:    Any             =					CLIPTextConfig(
					    bos_token_id=0					,			eos_token_id=2					,			hidden_size=32					,			intermediate_size=37					,			layer_norm_eps=1E-05					,			num_attention_heads=4					,			num_hidden_layers=5					,			pad_token_id=1					,			vocab_size=1_000					,			)
					snake_case		:    List[Any]             =					CLIPTextModel(SCREAMING_SNAKE_CASE				)
					snake_case		:    List[str]             =					CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"				)

					snake_case		:    Optional[Any]             =					{
					    "unet": unet,
					    "controlnet": controlnet,
					    "scheduler": scheduler,
					    "vae": vae,
					    "text_encoder": text_encoder,
					    "tokenizer": tokenizer,
					    "safety_checker": None,
					    "feature_extractor": None,
					}
					return components


			def 				lowerCamelCase_					(  self					,			SCREAMING_SNAKE_CASE					,			SCREAMING_SNAKE_CASE=0				):



					"""simple docstring"""







					if str(SCREAMING_SNAKE_CASE				).startswith("mps"				):
							snake_case		:    List[Any]             =					torch.manual_seed(SCREAMING_SNAKE_CASE				)
					else:
							snake_case		:    Any             =					torch.Generator(device=SCREAMING_SNAKE_CASE				).manual_seed(SCREAMING_SNAKE_CASE				)

					snake_case		:    List[Any]             =					2
					snake_case		:    Any             =					randn_tensor(
					    (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor)					,			generator=SCREAMING_SNAKE_CASE					,			device=torch.device(SCREAMING_SNAKE_CASE				)					,			)
					snake_case		:    Tuple             =					floats_tensor(control_image.shape					,			rng=random.Random(SCREAMING_SNAKE_CASE				)				).to(SCREAMING_SNAKE_CASE				)
					snake_case		:    Any             =					image.cpu().permute(0					,			2					,			3					,			1				)[0]
					snake_case		:    Dict             =					Image.fromarray(np.uinta(SCREAMING_SNAKE_CASE				)				).convert("RGB"				).resize((64, 64)				)
					snake_case		:    List[str]             =					{
					    "prompt": "A painting of a squirrel eating a burger",
					    "generator": generator,
					    "num_inference_steps": 2,
					    "guidance_scale": 6.0,
					    "output_type": "numpy",
					    "image": image,
					    "control_image": control_image,
					}

					return inputs


			def 				lowerCamelCase_					(  self				):



					"""simple docstring"""







					return self._test_attention_slicing_forward_pass(expected_max_diff=2E-3				)


			@unittest.skipIf(
			    torch_device != "cuda" or not is_xformers_available()					,			reason="XFormers attention is only available with CUDA and `xformers` installed"					,			)
			def 				lowerCamelCase_					(  self				):



					"""simple docstring"""







					self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2E-3				)



			def 				lowerCamelCase_					(  self				):



					"""simple docstring"""







					self._test_inference_batch_single_identical(expected_max_diff=2E-3				)







class       lowerCamelCase__     (	UpperCamelCase__					,	UpperCamelCase__					,	unittest.TestCase      ):
			a__			:   List[Any]             =  StableDiffusionControlNetImgaImgPipeline
			a__			:   Optional[Any]             =  TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {'height', 'width'}
			a__			:   str             =  TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS
			a__			:   List[str]             =  frozenset([]      )  # TO_DO: add image_params once refactored VaeImageProcessor.preprocess


			def 				lowerCamelCase_					(  self				):



					"""simple docstring"""







					torch.manual_seed(0				)
					snake_case		:    Optional[Any]             =					UNetaDConditionModel(
					    block_out_channels=(32, 64)					,			layers_per_block=2					,			sample_size=32					,			in_channels=4					,			out_channels=4					,			down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")					,			up_block_types=("CrossAttnUpBlock2D", "UpBlock2D")					,			cross_attention_dim=32					,			)
					torch.manual_seed(0				)

					def init_weights(SCREAMING_SNAKE_CASE				):
							if isinstance(SCREAMING_SNAKE_CASE					,			torch.nn.Convad				):
									torch.nn.init.normal(m.weight				)
									m.bias.data.fill_(1.0				)

					snake_case		:    Any             =					ControlNetModel(
					    block_out_channels=(32, 64)					,			layers_per_block=2					,			in_channels=4					,			down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")					,			cross_attention_dim=32					,			conditioning_embedding_out_channels=(16, 32)					,			)
					controlneta.controlnet_down_blocks.apply(SCREAMING_SNAKE_CASE				)

					torch.manual_seed(0				)
					snake_case		:    int             =					ControlNetModel(
					    block_out_channels=(32, 64)					,			layers_per_block=2					,			in_channels=4					,			down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")					,			cross_attention_dim=32					,			conditioning_embedding_out_channels=(16, 32)					,			)
					controlneta.controlnet_down_blocks.apply(SCREAMING_SNAKE_CASE				)

					torch.manual_seed(0				)
					snake_case		:    int             =					DDIMScheduler(
					    beta_start=0.0_00_85					,			beta_end=0.0_12					,			beta_schedule="scaled_linear"					,			clip_sample=SCREAMING_SNAKE_CASE					,			set_alpha_to_one=SCREAMING_SNAKE_CASE					,			)
					torch.manual_seed(0				)
					snake_case		:    List[Any]             =					AutoencoderKL(
					    block_out_channels=[32, 64]					,			in_channels=3					,			out_channels=3					,			down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"]					,			up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"]					,			latent_channels=4					,			)
					torch.manual_seed(0				)
					snake_case		:    str             =					CLIPTextConfig(
					    bos_token_id=0					,			eos_token_id=2					,			hidden_size=32					,			intermediate_size=37					,			layer_norm_eps=1E-05					,			num_attention_heads=4					,			num_hidden_layers=5					,			pad_token_id=1					,			vocab_size=1_000					,			)
					snake_case		:    Tuple             =					CLIPTextModel(SCREAMING_SNAKE_CASE				)
					snake_case		:    Any             =					CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"				)

					snake_case		:    int             =					MultiControlNetModel([controlneta, controlneta]				)

					snake_case		:    List[str]             =					{
					    "unet": unet,
					    "controlnet": controlnet,
					    "scheduler": scheduler,
					    "vae": vae,
					    "text_encoder": text_encoder,
					    "tokenizer": tokenizer,
					    "safety_checker": None,
					    "feature_extractor": None,
					}
					return components


			def 				lowerCamelCase_					(  self					,			SCREAMING_SNAKE_CASE					,			SCREAMING_SNAKE_CASE=0				):



					"""simple docstring"""







					if str(SCREAMING_SNAKE_CASE				).startswith("mps"				):
							snake_case		:    List[Any]             =					torch.manual_seed(SCREAMING_SNAKE_CASE				)
					else:
							snake_case		:    Union[str, Any]             =					torch.Generator(device=SCREAMING_SNAKE_CASE				).manual_seed(SCREAMING_SNAKE_CASE				)

					snake_case		:    Optional[Any]             =					2

					snake_case		:    Union[str, Any]             =					[
					    randn_tensor(
					        (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor)					,			generator=SCREAMING_SNAKE_CASE					,			device=torch.device(SCREAMING_SNAKE_CASE				)					,			),
					    randn_tensor(
					        (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor)					,			generator=SCREAMING_SNAKE_CASE					,			device=torch.device(SCREAMING_SNAKE_CASE				)					,			),
					]

					snake_case		:    str             =					floats_tensor(control_image[0].shape					,			rng=random.Random(SCREAMING_SNAKE_CASE				)				).to(SCREAMING_SNAKE_CASE				)
					snake_case		:    str             =					image.cpu().permute(0					,			2					,			3					,			1				)[0]
					snake_case		:    Optional[Any]             =					Image.fromarray(np.uinta(SCREAMING_SNAKE_CASE				)				).convert("RGB"				).resize((64, 64)				)
					snake_case		:    Optional[int]             =					{
					    "prompt": "A painting of a squirrel eating a burger",
					    "generator": generator,
					    "num_inference_steps": 2,
					    "guidance_scale": 6.0,
					    "output_type": "numpy",
					    "image": image,
					    "control_image": control_image,
					}

					return inputs


			def 				lowerCamelCase_					(  self				):



					"""simple docstring"""







					snake_case		:    Dict             =					self.get_dummy_components()
					snake_case		:    Optional[Any]             =					self.pipeline_class(**SCREAMING_SNAKE_CASE				)
					pipe.to(SCREAMING_SNAKE_CASE				)

					snake_case		:    Dict             =					10.0
					snake_case		:    List[Any]             =					4

					snake_case		:    Optional[int]             =					self.get_dummy_inputs(SCREAMING_SNAKE_CASE				)
					snake_case		:    List[Any]             =					steps
					snake_case		:    str             =					scale
					snake_case		:    Union[str, Any]             =					pipe(**SCREAMING_SNAKE_CASE				)[0]

					snake_case		:    Dict             =					self.get_dummy_inputs(SCREAMING_SNAKE_CASE				)
					snake_case		:    int             =					steps
					snake_case		:    List[Any]             =					scale
					snake_case		:    Optional[Any]             =					pipe(**SCREAMING_SNAKE_CASE					,			control_guidance_start=0.1					,			control_guidance_end=0.2				)[0]

					snake_case		:    List[Any]             =					self.get_dummy_inputs(SCREAMING_SNAKE_CASE				)
					snake_case		:    Optional[Any]             =					steps
					snake_case		:    List[str]             =					scale
					snake_case		:    Union[str, Any]             =					pipe(**SCREAMING_SNAKE_CASE					,			control_guidance_start=[0.1, 0.3]					,			control_guidance_end=[0.2, 0.7]				)[0]

					snake_case		:    List[Any]             =					self.get_dummy_inputs(SCREAMING_SNAKE_CASE				)
					snake_case		:    Optional[Any]             =					steps
					snake_case		:    Union[str, Any]             =					scale
					snake_case		:    Tuple             =					pipe(**SCREAMING_SNAKE_CASE					,			control_guidance_start=0.4					,			control_guidance_end=[0.5, 0.8]				)[0]

					# make sure that all outputs are different
					assert np.sum(np.abs(output_a - output_a				)				) > 1E-3
					assert np.sum(np.abs(output_a - output_a				)				) > 1E-3
					assert np.sum(np.abs(output_a - output_a				)				) > 1E-3


			def 				lowerCamelCase_					(  self				):



					"""simple docstring"""







					return self._test_attention_slicing_forward_pass(expected_max_diff=2E-3				)


			@unittest.skipIf(
			    torch_device != "cuda" or not is_xformers_available()					,			reason="XFormers attention is only available with CUDA and `xformers` installed"					,			)
			def 				lowerCamelCase_					(  self				):



					"""simple docstring"""







					self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2E-3				)


			def 				lowerCamelCase_					(  self				):



					"""simple docstring"""







					self._test_inference_batch_single_identical(expected_max_diff=2E-3				)



			def 				lowerCamelCase_					(  self				):



					"""simple docstring"""







					snake_case		:    Any             =					self.get_dummy_components()
					snake_case		:    List[str]             =					self.pipeline_class(**SCREAMING_SNAKE_CASE				)
					pipe.to(SCREAMING_SNAKE_CASE				)
					pipe.set_progress_bar_config(disable=SCREAMING_SNAKE_CASE				)
					with tempfile.TemporaryDirectory() as tmpdir:
							try:
									# save_pretrained is not implemented for Multi-ControlNet
									pipe.save_pretrained(SCREAMING_SNAKE_CASE				)
							except NotImplementedError:
									pass







@slow
@require_torch_gpu
class       lowerCamelCase__     (	unittest.TestCase      ):


			def 				lowerCamelCase_					(  self				):



					"""simple docstring"""







					super().tearDown()
					gc.collect()
					torch.cuda.empty_cache()



			def 				lowerCamelCase_					(  self				):



					"""simple docstring"""







					snake_case		:    List[Any]             =					ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny"				)

					snake_case		:    Any             =					StableDiffusionControlNetImgaImgPipeline.from_pretrained(
					    "runwayml/stable-diffusion-v1-5"					,			safety_checker=SCREAMING_SNAKE_CASE					,			controlnet=SCREAMING_SNAKE_CASE				)
					pipe.enable_model_cpu_offload()
					pipe.set_progress_bar_config(disable=SCREAMING_SNAKE_CASE				)

					snake_case		:    str             =					torch.Generator(device="cpu"				).manual_seed(0				)
					snake_case		:    Tuple             =					"evil space-punk bird"
					snake_case		:    List[str]             =					load_image(
					    "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png"				).resize((512, 512)				)
					snake_case		:    Optional[Any]             =					load_image(
					    "https://huggingface.co/lllyasviel/sd-controlnet-canny/resolve/main/images/bird.png"				).resize((512, 512)				)

					snake_case		:    Union[str, Any]             =					pipe(
					    SCREAMING_SNAKE_CASE					,			SCREAMING_SNAKE_CASE					,			control_image=SCREAMING_SNAKE_CASE					,			generator=SCREAMING_SNAKE_CASE					,			output_type="np"					,			num_inference_steps=50					,			strength=0.6					,			)

					snake_case		:    List[Any]             =					output.images[0]

					assert image.shape == (512, 512, 3)

					snake_case		:    Union[str, Any]             =					load_numpy(
					    "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/img2img.npy"				)

					assert np.abs(expected_image - image				).max() < 9E-2



'''simple docstring'''
import unittest

import torch

from diffusers import DDIMScheduler, DDPMScheduler, UNetaDModel
from diffusers.training_utils import set_seed
from diffusers.utils.testing_utils import slow


lowercase__											=False







class   a_ (      unittest.TestCase					):




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase=32	):
				set_seed(0	)
				a_      				=						UNetaDModel(sample_size=UpperCAmelCase					,   in_channels=3					,   out_channels=3	)
				a_      				=						torch.optim.SGD(model.parameters()					,   lr=0.00_01	)
				return model, optimizer






			@slow
			def 					lowerCAmelCase__					(       self	):
				a_      				=						"""cpu"""  # ensure full determinism without setting the CUBLAS_WORKSPACE_CONFIG env variable
				a_      				=						DDPMScheduler(
				    num_train_timesteps=10_00					,   beta_start=0.00_01					,   beta_end=0.02					,   beta_schedule="""linear"""					,   clip_sample=UpperCAmelCase					,   )
				a_      				=						DDIMScheduler(
				    num_train_timesteps=10_00					,   beta_start=0.00_01					,   beta_end=0.02					,   beta_schedule="""linear"""					,   clip_sample=UpperCAmelCase					,   )

				assert ddpm_scheduler.config.num_train_timesteps == ddim_scheduler.config.num_train_timesteps

				# shared batches for DDPM and DDIM
				set_seed(0	)
				a_      				=						[torch.randn((4, 3, 32, 32)	).clip(-1					,   1	).to(UpperCAmelCase	) for _ in range(4	)]
				a_      				=						[torch.randn((4, 3, 32, 32)	).to(UpperCAmelCase	) for _ in range(4	)]
				a_      				=						[torch.randint(0					,   10_00					,   (4,)	).long().to(UpperCAmelCase	) for _ in range(4	)]

				# train with a DDPM scheduler
				a_				,  a_      				=						self.get_model_optimizer(resolution=32	)
				model.train().to(UpperCAmelCase	)
				for i in range(4	):
					optimizer.zero_grad()
					a_      				=						ddpm_scheduler.add_noise(clean_images[i]					,   noise[i]					,   timesteps[i]	)
					a_      				=						model(UpperCAmelCase					,   timesteps[i]	).sample
					a_      				=						torch.nn.functional.mse_loss(UpperCAmelCase					,   noise[i]	)
					loss.backward()
					optimizer.step()
				del model, optimizer

				# recreate the model and optimizer, and retry with DDIM
				a_				,  a_      				=						self.get_model_optimizer(resolution=32	)
				model.train().to(UpperCAmelCase	)
				for i in range(4	):
					optimizer.zero_grad()
					a_      				=						ddim_scheduler.add_noise(clean_images[i]					,   noise[i]					,   timesteps[i]	)
					a_      				=						model(UpperCAmelCase					,   timesteps[i]	).sample
					a_      				=						torch.nn.functional.mse_loss(UpperCAmelCase					,   noise[i]	)
					loss.backward()
					optimizer.step()
				del model, optimizer

				self.assertTrue(torch.allclose(UpperCAmelCase					,   UpperCAmelCase					,   atol=1e-5	)	)
				self.assertTrue(torch.allclose(UpperCAmelCase					,   UpperCAmelCase					,   atol=1e-5	)	)
