

"""simple docstring"""




import os

from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE, hf_cache_home


__UpperCAmelCase =				HUGGINGFACE_HUB_CACHE


__UpperCAmelCase =				'config.json'
__UpperCAmelCase =				'diffusion_pytorch_model.bin'
__UpperCAmelCase =				'diffusion_flax_model.msgpack'
__UpperCAmelCase =				'model.onnx'
__UpperCAmelCase =				'diffusion_pytorch_model.safetensors'
__UpperCAmelCase =				'weights.pb'
__UpperCAmelCase =				'https://huggingface.co'
__UpperCAmelCase =				default_cache_path
__UpperCAmelCase =				'diffusers_modules'
__UpperCAmelCase =				os.getenv('HF_MODULES_CACHE', os.path.join(hf_cache_home, 'modules'))
__UpperCAmelCase =				['fp16', 'non-ema']
__UpperCAmelCase =				'.self_attn'








"""simple docstring"""




# This model implementation is heavily inspired by https://github.com/haofanwang/ControlNet-for-Diffusers/

import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    ControlNetModel,
    DDIMScheduler,
    StableDiffusionControlNetImgaImgPipeline,
    UNetaDConditionModel,
)
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_controlnet import MultiControlNetModel
from diffusers.utils import floats_tensor, load_image, load_numpy, randn_tensor, slow, torch_device
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import (
    IMAGE_TO_IMAGE_IMAGE_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_PARAMS,
)
from ..test_pipelines_common import (
    PipelineKarrasSchedulerTesterMixin,
    PipelineLatentTesterMixin,
    PipelineTesterMixin,
)


enable_full_determinism()







class      __lowercase			(    __lowerCamelCase	,			__lowerCamelCase	,			__lowerCamelCase	,			unittest.TestCase					):
  snake_case_				       =       StableDiffusionControlNetImgaImgPipeline
  snake_case_				       =       TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"""height""", """width"""}
  snake_case_				       =       TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS
  snake_case_				       =       IMAGE_TO_IMAGE_IMAGE_PARAMS.union({"""control_image"""}					)
  snake_case_				       =       IMAGE_TO_IMAGE_IMAGE_PARAMS

  def    __lowercase       (  self		:			str    ):






   '''simple docstring'''
   torch.manual_seed(0    )
   UpperCAmelCase__					:  Union[str, Any]					   =					UNetaDConditionModel(
       block_out_channels=(32, 64)					,layers_per_block=2					,sample_size=32					,in_channels=4					,out_channels=4					,down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")					,up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")					,cross_attention_dim=32					,)
   torch.manual_seed(0    )
   UpperCAmelCase__					:  Optional[int]					   =					ControlNetModel(
       block_out_channels=(32, 64)					,layers_per_block=2					,in_channels=4					,down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")					,cross_attention_dim=32					,conditioning_embedding_out_channels=(16, 32)					,)
   torch.manual_seed(0    )
   UpperCAmelCase__					:  Tuple					   =					DDIMScheduler(
       beta_start=0.0_0_0_8_5					,beta_end=0.0_1_2					,beta_schedule="""scaled_linear"""					,clip_sample=A					,set_alpha_to_one=A					,)
   torch.manual_seed(0    )
   UpperCAmelCase__					:  Any					   =					AutoencoderKL(
       block_out_channels=[32, 64]					,in_channels=3					,out_channels=3					,down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]					,up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]					,latent_channels=4					,)
   torch.manual_seed(0    )
   UpperCAmelCase__					:  int					   =					CLIPTextConfig(
       bos_token_id=0					,eos_token_id=2					,hidden_size=32					,intermediate_size=37					,layer_norm_eps=1e-05					,num_attention_heads=4					,num_hidden_layers=5					,pad_token_id=1					,vocab_size=1_000					,)
   UpperCAmelCase__					:  List[str]					   =					CLIPTextModel(A    )
   UpperCAmelCase__					:  Dict					   =					CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""    )

   UpperCAmelCase__					:  Optional[int]					   =					{
       """unet""": unet,
       """controlnet""": controlnet,
       """scheduler""": scheduler,
       """vae""": vae,
       """text_encoder""": text_encoder,
       """tokenizer""": tokenizer,
       """safety_checker""": None,
       """feature_extractor""": None,
   }
   return components

  def    __lowercase       (  self		:			Union[str, Any]					,A		:			Dict					,A		:			Optional[Any]=0    ):






   '''simple docstring'''
   if str(A    ).startswith("""mps"""    ):
    UpperCAmelCase__					:  List[str]					   =					torch.manual_seed(A    )
   else:
    UpperCAmelCase__					:  List[Any]					   =					torch.Generator(device=A    ).manual_seed(A    )

   UpperCAmelCase__					:  Tuple					   =					2
   UpperCAmelCase__					:  Any					   =					randn_tensor(
       (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor)					,generator=A					,device=torch.device(A    )					,)
   UpperCAmelCase__					:  str					   =					floats_tensor(control_image.shape					,rng=random.Random(A    )    ).to(A    )
   UpperCAmelCase__					:  Dict					   =					image.cpu().permute(0					,2					,3					,1    )[0]
   UpperCAmelCase__					:  List[str]					   =					Image.fromarray(np.uinta(A    )    ).convert("""RGB"""    ).resize((64, 64)    )
   UpperCAmelCase__					:  Any					   =					{
       """prompt""": """A painting of a squirrel eating a burger""",
       """generator""": generator,
       """num_inference_steps""": 2,
       """guidance_scale""": 6.0,
       """output_type""": """numpy""",
       """image""": image,
       """control_image""": control_image,
   }

   return inputs

  def    __lowercase       (  self		:			Optional[Any]    ):






   '''simple docstring'''
   return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3    )

  @unittest.skipIf(
      torch_device != """cuda""" or not is_xformers_available()					,reason="""XFormers attention is only available with CUDA and `xformers` installed"""					,)
  def    __lowercase       (  self		:			Dict    ):






   '''simple docstring'''
   self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3    )




  def    __lowercase       (  self		:			Any    ):






   '''simple docstring'''
   self._test_inference_batch_single_identical(expected_max_diff=2e-3    )







class      __lowercase			(    __lowerCamelCase	,			__lowerCamelCase	,			unittest.TestCase					):
  snake_case_				       =       StableDiffusionControlNetImgaImgPipeline
  snake_case_				       =       TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"""height""", """width"""}
  snake_case_				       =       TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS
  snake_case_				       =       frozenset([]					)  # TO_DO: add image_params once refactored VaeImageProcessor.preprocess

  def    __lowercase       (  self		:			str    ):






   '''simple docstring'''
   torch.manual_seed(0    )
   UpperCAmelCase__					:  Tuple					   =					UNetaDConditionModel(
       block_out_channels=(32, 64)					,layers_per_block=2					,sample_size=32					,in_channels=4					,out_channels=4					,down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")					,up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")					,cross_attention_dim=32					,)
   torch.manual_seed(0    )

   def init_weights(A		:			Union[str, Any]    ):
    if isinstance(A					,torch.nn.Convad    ):
     torch.nn.init.normal(m.weight    )
     m.bias.data.fill_(1.0    )

   UpperCAmelCase__					:  int					   =					ControlNetModel(
       block_out_channels=(32, 64)					,layers_per_block=2					,in_channels=4					,down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")					,cross_attention_dim=32					,conditioning_embedding_out_channels=(16, 32)					,)
   controlneta.controlnet_down_blocks.apply(A    )

   torch.manual_seed(0    )
   UpperCAmelCase__					:  Dict					   =					ControlNetModel(
       block_out_channels=(32, 64)					,layers_per_block=2					,in_channels=4					,down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")					,cross_attention_dim=32					,conditioning_embedding_out_channels=(16, 32)					,)
   controlneta.controlnet_down_blocks.apply(A    )

   torch.manual_seed(0    )
   UpperCAmelCase__					:  str					   =					DDIMScheduler(
       beta_start=0.0_0_0_8_5					,beta_end=0.0_1_2					,beta_schedule="""scaled_linear"""					,clip_sample=A					,set_alpha_to_one=A					,)
   torch.manual_seed(0    )
   UpperCAmelCase__					:  Dict					   =					AutoencoderKL(
       block_out_channels=[32, 64]					,in_channels=3					,out_channels=3					,down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]					,up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]					,latent_channels=4					,)
   torch.manual_seed(0    )
   UpperCAmelCase__					:  Tuple					   =					CLIPTextConfig(
       bos_token_id=0					,eos_token_id=2					,hidden_size=32					,intermediate_size=37					,layer_norm_eps=1e-05					,num_attention_heads=4					,num_hidden_layers=5					,pad_token_id=1					,vocab_size=1_000					,)
   UpperCAmelCase__					:  Optional[Any]					   =					CLIPTextModel(A    )
   UpperCAmelCase__					:  List[str]					   =					CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""    )

   UpperCAmelCase__					:  Dict					   =					MultiControlNetModel([controlneta, controlneta]    )

   UpperCAmelCase__					:  Union[str, Any]					   =					{
       """unet""": unet,
       """controlnet""": controlnet,
       """scheduler""": scheduler,
       """vae""": vae,
       """text_encoder""": text_encoder,
       """tokenizer""": tokenizer,
       """safety_checker""": None,
       """feature_extractor""": None,
   }
   return components

  def    __lowercase       (  self		:			Optional[int]					,A		:			List[Any]					,A		:			List[Any]=0    ):






   '''simple docstring'''
   if str(A    ).startswith("""mps"""    ):
    UpperCAmelCase__					:  List[str]					   =					torch.manual_seed(A    )
   else:
    UpperCAmelCase__					:  int					   =					torch.Generator(device=A    ).manual_seed(A    )

   UpperCAmelCase__					:  Optional[int]					   =					2

   UpperCAmelCase__					:  List[Any]					   =					[
       randn_tensor(
           (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor)					,generator=A					,device=torch.device(A    )					,),
       randn_tensor(
           (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor)					,generator=A					,device=torch.device(A    )					,),
   ]

   UpperCAmelCase__					:  List[Any]					   =					floats_tensor(control_image[0].shape					,rng=random.Random(A    )    ).to(A    )
   UpperCAmelCase__					:  str					   =					image.cpu().permute(0					,2					,3					,1    )[0]
   UpperCAmelCase__					:  Union[str, Any]					   =					Image.fromarray(np.uinta(A    )    ).convert("""RGB"""    ).resize((64, 64)    )
   UpperCAmelCase__					:  Any					   =					{
       """prompt""": """A painting of a squirrel eating a burger""",
       """generator""": generator,
       """num_inference_steps""": 2,
       """guidance_scale""": 6.0,
       """output_type""": """numpy""",
       """image""": image,
       """control_image""": control_image,
   }

   return inputs

  def    __lowercase       (  self		:			Dict    ):






   '''simple docstring'''
   UpperCAmelCase__					:  Optional[int]					   =					self.get_dummy_components()
   UpperCAmelCase__					:  List[str]					   =					self.pipeline_class(**A    )
   pipe.to(A    )

   UpperCAmelCase__					:  Any					   =					1_0.0
   UpperCAmelCase__					:  Any					   =					4

   UpperCAmelCase__					:  Optional[int]					   =					self.get_dummy_inputs(A    )
   UpperCAmelCase__					:  Optional[Any]					   =					steps
   UpperCAmelCase__					:  List[str]					   =					scale
   UpperCAmelCase__					:  Optional[Any]					   =					pipe(**A    )[0]

   UpperCAmelCase__					:  Any					   =					self.get_dummy_inputs(A    )
   UpperCAmelCase__					:  Optional[int]					   =					steps
   UpperCAmelCase__					:  Tuple					   =					scale
   UpperCAmelCase__					:  List[Any]					   =					pipe(**A					,control_guidance_start=0.1					,control_guidance_end=0.2    )[0]

   UpperCAmelCase__					:  List[str]					   =					self.get_dummy_inputs(A    )
   UpperCAmelCase__					:  Union[str, Any]					   =					steps
   UpperCAmelCase__					:  Any					   =					scale
   UpperCAmelCase__					:  str					   =					pipe(**A					,control_guidance_start=[0.1, 0.3]					,control_guidance_end=[0.2, 0.7]    )[0]

   UpperCAmelCase__					:  Any					   =					self.get_dummy_inputs(A    )
   UpperCAmelCase__					:  List[str]					   =					steps
   UpperCAmelCase__					:  Union[str, Any]					   =					scale
   UpperCAmelCase__					:  Union[str, Any]					   =					pipe(**A					,control_guidance_start=0.4					,control_guidance_end=[0.5, 0.8]    )[0]

   # make sure that all outputs are different
   assert np.sum(np.abs(output_a - output_a    )    ) > 1e-3
   assert np.sum(np.abs(output_a - output_a    )    ) > 1e-3
   assert np.sum(np.abs(output_a - output_a    )    ) > 1e-3

  def    __lowercase       (  self		:			Any    ):






   '''simple docstring'''
   return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3    )

  @unittest.skipIf(
      torch_device != """cuda""" or not is_xformers_available()					,reason="""XFormers attention is only available with CUDA and `xformers` installed"""					,)
  def    __lowercase       (  self		:			Optional[int]    ):






   '''simple docstring'''
   self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3    )

  def    __lowercase       (  self		:			Union[str, Any]    ):






   '''simple docstring'''
   self._test_inference_batch_single_identical(expected_max_diff=2e-3    )




  def    __lowercase       (  self		:			Dict    ):






   '''simple docstring'''
   UpperCAmelCase__					:  Dict					   =					self.get_dummy_components()
   UpperCAmelCase__					:  str					   =					self.pipeline_class(**A    )
   pipe.to(A    )
   pipe.set_progress_bar_config(disable=A    )
   with tempfile.TemporaryDirectory() as tmpdir:
    try:
     # save_pretrained is not implemented for Multi-ControlNet
     pipe.save_pretrained(A    )
    except NotImplementedError:
     pass







@slow
@require_torch_gpu
class      __lowercase			(    unittest.TestCase					):

  def    __lowercase       (  self		:			List[str]    ):






   '''simple docstring'''
   super().tearDown()
   gc.collect()
   torch.cuda.empty_cache()




  def    __lowercase       (  self		:			Union[str, Any]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  str					   =					ControlNetModel.from_pretrained("""lllyasviel/sd-controlnet-canny"""    )

   UpperCAmelCase__					:  Tuple					   =					StableDiffusionControlNetImgaImgPipeline.from_pretrained(
       """runwayml/stable-diffusion-v1-5"""					,safety_checker=A					,controlnet=A    )
   pipe.enable_model_cpu_offload()
   pipe.set_progress_bar_config(disable=A    )

   UpperCAmelCase__					:  Optional[int]					   =					torch.Generator(device="""cpu"""    ).manual_seed(0    )
   UpperCAmelCase__					:  Dict					   =					"""evil space-punk bird"""
   UpperCAmelCase__					:  Optional[int]					   =					load_image(
       """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png"""    ).resize((512, 512)    )
   UpperCAmelCase__					:  Optional[Any]					   =					load_image(
       """https://huggingface.co/lllyasviel/sd-controlnet-canny/resolve/main/images/bird.png"""    ).resize((512, 512)    )

   UpperCAmelCase__					:  List[str]					   =					pipe(
       A					,A					,control_image=A					,generator=A					,output_type="""np"""					,num_inference_steps=50					,strength=0.6					,)

   UpperCAmelCase__					:  Optional[Any]					   =					output.images[0]

   assert image.shape == (512, 512, 3)

   UpperCAmelCase__					:  Any					   =					load_numpy(
       """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/img2img.npy"""    )

   assert np.abs(expected_image - image    ).max() < 9e-2





