




'''simple docstring'''







import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, DDPMScheduler, StableDiffusionUpscalePipeline, UNetaDConditionModel
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu


enable_full_determinism()

class 	_snake_case(			unittest.TestCase      ):






 def 						_UpperCamelCase (self       :  List[str]    )       ->   str:






    """simple docstring"""


    super().tearDown()
    gc.collect()
    torch.cuda.empty_cache()






 @property
 def 						_UpperCamelCase (self       :  int    )       ->   Any:






    """simple docstring"""


    A__       =     1
    A__       =     3
    A__       =     (32, 32)

    A__       =     floats_tensor((batch_size, num_channels) + sizes    ,		rng=random.Random(0    )    ).to(a    )
    return image






 @property
 def 						_UpperCamelCase (self       :  Union[str, Any]    )       ->   Tuple:






    """simple docstring"""


    torch.manual_seed(0    )
    A__       =     UNetaDConditionModel(
        block_out_channels=(32, 32, 64)    ,		layers_per_block=2    ,		sample_size=32    ,		in_channels=7    ,		out_channels=4    ,		down_block_types=('DownBlock2D', 'CrossAttnDownBlock2D', 'CrossAttnDownBlock2D')    ,		up_block_types=('CrossAttnUpBlock2D', 'CrossAttnUpBlock2D', 'UpBlock2D')    ,		cross_attention_dim=32    ,		attention_head_dim=8    ,		use_linear_projection=a    ,		only_cross_attention=(True, True, False)    ,		num_class_embeds=1_00    ,		)
    return model






 @property
 def 						_UpperCamelCase (self       :  Tuple    )       ->   Optional[Any]:






    """simple docstring"""


    torch.manual_seed(0    )
    A__       =     AutoencoderKL(
        block_out_channels=[32, 32, 64]    ,		in_channels=3    ,		out_channels=3    ,		down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D', 'DownEncoderBlock2D']    ,		up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D', 'UpDecoderBlock2D']    ,		latent_channels=4    ,		)
    return model






 @property
 def 						_UpperCamelCase (self       :  str    )       ->   List[str]:






    """simple docstring"""


    torch.manual_seed(0    )
    A__       =     CLIPTextConfig(
        bos_token_id=0    ,		eos_token_id=2    ,		hidden_size=32    ,		intermediate_size=37    ,		layer_norm_eps=1e-05    ,		num_attention_heads=4    ,		num_hidden_layers=5    ,		pad_token_id=1    ,		vocab_size=10_00    ,		hidden_act='gelu'    ,		projection_dim=5_12    ,		)
    return CLIPTextModel(a    )






 def 						_UpperCamelCase (self       :  str    )       ->   List[str]:






    """simple docstring"""


    A__       =     'cpu'  # ensure determinism for the device-dependent torch.Generator
    A__       =     self.dummy_cond_unet_upscale
    A__       =     DDPMScheduler()
    A__       =     DDIMScheduler(prediction_type='v_prediction'    )
    A__       =     self.dummy_vae
    A__       =     self.dummy_text_encoder
    A__       =     CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'    )

    A__       =     self.dummy_image.cpu().permute(0    ,		2    ,		3    ,		1    )[0]
    A__       =     Image.fromarray(np.uinta(a    )    ).convert('RGB'    ).resize((64, 64)    )

    # make sure here that pndm scheduler skips prk
    A__       =     StableDiffusionUpscalePipeline(
        unet=a    ,		low_res_scheduler=a    ,		scheduler=a    ,		vae=a    ,		text_encoder=a    ,		tokenizer=a    ,		max_noise_level=3_50    ,		)
    A__       =     sd_pipe.to(a    )
    sd_pipe.set_progress_bar_config(disable=a    )

    A__       =     'A painting of a squirrel eating a burger'
    A__       =     torch.Generator(device=a    ).manual_seed(0    )
    A__       =     sd_pipe(
        [prompt]    ,		image=a    ,		generator=a    ,		guidance_scale=6.0    ,		noise_level=20    ,		num_inference_steps=2    ,		output_type='np'    ,		)

    A__       =     output.images

    A__       =     torch.Generator(device=a    ).manual_seed(0    )
    A__       =     sd_pipe(
        [prompt]    ,		image=a    ,		generator=a    ,		guidance_scale=6.0    ,		noise_level=20    ,		num_inference_steps=2    ,		output_type='np'    ,		return_dict=a    ,		)[0]

    A__       =     image[0, -3:, -3:, -1]
    A__       =     image_from_tuple[0, -3:, -3:, -1]

    A__       =     low_res_image.size[0] * 4
    assert image.shape == (1, expected_height_width, expected_height_width, 3)
    A__       =     np.array([0.3113, 0.3910, 0.4272, 0.4859, 0.5061, 0.4652, 0.5362, 0.5715, 0.5661]    )

    assert np.abs(image_slice.flatten() - expected_slice    ).max() < 1e-2
    assert np.abs(image_from_tuple_slice.flatten() - expected_slice    ).max() < 1e-2






 def 						_UpperCamelCase (self       :  Optional[int]    )       ->   Optional[int]:






    """simple docstring"""


    A__       =     'cpu'  # ensure determinism for the device-dependent torch.Generator
    A__       =     self.dummy_cond_unet_upscale
    A__       =     DDPMScheduler()
    A__       =     DDIMScheduler(prediction_type='v_prediction'    )
    A__       =     self.dummy_vae
    A__       =     self.dummy_text_encoder
    A__       =     CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'    )

    A__       =     self.dummy_image.cpu().permute(0    ,		2    ,		3    ,		1    )[0]
    A__       =     Image.fromarray(np.uinta(a    )    ).convert('RGB'    ).resize((64, 64)    )

    # make sure here that pndm scheduler skips prk
    A__       =     StableDiffusionUpscalePipeline(
        unet=a    ,		low_res_scheduler=a    ,		scheduler=a    ,		vae=a    ,		text_encoder=a    ,		tokenizer=a    ,		max_noise_level=3_50    ,		)
    A__       =     sd_pipe.to(a    )
    sd_pipe.set_progress_bar_config(disable=a    )

    A__       =     'A painting of a squirrel eating a burger'
    A__       =     sd_pipe(
        2 * [prompt]    ,		image=2 * [low_res_image]    ,		guidance_scale=6.0    ,		noise_level=20    ,		num_inference_steps=2    ,		output_type='np'    ,		)
    A__       =     output.images
    assert image.shape[0] == 2

    A__       =     torch.Generator(device=a    ).manual_seed(0    )
    A__       =     sd_pipe(
        [prompt]    ,		image=a    ,		generator=a    ,		num_images_per_prompt=2    ,		guidance_scale=6.0    ,		noise_level=20    ,		num_inference_steps=2    ,		output_type='np'    ,		)
    A__       =     output.images
    assert image.shape[0] == 2






 @unittest.skipIf(torch_device != 'cuda'    ,		'This test requires a GPU'    )
 def 						_UpperCamelCase (self       :  List[Any]    )       ->   Union[str, Any]:






    """simple docstring"""


    A__       =     self.dummy_cond_unet_upscale
    A__       =     DDPMScheduler()
    A__       =     DDIMScheduler(prediction_type='v_prediction'    )
    A__       =     self.dummy_vae
    A__       =     self.dummy_text_encoder
    A__       =     CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'    )

    A__       =     self.dummy_image.cpu().permute(0    ,		2    ,		3    ,		1    )[0]
    A__       =     Image.fromarray(np.uinta(a    )    ).convert('RGB'    ).resize((64, 64)    )

    # put models in fp16, except vae as it overflows in fp16
    A__       =     unet.half()
    A__       =     text_encoder.half()

    # make sure here that pndm scheduler skips prk
    A__       =     StableDiffusionUpscalePipeline(
        unet=a    ,		low_res_scheduler=a    ,		scheduler=a    ,		vae=a    ,		text_encoder=a    ,		tokenizer=a    ,		max_noise_level=3_50    ,		)
    A__       =     sd_pipe.to(a    )
    sd_pipe.set_progress_bar_config(disable=a    )

    A__       =     'A painting of a squirrel eating a burger'
    A__       =     torch.manual_seed(0    )
    A__       =     sd_pipe(
        [prompt]    ,		image=a    ,		generator=a    ,		num_inference_steps=2    ,		output_type='np'    ,		).images

    A__       =     low_res_image.size[0] * 4
    assert image.shape == (1, expected_height_width, expected_height_width, 3)




@slow
@require_torch_gpu
class 	_snake_case(			unittest.TestCase      ):






 def 						_UpperCamelCase (self       :  Union[str, Any]    )       ->   Dict:






    """simple docstring"""


    super().tearDown()
    gc.collect()
    torch.cuda.empty_cache()






 def 						_UpperCamelCase (self       :  Optional[Any]    )       ->   Optional[int]:






    """simple docstring"""


    A__       =     load_image(
        'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
        '/sd2-upscale/low_res_cat.png'    )
    A__       =     load_numpy(
        'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale'
        '/upsampled_cat.npy'    )

    A__       =     'stabilityai/stable-diffusion-x4-upscaler'
    A__       =     StableDiffusionUpscalePipeline.from_pretrained(a    )
    pipe.to(a    )
    pipe.set_progress_bar_config(disable=a    )
    pipe.enable_attention_slicing()

    A__       =     'a cat sitting on a park bench'

    A__       =     torch.manual_seed(0    )
    A__       =     pipe(
        prompt=a    ,		image=a    ,		generator=a    ,		output_type='np'    ,		)
    A__       =     output.images[0]

    assert image.shape == (5_12, 5_12, 3)
    assert np.abs(expected_image - image    ).max() < 1e-3






 def 						_UpperCamelCase (self       :  Any    )       ->   str:






    """simple docstring"""


    A__       =     load_image(
        'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
        '/sd2-upscale/low_res_cat.png'    )
    A__       =     load_numpy(
        'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale'
        '/upsampled_cat_fp16.npy'    )

    A__       =     'stabilityai/stable-diffusion-x4-upscaler'
    A__       =     StableDiffusionUpscalePipeline.from_pretrained(
        a    ,		torch_dtype=torch.floataa    ,		)
    pipe.to(a    )
    pipe.set_progress_bar_config(disable=a    )
    pipe.enable_attention_slicing()

    A__       =     'a cat sitting on a park bench'

    A__       =     torch.manual_seed(0    )
    A__       =     pipe(
        prompt=a    ,		image=a    ,		generator=a    ,		output_type='np'    ,		)
    A__       =     output.images[0]

    assert image.shape == (5_12, 5_12, 3)
    assert np.abs(expected_image - image    ).max() < 5e-1






 def 						_UpperCamelCase (self       :  Dict    )       ->   List[str]:






    """simple docstring"""


    torch.cuda.empty_cache()
    torch.cuda.reset_max_memory_allocated()
    torch.cuda.reset_peak_memory_stats()

    A__       =     load_image(
        'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
        '/sd2-upscale/low_res_cat.png'    )

    A__       =     'stabilityai/stable-diffusion-x4-upscaler'
    A__       =     StableDiffusionUpscalePipeline.from_pretrained(
        a    ,		torch_dtype=torch.floataa    ,		)
    pipe.to(a    )
    pipe.set_progress_bar_config(disable=a    )
    pipe.enable_attention_slicing(1    )
    pipe.enable_sequential_cpu_offload()

    A__       =     'a cat sitting on a park bench'

    A__       =     torch.manual_seed(0    )
    A__       =     pipe(
        prompt=a    ,		image=a    ,		generator=a    ,		num_inference_steps=5    ,		output_type='np'    ,		)

    A__       =     torch.cuda.max_memory_allocated()
    # make sure that less than 2.9 GB is allocated
    assert mem_bytes < 2.9 * 10**9








'''simple docstring'''







import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version('''>=''', FSDP_PYTORCH_VERSION):
    import torch.distributed.checkpoint as dist_cp
    from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
    from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
    from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
    from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


lowerCAmelCase_     						=				get_logger(__name__)







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase=0						):






   '''simple docstring'''







   os.makedirs(UpperCAmelCase      ,exist_ok=UpperCAmelCase						)
   with FSDP.state_dict_type(
       UpperCAmelCase      ,fsdp_plugin.state_dict_type      ,fsdp_plugin.state_dict_config      ,fsdp_plugin.optim_state_dict_config						):
      A__       =     model.state_dict()
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         A__       =     F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
         A__       =     os.path.join(UpperCAmelCase      ,UpperCAmelCase						)
         if accelerator.process_index == 0:
            logger.info(F"""Saving model to {output_model_file}"""						)
            torch.save(UpperCAmelCase      ,UpperCAmelCase						)
            logger.info(F"""Model saved to {output_model_file}"""						)
      elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
         A__       =     (
             F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
             if model_index == 0
             else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
         )
         A__       =     os.path.join(UpperCAmelCase      ,UpperCAmelCase						)
         logger.info(F"""Saving model to {output_model_file}"""						)
         torch.save(UpperCAmelCase      ,UpperCAmelCase						)
         logger.info(F"""Model saved to {output_model_file}"""						)
      elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
         A__       =     os.path.join(UpperCAmelCase      ,F"""{MODEL_NAME}_{model_index}"""						)
         os.makedirs(UpperCAmelCase      ,exist_ok=UpperCAmelCase						)
         logger.info(F"""Saving model to {ckpt_dir}"""						)
         A__       =     {'model': state_dict}

         dist_cp.save_state_dict(
             state_dict=UpperCAmelCase      ,storage_writer=dist_cp.FileSystemWriter(UpperCAmelCase						)      ,planner=DefaultSavePlanner()      ,)
         logger.info(F"""Model saved to {ckpt_dir}"""						)







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase=0						):






   '''simple docstring'''







   accelerator.wait_for_everyone()
   with FSDP.state_dict_type(
       UpperCAmelCase      ,fsdp_plugin.state_dict_type      ,fsdp_plugin.state_dict_config      ,fsdp_plugin.optim_state_dict_config						):
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         if type(UpperCAmelCase						) != FSDP and accelerator.process_index != 0:
            if not fsdp_plugin.sync_module_states:
               raise ValueError(
                   'Set the `sync_module_states` flag to `True` so that model states are synced across processes when '
                   'initializing FSDP object'						)
            return
         A__       =     F"""{MODEL_NAME}.bin""" if model_index == 0 else F"""{MODEL_NAME}_{model_index}.bin"""
         A__       =     os.path.join(UpperCAmelCase      ,UpperCAmelCase						)
         logger.info(F"""Loading model from {input_model_file}"""						)
         A__       =     torch.load(UpperCAmelCase						)
         logger.info(F"""Model loaded from {input_model_file}"""						)
      elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
         A__       =     (
             F"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
             if model_index == 0
             else F"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
         )
         A__       =     os.path.join(UpperCAmelCase      ,UpperCAmelCase						)
         logger.info(F"""Loading model from {input_model_file}"""						)
         A__       =     torch.load(UpperCAmelCase						)
         logger.info(F"""Model loaded from {input_model_file}"""						)
      elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
         A__       =     (
             os.path.join(UpperCAmelCase      ,F"""{MODEL_NAME}_{model_index}"""						)
             if F"""{MODEL_NAME}""" not in input_dir
             else input_dir
         )
         logger.info(F"""Loading model from {ckpt_dir}"""						)
         A__       =     {'model': model.state_dict()}
         dist_cp.load_state_dict(
             state_dict=UpperCAmelCase      ,storage_reader=dist_cp.FileSystemReader(UpperCAmelCase						)      ,planner=DefaultLoadPlanner()      ,)
         A__       =     state_dict['model']
         logger.info(F"""Model loaded from {ckpt_dir}"""						)
      model.load_state_dict(UpperCAmelCase						)







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase=0						):






   '''simple docstring'''







   os.makedirs(UpperCAmelCase      ,exist_ok=UpperCAmelCase						)
   with FSDP.state_dict_type(
       UpperCAmelCase      ,fsdp_plugin.state_dict_type      ,fsdp_plugin.state_dict_config      ,fsdp_plugin.optim_state_dict_config						):
      A__       =     FSDP.optim_state_dict(UpperCAmelCase      ,UpperCAmelCase						)
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         if accelerator.process_index == 0:
            A__       =     (
                F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
            )
            A__       =     os.path.join(UpperCAmelCase      ,UpperCAmelCase						)
            logger.info(F"""Saving Optimizer state to {output_optimizer_file}"""						)
            torch.save(UpperCAmelCase      ,UpperCAmelCase						)
            logger.info(F"""Optimizer state saved in {output_optimizer_file}"""						)
      else:
         A__       =     os.path.join(UpperCAmelCase      ,F"""{OPTIMIZER_NAME}_{optimizer_index}"""						)
         os.makedirs(UpperCAmelCase      ,exist_ok=UpperCAmelCase						)
         logger.info(F"""Saving Optimizer state to {ckpt_dir}"""						)
         dist_cp.save_state_dict(
             state_dict={'optimizer': optim_state}      ,storage_writer=dist_cp.FileSystemWriter(UpperCAmelCase						)      ,planner=DefaultSavePlanner()      ,)
         logger.info(F"""Optimizer state saved in {ckpt_dir}"""						)







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase=0						):






   '''simple docstring'''







   accelerator.wait_for_everyone()
   with FSDP.state_dict_type(
       UpperCAmelCase      ,fsdp_plugin.state_dict_type      ,fsdp_plugin.state_dict_config      ,fsdp_plugin.optim_state_dict_config						):
      if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
         A__       =     None
         # below check should work but currently it isn't working (mostly opytorch issue),
         # in the meantime disabling it at the cost of excess memory usage
         # if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
         A__       =     (
             F"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else F"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
         )
         A__       =     os.path.join(UpperCAmelCase      ,UpperCAmelCase						)
         logger.info(F"""Loading Optimizer state from {input_optimizer_file}"""						)
         A__       =     torch.load(UpperCAmelCase						)
         logger.info(F"""Optimizer state loaded from {input_optimizer_file}"""						)
      else:
         A__       =     (
             os.path.join(UpperCAmelCase      ,F"""{OPTIMIZER_NAME}_{optimizer_index}"""						)
             if F"""{OPTIMIZER_NAME}""" not in input_dir
             else input_dir
         )
         logger.info(F"""Loading Optimizer from {ckpt_dir}"""						)
         A__       =     load_sharded_optimizer_state_dict(
             model_state_dict=model.state_dict()      ,optimizer_key='optimizer'      ,storage_reader=dist_cp.FileSystemReader(UpperCAmelCase						)      ,)
         A__       =     optim_state['optimizer']
         logger.info(F"""Optimizer loaded from {ckpt_dir}"""						)
      A__       =     FSDP.optim_state_dict_to_load(UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase						)
      optimizer.load_state_dict(UpperCAmelCase						)


