

import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler, UNetaDConditionModel
from diffusers.pipelines.semantic_stable_diffusion import SemanticStableDiffusionPipeline as StableDiffusionPipeline
from diffusers.utils import floats_tensor, nightly, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu


enable_full_determinism()

class  a     (		unittest.TestCase ):


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										super().tearDown()
										gc.collect()
										torch.cuda.empty_cache()


						@property
						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						Dict       						=			1
										_UpperCAmelCase				:						Union[str, Any]       						=			3
										_UpperCAmelCase				:						Any       						=			(32, 32)

										_UpperCAmelCase				:						List[str]       						=			floats_tensor((batch_size, num_channels) + sizes  ,    rng=random.Random(0		)		).to(A_		)
										return image


						@property
						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										torch.manual_seed(0		)
										_UpperCAmelCase				:						Dict       						=			UNetaDConditionModel(
										    block_out_channels=(32, 64)  ,    layers_per_block=2  ,    sample_size=32  ,    in_channels=4  ,    out_channels=4  ,    down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")  ,    up_block_types=("CrossAttnUpBlock2D", "UpBlock2D")  ,    cross_attention_dim=32  ,    )
										return model


						@property
						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										torch.manual_seed(0		)
										_UpperCAmelCase				:						Optional[int]       						=			AutoencoderKL(
										    block_out_channels=[32, 64]  ,    in_channels=3  ,    out_channels=3  ,    down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"]  ,    up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"]  ,    latent_channels=4  ,    )
										return model


						@property
						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										torch.manual_seed(0		)
										_UpperCAmelCase				:						Tuple       						=			CLIPTextConfig(
										    bos_token_id=0  ,    eos_token_id=2  ,    hidden_size=32  ,    intermediate_size=37  ,    layer_norm_eps=1e-05  ,    num_attention_heads=4  ,    num_hidden_layers=5  ,    pad_token_id=1  ,    vocab_size=1000  ,    )
										return CLIPTextModel(A_		)


						@property
						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										def extract(*A_  ,    **A_		):

														class  a     :


																				def __init__(				self		):







																								'''simple docstring'''


																								_UpperCAmelCase				:						Dict       						=			torch.ones([0]		)





																				def 				_UpperCAmelCase (				self  ,    A_		):







																								'''simple docstring'''


																								self.pixel_values.to(A_		)
																								return self







														return Out()

										return extract


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						Dict       						=			"cpu"  # ensure determinism for the device-dependent torch.Generator
										_UpperCAmelCase				:						int       						=			self.dummy_cond_unet
										_UpperCAmelCase				:						str       						=			DDIMScheduler(
										    beta_start=0.0_00_85  ,    beta_end=0.0_12  ,    beta_schedule="scaled_linear"  ,    clip_sample=A_  ,    set_alpha_to_one=A_  ,    )

										_UpperCAmelCase				:						int       						=			self.dummy_vae
										_UpperCAmelCase				:						Optional[int]       						=			self.dummy_text_encoder
										_UpperCAmelCase				:						int       						=			CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"		)

										# make sure here that pndm scheduler skips prk
										_UpperCAmelCase				:						Tuple       						=			StableDiffusionPipeline(
										    unet=A_  ,    scheduler=A_  ,    vae=A_  ,    text_encoder=A_  ,    tokenizer=A_  ,    safety_checker=A_  ,    feature_extractor=self.dummy_extractor  ,    )
										_UpperCAmelCase				:						str       						=			sd_pipe.to(A_		)
										sd_pipe.set_progress_bar_config(disable=A_		)

										_UpperCAmelCase				:						Dict       						=			"A painting of a squirrel eating a burger"

										_UpperCAmelCase				:						Optional[int]       						=			torch.Generator(device=A_		).manual_seed(0		)
										_UpperCAmelCase				:						List[str]       						=			sd_pipe([prompt]  ,    generator=A_  ,    guidance_scale=6.0  ,    num_inference_steps=2  ,    output_type="np"		)
										_UpperCAmelCase				:						Any       						=			output.images

										_UpperCAmelCase				:						Any       						=			torch.Generator(device=A_		).manual_seed(0		)
										_UpperCAmelCase				:						int       						=			sd_pipe(
										    [prompt]  ,    generator=A_  ,    guidance_scale=6.0  ,    num_inference_steps=2  ,    output_type="np"  ,    return_dict=A_  ,    )[0]

										_UpperCAmelCase				:						Dict       						=			image[0, -3:, -3:, -1]
										_UpperCAmelCase				:						Union[str, Any]       						=			image_from_tuple[0, -3:, -3:, -1]

										assert image.shape == (1, 64, 64, 3)
										_UpperCAmelCase				:						List[str]       						=			np.array([0.57_53, 0.61_14, 0.50_01, 0.50_34, 0.54_70, 0.47_29, 0.49_71, 0.48_67, 0.48_67]		)

										assert np.abs(image_slice.flatten() - expected_slice		).max() < 1e-2
										assert np.abs(image_from_tuple_slice.flatten() - expected_slice		).max() < 1e-2


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						Optional[int]       						=			"cpu"  # ensure determinism for the device-dependent torch.Generator
										_UpperCAmelCase				:						Union[str, Any]       						=			self.dummy_cond_unet
										_UpperCAmelCase				:						Dict       						=			PNDMScheduler(skip_prk_steps=A_		)
										_UpperCAmelCase				:						List[Any]       						=			self.dummy_vae
										_UpperCAmelCase				:						List[Any]       						=			self.dummy_text_encoder
										_UpperCAmelCase				:						Optional[int]       						=			CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"		)

										# make sure here that pndm scheduler skips prk
										_UpperCAmelCase				:						Dict       						=			StableDiffusionPipeline(
										    unet=A_  ,    scheduler=A_  ,    vae=A_  ,    text_encoder=A_  ,    tokenizer=A_  ,    safety_checker=A_  ,    feature_extractor=self.dummy_extractor  ,    )
										_UpperCAmelCase				:						str       						=			sd_pipe.to(A_		)
										sd_pipe.set_progress_bar_config(disable=A_		)

										_UpperCAmelCase				:						int       						=			"A painting of a squirrel eating a burger"
										_UpperCAmelCase				:						Union[str, Any]       						=			torch.Generator(device=A_		).manual_seed(0		)
										_UpperCAmelCase				:						Dict       						=			sd_pipe([prompt]  ,    generator=A_  ,    guidance_scale=6.0  ,    num_inference_steps=2  ,    output_type="np"		)

										_UpperCAmelCase				:						str       						=			output.images

										_UpperCAmelCase				:						List[Any]       						=			torch.Generator(device=A_		).manual_seed(0		)
										_UpperCAmelCase				:						Union[str, Any]       						=			sd_pipe(
										    [prompt]  ,    generator=A_  ,    guidance_scale=6.0  ,    num_inference_steps=2  ,    output_type="np"  ,    return_dict=A_  ,    )[0]

										_UpperCAmelCase				:						int       						=			image[0, -3:, -3:, -1]
										_UpperCAmelCase				:						Union[str, Any]       						=			image_from_tuple[0, -3:, -3:, -1]

										assert image.shape == (1, 64, 64, 3)
										_UpperCAmelCase				:						Optional[int]       						=			np.array([0.51_22, 0.57_12, 0.48_25, 0.50_53, 0.56_46, 0.47_69, 0.51_79, 0.48_94, 0.49_94]		)

										assert np.abs(image_slice.flatten() - expected_slice		).max() < 1e-2
										assert np.abs(image_from_tuple_slice.flatten() - expected_slice		).max() < 1e-2


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						List[str]       						=			StableDiffusionPipeline.from_pretrained(
										    "hf-internal-testing/tiny-stable-diffusion-lms-pipe"  ,    safety_checker=A_		)
										assert isinstance(A_  ,    A_		)
										assert isinstance(pipe.scheduler  ,    A_		)
										assert pipe.safety_checker is None

										_UpperCAmelCase				:						Optional[int]       						=			pipe("example prompt"  ,    num_inference_steps=2		).images[0]
										assert image is not None

										# check that there's no error when saving a pipeline with one of the models being None
										with tempfile.TemporaryDirectory() as tmpdirname:
														pipe.save_pretrained(A_		)
														_UpperCAmelCase				:						str       						=			StableDiffusionPipeline.from_pretrained(A_		)

										# sanity check that the pipeline still works
										assert pipe.safety_checker is None
										_UpperCAmelCase				:						List[Any]       						=			pipe("example prompt"  ,    num_inference_steps=2		).images[0]
										assert image is not None





						@unittest.skipIf(torch_device != "cuda"  ,    "This test requires a GPU"		)
						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						Optional[int]       						=			self.dummy_cond_unet
										_UpperCAmelCase				:						Any       						=			PNDMScheduler(skip_prk_steps=A_		)
										_UpperCAmelCase				:						Any       						=			self.dummy_vae
										_UpperCAmelCase				:						Any       						=			self.dummy_text_encoder
										_UpperCAmelCase				:						Union[str, Any]       						=			CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"		)

										# put models in fp16
										_UpperCAmelCase				:						int       						=			unet.half()
										_UpperCAmelCase				:						str       						=			vae.half()
										_UpperCAmelCase				:						Optional[Any]       						=			bert.half()

										# make sure here that pndm scheduler skips prk
										_UpperCAmelCase				:						Union[str, Any]       						=			StableDiffusionPipeline(
										    unet=A_  ,    scheduler=A_  ,    vae=A_  ,    text_encoder=A_  ,    tokenizer=A_  ,    safety_checker=A_  ,    feature_extractor=self.dummy_extractor  ,    )
										_UpperCAmelCase				:						Tuple       						=			sd_pipe.to(A_		)
										sd_pipe.set_progress_bar_config(disable=A_		)

										_UpperCAmelCase				:						Union[str, Any]       						=			"A painting of a squirrel eating a burger"
										_UpperCAmelCase				:						List[Any]       						=			sd_pipe([prompt]  ,    num_inference_steps=2  ,    output_type="np"		).images

										assert image.shape == (1, 64, 64, 3)







@nightly
@require_torch_gpu
class  a     (		unittest.TestCase ):


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										super().tearDown()
										gc.collect()
										torch.cuda.empty_cache()


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						List[Any]       						=			"cuda"
										_UpperCAmelCase				:						str       						=			StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5"		)
										_UpperCAmelCase				:						Union[str, Any]       						=			pipe.to(A_		)
										pipe.set_progress_bar_config(disable=A_		)

										_UpperCAmelCase				:						Union[str, Any]       						=			"a photo of a cat"
										_UpperCAmelCase				:						Any       						=			{
										    "editing_prompt": ["sunglasses"],
										    "reverse_editing_direction": [False],
										    "edit_warmup_steps": 10,
										    "edit_guidance_scale": 6,
										    "edit_threshold": 0.95,
										    "edit_momentum_scale": 0.5,
										    "edit_mom_beta": 0.6,
										}

										_UpperCAmelCase				:						Union[str, Any]       						=			3
										_UpperCAmelCase				:						Optional[Any]       						=			7

										# no sega enabled
										_UpperCAmelCase				:						Any       						=			torch.Generator(A_		)
										generator.manual_seed(A_		)
										_UpperCAmelCase				:						Optional[Any]       						=			pipe(
										    [prompt]  ,    generator=A_  ,    guidance_scale=A_  ,    num_inference_steps=50  ,    output_type="np"  ,    width=512  ,    height=512  ,    )

										_UpperCAmelCase				:						Optional[Any]       						=			output.images
										_UpperCAmelCase				:						Union[str, Any]       						=			image[0, -3:, -3:, -1]
										_UpperCAmelCase				:						Tuple       						=			[
										    0.34_67_31_13,
										    0.38_49_27_33,
										    0.37_59_73_52,
										    0.34_08_63_35,
										    0.35_65_07_48,
										    0.35_57_92_05,
										    0.3_38_47_63,
										    0.34_34_02_36,
										    0.3_57_32_71,
										]

										assert image.shape == (1, 512, 512, 3)

										assert np.abs(image_slice.flatten() - expected_slice		).max() < 1e-2

										# with sega enabled
										# generator = torch.manual_seed(seed)
										generator.manual_seed(A_		)
										_UpperCAmelCase				:						Dict       						=			pipe(
										    [prompt]  ,    generator=A_  ,    guidance_scale=A_  ,    num_inference_steps=50  ,    output_type="np"  ,    width=512  ,    height=512  ,    **A_  ,    )

										_UpperCAmelCase				:						Tuple       						=			output.images
										_UpperCAmelCase				:						Optional[int]       						=			image[0, -3:, -3:, -1]
										_UpperCAmelCase				:						Any       						=			[
										    0.41_88_78_26,
										    0.37_72_87_66,
										    0.30_13_82_72,
										    0.41_41_63_35,
										    0.41_66_49_85,
										    0.36_28_33_92,
										    0.36_19_12_46,
										    0.43_36_44_65,
										    0.43_00_17_32,
										]

										assert image.shape == (1, 512, 512, 3)

										assert np.abs(image_slice.flatten() - expected_slice		).max() < 1e-2


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						List[str]       						=			"cuda"
										_UpperCAmelCase				:						Dict       						=			StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5"		)
										_UpperCAmelCase				:						str       						=			pipe.to(A_		)
										pipe.set_progress_bar_config(disable=A_		)

										_UpperCAmelCase				:						Union[str, Any]       						=			"an image of a crowded boulevard, realistic, 4k"
										_UpperCAmelCase				:						Union[str, Any]       						=			{
										    "editing_prompt": "crowd, crowded, people",
										    "reverse_editing_direction": True,
										    "edit_warmup_steps": 10,
										    "edit_guidance_scale": 8.3,
										    "edit_threshold": 0.9,
										    "edit_momentum_scale": 0.5,
										    "edit_mom_beta": 0.6,
										}

										_UpperCAmelCase				:						List[str]       						=			9
										_UpperCAmelCase				:						List[str]       						=			7

										# no sega enabled
										_UpperCAmelCase				:						Tuple       						=			torch.Generator(A_		)
										generator.manual_seed(A_		)
										_UpperCAmelCase				:						Any       						=			pipe(
										    [prompt]  ,    generator=A_  ,    guidance_scale=A_  ,    num_inference_steps=50  ,    output_type="np"  ,    width=512  ,    height=512  ,    )

										_UpperCAmelCase				:						Optional[Any]       						=			output.images
										_UpperCAmelCase				:						List[Any]       						=			image[0, -3:, -3:, -1]
										_UpperCAmelCase				:						Dict       						=			[
										    0.43_49_79_98,
										    0.91_81_40_65,
										    0.7_54_07_39,
										    0.55_58_02_05,
										    0.8_46_72_65,
										    0.5_38_96_91,
										    0.62_57_45_06,
										    0.58_89_77_63,
										    0.50_92_67_57,
										]

										assert image.shape == (1, 512, 512, 3)

										assert np.abs(image_slice.flatten() - expected_slice		).max() < 1e-2

										# with sega enabled
										# generator = torch.manual_seed(seed)
										generator.manual_seed(A_		)
										_UpperCAmelCase				:						int       						=			pipe(
										    [prompt]  ,    generator=A_  ,    guidance_scale=A_  ,    num_inference_steps=50  ,    output_type="np"  ,    width=512  ,    height=512  ,    **A_  ,    )

										_UpperCAmelCase				:						str       						=			output.images
										_UpperCAmelCase				:						List[Any]       						=			image[0, -3:, -3:, -1]
										_UpperCAmelCase				:						Optional[int]       						=			[
										    0.3_08_97_19,
										    0.30_50_01_44,
										    0.29_01_60_42,
										    0.30_63_09_64,
										    0.32_56_87,
										    0.29_41_92_25,
										    0.2_90_80_91,
										    0.28_72_35_98,
										    0.27_69_62_94,
										]

										assert image.shape == (1, 512, 512, 3)

										assert np.abs(image_slice.flatten() - expected_slice		).max() < 1e-2


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						str       						=			"cuda"
										_UpperCAmelCase				:						List[Any]       						=			StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5"		)
										_UpperCAmelCase				:						Dict       						=			pipe.to(A_		)
										pipe.set_progress_bar_config(disable=A_		)

										_UpperCAmelCase				:						Optional[int]       						=			"a castle next to a river"
										_UpperCAmelCase				:						Dict       						=			{
										    "editing_prompt": ["boat on a river, boat", "monet, impression, sunrise"],
										    "reverse_editing_direction": False,
										    "edit_warmup_steps": [15, 18],
										    "edit_guidance_scale": 6,
										    "edit_threshold": [0.9, 0.8],
										    "edit_momentum_scale": 0.5,
										    "edit_mom_beta": 0.6,
										}

										_UpperCAmelCase				:						Optional[Any]       						=			48
										_UpperCAmelCase				:						Dict       						=			7

										# no sega enabled
										_UpperCAmelCase				:						Dict       						=			torch.Generator(A_		)
										generator.manual_seed(A_		)
										_UpperCAmelCase				:						List[str]       						=			pipe(
										    [prompt]  ,    generator=A_  ,    guidance_scale=A_  ,    num_inference_steps=50  ,    output_type="np"  ,    width=512  ,    height=512  ,    )

										_UpperCAmelCase				:						Dict       						=			output.images
										_UpperCAmelCase				:						Optional[Any]       						=			image[0, -3:, -3:, -1]
										_UpperCAmelCase				:						Tuple       						=			[
										    0.75_16_35_55,
										    0.76_03_71_45,
										    0.6_17_85,
										    0.9_18_96_73,
										    0.8_62_77_01,
										    0.85_18_96_94,
										    0.8_51_28_13,
										    0.87_01_20_76,
										    0.8_31_28_57,
										]

										assert image.shape == (1, 512, 512, 3)

										assert np.abs(image_slice.flatten() - expected_slice		).max() < 1e-2

										# with sega enabled
										# generator = torch.manual_seed(seed)
										generator.manual_seed(A_		)
										_UpperCAmelCase				:						Dict       						=			pipe(
										    [prompt]  ,    generator=A_  ,    guidance_scale=A_  ,    num_inference_steps=50  ,    output_type="np"  ,    width=512  ,    height=512  ,    **A_  ,    )

										_UpperCAmelCase				:						Optional[int]       						=			output.images
										_UpperCAmelCase				:						str       						=			image[0, -3:, -3:, -1]
										_UpperCAmelCase				:						Dict       						=			[
										    0.73_55_33_65,
										    0.7_53_72_71,
										    0.74_34_19_05,
										    0.66_48_03_56,
										    0.6_47_29_25,
										    0.63_03_94_16,
										    0.64_81_29_05,
										    0.6_74_97_17,
										    0.6_51_71_02,
										]

										assert image.shape == (1, 512, 512, 3)

										assert np.abs(image_slice.flatten() - expected_slice		).max() < 1e-2





						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						List[Any]       						=			"cuda"
										_UpperCAmelCase				:						Any       						=			StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5"  ,    torch_dtype=torch.floataa		)
										_UpperCAmelCase				:						Union[str, Any]       						=			pipe.to(A_		)
										pipe.set_progress_bar_config(disable=A_		)

										_UpperCAmelCase				:						Union[str, Any]       						=			"a photo of a cat"
										_UpperCAmelCase				:						Optional[Any]       						=			{
										    "editing_prompt": ["sunglasses"],
										    "reverse_editing_direction": [False],
										    "edit_warmup_steps": 10,
										    "edit_guidance_scale": 6,
										    "edit_threshold": 0.95,
										    "edit_momentum_scale": 0.5,
										    "edit_mom_beta": 0.6,
										}

										_UpperCAmelCase				:						Dict       						=			3
										_UpperCAmelCase				:						Any       						=			7

										# no sega enabled
										_UpperCAmelCase				:						Optional[int]       						=			torch.Generator(A_		)
										generator.manual_seed(A_		)
										_UpperCAmelCase				:						Optional[Any]       						=			pipe(
										    [prompt]  ,    generator=A_  ,    guidance_scale=A_  ,    num_inference_steps=50  ,    output_type="np"  ,    width=512  ,    height=512  ,    )

										_UpperCAmelCase				:						Any       						=			output.images
										_UpperCAmelCase				:						Union[str, Any]       						=			image[0, -3:, -3:, -1]
										_UpperCAmelCase				:						Dict       						=			[
										    0.34_88_76_95,
										    0.3_87_69_53,
										    0.3_75,
										    0.34_42_38_28,
										    0.3_58_15_43,
										    0.35_71_77_73,
										    0.3_38_37_89,
										    0.34_57_03_12,
										    0.35_93_75,
										]

										assert image.shape == (1, 512, 512, 3)

										assert np.abs(image_slice.flatten() - expected_slice		).max() < 1e-2

										# with sega enabled
										# generator = torch.manual_seed(seed)
										generator.manual_seed(A_		)
										_UpperCAmelCase				:						Dict       						=			pipe(
										    [prompt]  ,    generator=A_  ,    guidance_scale=A_  ,    num_inference_steps=50  ,    output_type="np"  ,    width=512  ,    height=512  ,    **A_  ,    )

										_UpperCAmelCase				:						Union[str, Any]       						=			output.images
										_UpperCAmelCase				:						str       						=			image[0, -3:, -3:, -1]
										_UpperCAmelCase				:						Any       						=			[
										    0.42_28_51_56,
										    0.36_91_40_62,
										    0.29_07_71_48,
										    0.42_04_10_16,
										    0.41_91_89_45,
										    0.35_49_80_47,
										    0.3_61_81_64,
										    0.4_42_38_28,
										    0.43_11_52_34,
										]

										assert image.shape == (1, 512, 512, 3)

										assert np.abs(image_slice.flatten() - expected_slice		).max() < 1e-2








import os
import random
from pathlib import Path
from typing import List

import numpy as np
import torch
from torch.cuda.amp import GradScaler

from .utils import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    get_pretty_name,
    is_tpu_available,
    is_xpu_available,
    save,
)


if is_tpu_available(check_device=False):
      import torch_xla.core.xla_model as xm

from .logging import get_logger
from .state import PartialState


__SCREAMING_SNAKE_CASE			:      Union[str, Any]      			=get_logger(__name__)

def 	lowerCAmelCase_	(     lowerCamelCase:				str   , lowerCamelCase:				List[dict]   , lowerCamelCase:				list   , lowerCamelCase:				list   , lowerCamelCase:				int   , lowerCamelCase:				GradScaler = None   , )    ->	Any:


      '''simple docstring'''




      for i, state in enumerate(lowerCamelCase  ):
            lowerCamelCase__					     = f"""{MODEL_NAME}.bin""" if i == 0 else f"""{MODEL_NAME}_{i}.bin"""
            lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
            save(lowerCamelCase   , lowerCamelCase  )
            logger.info(f"""Model weights saved in {output_model_file}"""  )
      # Optimizer states
      for i, opt in enumerate(lowerCamelCase  ):
            lowerCamelCase__					     = opt.state_dict()
            lowerCamelCase__					     = f"""{OPTIMIZER_NAME}.bin""" if i == 0 else f"""{OPTIMIZER_NAME}_{i}.bin"""
            lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
            save(lowerCamelCase   , lowerCamelCase  )
            logger.info(f"""Optimizer state saved in {output_optimizer_file}"""  )
      # Scheduler states
      for i, scheduler in enumerate(lowerCamelCase  ):
            lowerCamelCase__					     = scheduler.state_dict()
            lowerCamelCase__					     = f"""{SCHEDULER_NAME}.bin""" if i == 0 else f"""{SCHEDULER_NAME}_{i}.bin"""
            lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
            save(lowerCamelCase   , lowerCamelCase  )
            logger.info(f"""Scheduler state saved in {output_scheduler_file}"""  )
      # GradScaler state
      if scaler is not None:
            lowerCamelCase__					     = scaler.state_dict()
            lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
            torch.save(lowerCamelCase   , lowerCamelCase  )
            logger.info(f"""Gradient scaler state saved in {output_scaler_file}"""  )
      # Random number generator states
      lowerCamelCase__					     = {}
      lowerCamelCase__					     = f"""{RNG_STATE_NAME}_{process_index}.pkl"""
      lowerCamelCase__					     = random.getstate()
      lowerCamelCase__					     = np.random.get_state()
      lowerCamelCase__					     = torch.get_rng_state()
      if is_xpu_available():
            lowerCamelCase__					     = torch.xpu.get_rng_state_all()
      else:
            lowerCamelCase__					     = torch.cuda.get_rng_state_all()
      if is_tpu_available():
            lowerCamelCase__					     = xm.get_rng_state()
      lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
      torch.save(lowerCamelCase   , lowerCamelCase  )
      logger.info(f"""Random states saved in {output_states_file}"""  )
      return output_dir

def 	lowerCAmelCase_	(     lowerCamelCase:				int   , lowerCamelCase:				Dict   , lowerCamelCase:				Union[str, Any]   , lowerCamelCase:				Any   , lowerCamelCase:				Any   , lowerCamelCase:				Any=None   , lowerCamelCase:				Dict=None   , **lowerCamelCase:				Optional[Any]   , )    ->	Any:


      '''simple docstring'''




      if map_location not in [None, "cpu", "on_device"]:
            raise TypeError(
                'Unsupported optimizer map location passed, please choose one of `None`, `\'cpu\'`, or `\'on_device\'`'  )
      if map_location is None:
            lowerCamelCase__					     = 'cpu'
      elif map_location == "on_device":
            lowerCamelCase__					     = PartialState().device
      # Model states
      for i, model in enumerate(lowerCamelCase  ):
            lowerCamelCase__					     = f"""{MODEL_NAME}.bin""" if i == 0 else f"""{MODEL_NAME}_{i}.bin"""
            lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
            models[i].load_state_dict(torch.load(lowerCamelCase   , map_location=lowerCamelCase  )   , **lowerCamelCase  )
      logger.info('All model weights loaded successfully'  )

      # Optimizer states
      for i, opt in enumerate(lowerCamelCase  ):
            lowerCamelCase__					     = f"""{OPTIMIZER_NAME}.bin""" if i == 0 else f"""{OPTIMIZER_NAME}_{i}.bin"""
            lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
            lowerCamelCase__					     = torch.load(lowerCamelCase   , map_location=lowerCamelCase  )
            optimizers[i].load_state_dict(lowerCamelCase  )
      logger.info('All optimizer states loaded successfully'  )

      # Scheduler states
      for i, scheduler in enumerate(lowerCamelCase  ):
            lowerCamelCase__					     = f"""{SCHEDULER_NAME}.bin""" if i == 0 else f"""{SCHEDULER_NAME}_{i}.bin"""
            lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
            scheduler.load_state_dict(torch.load(lowerCamelCase  )  )
      logger.info('All scheduler states loaded successfully'  )

      # GradScaler state
      if scaler is not None:
            lowerCamelCase__					     = os.path.join(lowerCamelCase   , lowerCamelCase  )
            scaler.load_state_dict(torch.load(lowerCamelCase  )  )
            logger.info('GradScaler state loaded successfully'  )

      # Random states
      try:
            lowerCamelCase__					     = torch.load(os.path.join(lowerCamelCase   , f"""{RNG_STATE_NAME}_{process_index}.pkl"""  )  )
            random.setstate(states['random_state']  )
            np.random.set_state(states['numpy_random_seed']  )
            torch.set_rng_state(states['torch_manual_seed']  )
            if is_xpu_available():
                  torch.xpu.set_rng_state_all(states['torch_xpu_manual_seed']  )
            else:
                  torch.cuda.set_rng_state_all(states['torch_cuda_manual_seed']  )
            if is_tpu_available():
                  xm.set_rng_state(states['xm_seed']  )
            logger.info('All random states loaded successfully'  )
      except Exception:
            logger.info('Could not load random states'  )

def 	lowerCAmelCase_	(     lowerCamelCase:				Dict   , lowerCamelCase:				Any   , lowerCamelCase:				int = 0  )    ->	Union[str, Any]:


      '''simple docstring'''




      lowerCamelCase__					     = Path(lowerCamelCase  ) / f"""custom_checkpoint_{index}.pkl"""
      logger.info(f"""Saving the state of {get_pretty_name(lowerCamelCase  )} to {save_location}"""  )
      torch.save(obj.state_dict()   , lowerCamelCase  )


def 	lowerCAmelCase_	(     lowerCamelCase:				int   , lowerCamelCase:				Optional[int]   , lowerCamelCase:				int = 0  )    ->	Any:


      '''simple docstring'''




      lowerCamelCase__					     = f"""{path}/custom_checkpoint_{index}.pkl"""
      logger.info(f"""Loading the state of {get_pretty_name(lowerCamelCase  )} from {load_location}"""  )
      obj.load_state_dict(torch.load(lowerCamelCase   , map_location='cpu'  )  )