






import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    StableDiffusionSAGPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()






class 		lowercase						(			__SCREAMING_SNAKE_CASE			,   __SCREAMING_SNAKE_CASE			,   unittest.TestCase):






	"""simple docstring"""


	a__				: str		       =							StableDiffusionSAGPipeline
	a__				: Dict		       =							TEXT_TO_IMAGE_PARAMS
	a__				: List[Any]		       =							TEXT_TO_IMAGE_BATCH_PARAMS
	a__				: Optional[int]		       =							TEXT_TO_IMAGE_IMAGE_PARAMS
	a__				: List[Any]		       =							TEXT_TO_IMAGE_IMAGE_PARAMS
	a__				: int		       =							False




	def   _SCREAMING_SNAKE_CASE		(     self     :						List[Any]					)      ->			int:
							torch.manual_seed(0					)
							UpperCAmelCase_=       UNetaDConditionModel(
							    block_out_channels=(32, 64)						,  layers_per_block=2						,  sample_size=32						,  in_channels=4						,  out_channels=4						,  down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")						,  up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")						,  cross_attention_dim=32						,  )
							UpperCAmelCase_=       DDIMScheduler(
							    beta_start=0.00_085						,  beta_end=0.012						,  beta_schedule="""scaled_linear"""						,  clip_sample=__UpperCAmelCase						,  set_alpha_to_one=__UpperCAmelCase						,  )
							torch.manual_seed(0					)
							UpperCAmelCase_=       AutoencoderKL(
							    block_out_channels=[32, 64]						,  in_channels=3						,  out_channels=3						,  down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]						,  up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]						,  latent_channels=4						,  )
							torch.manual_seed(0					)
							UpperCAmelCase_=       CLIPTextConfig(
							    bos_token_id=0						,  eos_token_id=2						,  hidden_size=32						,  intermediate_size=37						,  layer_norm_eps=1E-05						,  num_attention_heads=4						,  num_hidden_layers=5						,  pad_token_id=1						,  vocab_size=1_000						,  )
							UpperCAmelCase_=       CLIPTextModel(__UpperCAmelCase					)
							UpperCAmelCase_=       CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""					)

							UpperCAmelCase_=       {
							    """unet""": unet,
							    """scheduler""": scheduler,
							    """vae""": vae,
							    """text_encoder""": text_encoder,
							    """tokenizer""": tokenizer,
							    """safety_checker""": None,
							    """feature_extractor""": None,
							}
							return components




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]						,  __UpperCAmelCase     :						Union[str, Any]						,  __UpperCAmelCase     :						str=0					)      ->			Dict:
							if str(__UpperCAmelCase					).startswith("""mps"""					):
													UpperCAmelCase_=       torch.manual_seed(__UpperCAmelCase					)
							else:
													UpperCAmelCase_=       torch.Generator(device=__UpperCAmelCase					).manual_seed(__UpperCAmelCase					)
							UpperCAmelCase_=       {
							    """prompt""": """.""",
							    """generator""": generator,
							    """num_inference_steps""": 2,
							    """guidance_scale""": 1.0,
							    """sag_scale""": 1.0,
							    """output_type""": """numpy""",
							}
							return inputs




	def   _SCREAMING_SNAKE_CASE		(     self     :						Union[str, Any]					)      ->			List[str]:
							super().test_inference_batch_single_identical(expected_max_diff=3E-3					)






@slow
@require_torch_gpu
class 		lowercase						(			unittest.TestCase):






	"""simple docstring"""


	def   _SCREAMING_SNAKE_CASE		(     self     :						Dict					)      ->			Tuple:
							super().tearDown()
							gc.collect()
							torch.cuda.empty_cache()




	def   _SCREAMING_SNAKE_CASE		(     self     :						Dict					)      ->			Optional[Any]:
							UpperCAmelCase_=       StableDiffusionSAGPipeline.from_pretrained("""CompVis/stable-diffusion-v1-4"""					)
							UpperCAmelCase_=       sag_pipe.to(__UpperCAmelCase					)
							sag_pipe.set_progress_bar_config(disable=__UpperCAmelCase					)

							UpperCAmelCase_=       """."""
							UpperCAmelCase_=       torch.manual_seed(0					)
							UpperCAmelCase_=       sag_pipe(
							    [prompt]						,  generator=__UpperCAmelCase						,  guidance_scale=7.5						,  sag_scale=1.0						,  num_inference_steps=20						,  output_type="""np"""					)

							UpperCAmelCase_=       output.images

							UpperCAmelCase_=       image[0, -3:, -3:, -1]

							assert image.shape == (1, 512, 512, 3)
							UpperCAmelCase_=       np.array([0.1_568, 0.1_738, 0.1_695, 0.1_693, 0.1_507, 0.1_705, 0.1_547, 0.1_751, 0.1_949]					)

							assert np.abs(image_slice.flatten() - expected_slice					).max() < 5E-2




	def   _SCREAMING_SNAKE_CASE		(     self     :						Any					)      ->			Any:
							UpperCAmelCase_=       StableDiffusionSAGPipeline.from_pretrained("""stabilityai/stable-diffusion-2-1-base"""					)
							UpperCAmelCase_=       sag_pipe.to(__UpperCAmelCase					)
							sag_pipe.set_progress_bar_config(disable=__UpperCAmelCase					)

							UpperCAmelCase_=       """."""
							UpperCAmelCase_=       torch.manual_seed(0					)
							UpperCAmelCase_=       sag_pipe(
							    [prompt]						,  generator=__UpperCAmelCase						,  guidance_scale=7.5						,  sag_scale=1.0						,  num_inference_steps=20						,  output_type="""np"""					)

							UpperCAmelCase_=       output.images

							UpperCAmelCase_=       image[0, -3:, -3:, -1]

							assert image.shape == (1, 512, 512, 3)
							UpperCAmelCase_=       np.array([0.3_459, 0.2_876, 0.2_537, 0.3_002, 0.2_671, 0.2_160, 0.3_026, 0.2_262, 0.2_371]					)

							assert np.abs(image_slice.flatten() - expected_slice					).max() < 5E-2




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[int]					)      ->			List[str]:
							UpperCAmelCase_=       StableDiffusionSAGPipeline.from_pretrained("""stabilityai/stable-diffusion-2-1-base"""					)
							UpperCAmelCase_=       sag_pipe.to(__UpperCAmelCase					)
							sag_pipe.set_progress_bar_config(disable=__UpperCAmelCase					)

							UpperCAmelCase_=       """."""
							UpperCAmelCase_=       torch.manual_seed(0					)
							UpperCAmelCase_=       sag_pipe(
							    [prompt]						,  width=768						,  height=512						,  generator=__UpperCAmelCase						,  guidance_scale=7.5						,  sag_scale=1.0						,  num_inference_steps=20						,  output_type="""np"""						,  )

							UpperCAmelCase_=       output.images

							assert image.shape == (1, 512, 768, 3)









import json
import os
import tempfile
from unittest.mock import patch

import torch
from torch.utils.data import DataLoader, TensorDataset

from accelerate import DistributedType, infer_auto_device_map, init_empty_weights
from accelerate.accelerator import Accelerator
from accelerate.state import GradientState, PartialState
from accelerate.test_utils import require_bnb, require_multi_gpu, slow
from accelerate.test_utils.testing import AccelerateTestCase, require_cuda
from accelerate.utils import patch_environment



def 	__a					(   )				->			str:






						'''simple docstring'''



						UpperCAmelCase_=       torch.nn.Linear(2      ,4    )
						UpperCAmelCase_=       torch.optim.AdamW(model.parameters()      ,lr=1.0    )
						UpperCAmelCase_=       torch.optim.lr_scheduler.OneCycleLR(lowerCAmelCase_      ,max_lr=0.01      ,steps_per_epoch=2      ,epochs=1    )
						UpperCAmelCase_=       DataLoader(TensorDataset(torch.tensor([1, 2, 3]    )    )    )
						UpperCAmelCase_=       DataLoader(TensorDataset(torch.tensor([4, 5, 6]    )    )    )

						return model, optimizer, scheduler, train_dl, valid_dl



def 	__a					(   lowerCAmelCase_       : Any    )				->			Union[str, Any]:






						'''simple docstring'''



						return (model.weight.abs().sum() + model.bias.abs().sum()).item()



def 	__a					(   lowerCAmelCase_       : Tuple    )				->			Tuple:






						'''simple docstring'''



						UpperCAmelCase_=       torch.nn.Linear(*tuple(model.weight.T.shape    )    ).state_dict()
						model.load_state_dict(lowerCAmelCase_    )






class 		lowercase						(			snake_case__):






	"""simple docstring"""


	@require_cuda
	def   _SCREAMING_SNAKE_CASE		(     self     :						List[str]					)      ->			Optional[Any]:
							UpperCAmelCase_=       Accelerator()
							assert PartialState._shared_state["_cpu"] is False
							assert PartialState._shared_state["device"].type == "cuda"
							with self.assertRaises(__UpperCAmelCase					):
													UpperCAmelCase_=       Accelerator(cpu=__UpperCAmelCase					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Tuple					)      ->			Union[str, Any]:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_=       GradientState()
							assert state.num_steps == 1
							UpperCAmelCase_=       4
							assert state.num_steps == 4

							assert state.sync_gradients is True
							UpperCAmelCase_=       False
							assert state.sync_gradients is False
							GradientState._reset_state()




	def   _SCREAMING_SNAKE_CASE		(     self     :						Dict					)      ->			List[str]:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()

							(
							    (
							    UpperCAmelCase_
							),       (
							    UpperCAmelCase_
							),       (
							    UpperCAmelCase_
							),       (
							    UpperCAmelCase_
							),       (
							    UpperCAmelCase_
							),       
							)=       accelerator.prepare(__UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)

							self.assertTrue(prepared_model in accelerator._models					)
							self.assertTrue(prepared_optimizer in accelerator._optimizers					)
							self.assertTrue(prepared_scheduler in accelerator._schedulers					)
							self.assertTrue(prepared_train_dl in accelerator._dataloaders					)
							self.assertTrue(prepared_valid_dl in accelerator._dataloaders					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						int					)      ->			Dict:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()
							accelerator.prepare(__UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)
							accelerator.free_memory()

							self.assertTrue(len(accelerator._models					) == 0					)
							self.assertTrue(len(accelerator._optimizers					) == 0					)
							self.assertTrue(len(accelerator._schedulers					) == 0					)
							self.assertTrue(len(accelerator._dataloaders					) == 0					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Dict					)      ->			Optional[Any]:
							PartialState._reset_state()

							# Mock torch.cuda.set_device to avoid an exception as the device doesn't exist
							def noop(*__UpperCAmelCase     :						Dict						,  **__UpperCAmelCase     :						Tuple					):
													pass

							with patch("""torch.cuda.set_device"""						,  __UpperCAmelCase					), patch_environment(ACCELERATE_TORCH_DEVICE="""cuda:64"""					):
													UpperCAmelCase_=       Accelerator()
													self.assertEqual(str(accelerator.state.device					)						,  """cuda:64"""					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]					)      ->			List[Any]:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()
							accelerator.prepare(__UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)

							UpperCAmelCase_=       get_signature(__UpperCAmelCase					)

							with tempfile.TemporaryDirectory() as tmpdirname:
													accelerator.save_state(__UpperCAmelCase					)

													# make sure random weights don't match
													load_random_weights(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) > 1E-3					)

													# make sure loaded weights match
													accelerator.load_state(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) < 1E-3					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]					)      ->			List[Any]:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()
							accelerator.prepare(__UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)

							UpperCAmelCase_=       get_signature(__UpperCAmelCase					)

							# saving hook
							def save_config(__UpperCAmelCase     :						Tuple						,  __UpperCAmelCase     :						List[str]						,  __UpperCAmelCase     :						Tuple					):
													UpperCAmelCase_=       {"""class_name""": models[0].__class__.__name__}

													with open(os.path.join(__UpperCAmelCase						,  """data.json"""					)						,  """w"""					) as f:
																			json.dump(__UpperCAmelCase						,  __UpperCAmelCase					)

        # loading hook
							def load_config(__UpperCAmelCase     :						Tuple						,  __UpperCAmelCase     :						Union[str, Any]					):
													with open(os.path.join(__UpperCAmelCase						,  """data.json"""					)						,  """r"""					) as f:
																			UpperCAmelCase_=       json.load(__UpperCAmelCase					)

													UpperCAmelCase_=       config["""class_name"""]

							UpperCAmelCase_=       accelerator.register_save_state_pre_hook(__UpperCAmelCase					)
							UpperCAmelCase_=       accelerator.register_load_state_pre_hook(__UpperCAmelCase					)

							with tempfile.TemporaryDirectory() as tmpdirname:
													accelerator.save_state(__UpperCAmelCase					)

													# make sure random weights don't match with hooks
													load_random_weights(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) > 1E-3					)

													# random class name to verify correct one is loaded
													UpperCAmelCase_=       """random"""

													# make sure loaded weights match with hooks
													accelerator.load_state(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) < 1E-3					)

													# mode.class_name is loaded from config
													self.assertTrue(model.class_name == model.__class__.__name__					)

							# remove hooks
							save_hook.remove()
							load_hook.remove()

							with tempfile.TemporaryDirectory() as tmpdirname:
													accelerator.save_state(__UpperCAmelCase					)

													# make sure random weights don't match with hooks removed
													load_random_weights(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) > 1E-3					)

													# random class name to verify correct one is loaded
													UpperCAmelCase_=       """random"""

													# make sure loaded weights match with hooks removed
													accelerator.load_state(__UpperCAmelCase					)
													self.assertTrue(abs(model_signature - get_signature(__UpperCAmelCase					)					) < 1E-3					)

													# mode.class_name is NOT loaded from config
													self.assertTrue(model.class_name != model.__class__.__name__					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						Dict					)      ->			Union[str, Any]:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()
							UpperCAmelCase_=       None

							# This should work
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       accelerator.prepare(
							    __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)
							self.assertTrue(dummy_obj is None					)




	def   _SCREAMING_SNAKE_CASE		(     self     :						List[str]					)      ->			Any:
							UpperCAmelCase_=       Accelerator()
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       create_components()
							UpperCAmelCase_=       [1, 2, 3]

							# This should work
							UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_,       UpperCAmelCase_=       accelerator.prepare(
							    __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase					)
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Dummy object should have `_is_accelerate_prepared` set to `True`"""						,  )
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Model is missing `_is_accelerator_prepared` or is set to `False`"""						,  )
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Optimizer is missing `_is_accelerator_prepared` or is set to `False`"""						,  )
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Scheduler is missing `_is_accelerator_prepared` or is set to `False`"""						,  )
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Train Dataloader is missing `_is_accelerator_prepared` or is set to `False`"""						,  )
							self.assertEqual(
							    getattr(__UpperCAmelCase						,  """_is_accelerate_prepared"""						,  __UpperCAmelCase					)						,  __UpperCAmelCase						,  """Valid Dataloader is missing `_is_accelerator_prepared` or is set to `False`"""						,  )




	@slow
	@require_bnb
	def   _SCREAMING_SNAKE_CASE		(     self     :						Union[str, Any]					)      ->			Optional[Any]:
							from transformers import AutoModelForCausalLM

							UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
							    """EleutherAI/gpt-neo-125m"""						,  load_in_abit=__UpperCAmelCase						,  device_map={"""""": 0}						,  )
							UpperCAmelCase_=       Accelerator()

							# This should work
							UpperCAmelCase_=       accelerator.prepare(__UpperCAmelCase					)




	@slow
	@require_bnb
	def   _SCREAMING_SNAKE_CASE		(     self     :						Union[str, Any]					)      ->			Tuple:
							from transformers import AutoModelForCausalLM

							UpperCAmelCase_=       Accelerator()

							with init_empty_weights():
													UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
													    """EleutherAI/gpt-neo-125m"""						,  )
													model.tie_weights()
													UpperCAmelCase_=       infer_auto_device_map(__UpperCAmelCase					)
													UpperCAmelCase_=       """cpu"""

							UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
							    """EleutherAI/gpt-neo-125m"""						,  device_map=__UpperCAmelCase						,  load_in_abit=__UpperCAmelCase						,  llm_inta_enable_fpaa_cpu_offload=__UpperCAmelCase					)

							# This should not work and get value error
							with self.assertRaises(__UpperCAmelCase					):
													UpperCAmelCase_=       accelerator.prepare(__UpperCAmelCase					)




	@slow
	@require_bnb
	@require_multi_gpu
	def   _SCREAMING_SNAKE_CASE		(     self     :						int					)      ->			Tuple:
							from transformers import AutoModelForCausalLM

							UpperCAmelCase_=       {"""distributed_type""": DistributedType.MULTI_GPU}

							with init_empty_weights():
													UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
													    """EleutherAI/gpt-neo-125m"""						,  )
													model.tie_weights()
													UpperCAmelCase_=       infer_auto_device_map(__UpperCAmelCase					)
													UpperCAmelCase_=       1

							UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
							    """EleutherAI/gpt-neo-125m"""						,  load_in_abit=__UpperCAmelCase						,  device_map=__UpperCAmelCase						,  )
							UpperCAmelCase_=       Accelerator()

							# This should not work and get value error
							with self.assertRaises(__UpperCAmelCase					):
													UpperCAmelCase_=       accelerator.prepare(__UpperCAmelCase					)

							PartialState._reset_state()




	@slow
	@require_bnb
	@require_multi_gpu
	def   _SCREAMING_SNAKE_CASE		(     self     :						Any					)      ->			Dict:
							from transformers import AutoModelForCausalLM

							with init_empty_weights():
													UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
													    """EleutherAI/gpt-neo-125m"""						,  )
													UpperCAmelCase_=       infer_auto_device_map(__UpperCAmelCase					)
													UpperCAmelCase_=       1

							UpperCAmelCase_=       AutoModelForCausalLM.from_pretrained(
							    """EleutherAI/gpt-neo-125m"""						,  load_in_abit=__UpperCAmelCase						,  device_map=__UpperCAmelCase						,  )
							UpperCAmelCase_=       Accelerator()

							# This should work
							UpperCAmelCase_=       accelerator.prepare(__UpperCAmelCase					)




	@require_cuda
	def   _SCREAMING_SNAKE_CASE		(     self     :						Optional[Any]					)      ->			Optional[int]:
							UpperCAmelCase_=       torch.nn.Linear(10						,  10					)
							UpperCAmelCase_=       torch.optim.SGD(model.parameters()						,  lr=0.01					)
							UpperCAmelCase_=       Accelerator(cpu=__UpperCAmelCase					)
							UpperCAmelCase_=       accelerator.prepare(__UpperCAmelCase					)




