




import gc
import random
import unittest

import torch

from diffusers import (
    IFImgaImgPipeline,
    IFImgaImgSuperResolutionPipeline,
    IFInpaintingPipeline,
    IFInpaintingSuperResolutionPipeline,
    IFPipeline,
    IFSuperResolutionPipeline,
)
from diffusers.models.attention_processor import AttnAddedKVProcessor
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import floats_tensor, load_numpy, require_torch_gpu, skip_mps, slow, torch_device

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference
from . import IFPipelineTesterMixin





@skip_mps
class 						A__							(  __a						,	__a						,	unittest.TestCase					):
	lowerCamelCase__						:					List[str]							       =IFPipeline
	lowerCamelCase__						:					List[str]							       =TEXT_TO_IMAGE_PARAMS - {'width', 'height', 'latents'}
	lowerCamelCase__						:					Union[str, Any]							       =TEXT_TO_IMAGE_BATCH_PARAMS
	lowerCamelCase__						:					Tuple							       =PipelineTesterMixin.required_optional_params - {'latents'}






	def 					lowercase (      self		) ->  Union[str, Any]:




						"""simple docstring"""






						return self._get_dummy_components()






	def 					lowercase (      self							,	lowerCamelCase							,	lowerCamelCase=0		) ->  int:




						"""simple docstring"""






						if str(_lowerCamelCase		).startswith('''mps'''		):
											__magic_name__   :				Optional[Any]						  =				torch.manual_seed(_lowerCamelCase		)
						else:
											__magic_name__   :				Any						  =				torch.Generator(device=_lowerCamelCase		).manual_seed(_lowerCamelCase		)

						__magic_name__   :				Optional[int]						  =				{
						    '''prompt''': '''A painting of a squirrel eating a burger''',
						    '''generator''': generator,
						    '''num_inference_steps''': 2,
						    '''output_type''': '''numpy''',
						}

						return inputs






	def 					lowercase (      self		) ->  Optional[int]:




						"""simple docstring"""






						self._test_save_load_optional_components()






	@unittest.skipIf(torch_device != '''cuda'''							,	reason='''float16 requires CUDA'''		)
	def 					lowercase (      self		) ->  List[Any]:




						"""simple docstring"""






						super().test_save_load_floataa(expected_max_diff=1e-1		)






	def 					lowercase (      self		) ->  int:




						"""simple docstring"""






						self._test_attention_slicing_forward_pass(expected_max_diff=1e-2		)






	def 					lowercase (      self		) ->  Any:




						"""simple docstring"""






						self._test_save_load_local()






	def 					lowercase (      self		) ->  Optional[int]:




						"""simple docstring"""






						self._test_inference_batch_single_identical(
						    expected_max_diff=1e-2							,	)






	@unittest.skipIf(
	    torch_device != '''cuda''' or not is_xformers_available()							,	reason='''XFormers attention is only available with CUDA and `xformers` installed'''							,	)
	def 					lowercase (      self		) ->  List[str]:




						"""simple docstring"""






						self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1e-3		)





@slow
@require_torch_gpu
class 						A__							(  unittest.TestCase					):






	def 					lowercase (      self		) ->  Tuple:




						"""simple docstring"""






						super().tearDown()
						gc.collect()
						torch.cuda.empty_cache()






	def 					lowercase (      self		) ->  Dict:




						"""simple docstring"""






						__magic_name__   :				List[Any]						  =				IFPipeline.from_pretrained('''DeepFloyd/IF-I-XL-v1.0'''							,	variant='''fp16'''							,	torch_dtype=torch.floataa		)

						__magic_name__   :				Optional[Any]						  =				IFSuperResolutionPipeline.from_pretrained(
						    '''DeepFloyd/IF-II-L-v1.0'''							,	variant='''fp16'''							,	torch_dtype=torch.floataa							,	text_encoder=_lowerCamelCase							,	tokenizer=_lowerCamelCase		)

						# pre compute text embeddings and remove T5 to save memory

						pipe_a.text_encoder.to('''cuda'''		)

						__magic_name__  ,					__magic_name__   :				str						  =				pipe_a.encode_prompt('''anime turtle'''							,	device='''cuda'''		)

						del pipe_a.tokenizer
						del pipe_a.text_encoder
						gc.collect()

						__magic_name__   :				str						  =				None
						__magic_name__   :				Union[str, Any]						  =				None

						pipe_a.enable_model_cpu_offload()
						pipe_a.enable_model_cpu_offload()

						pipe_a.unet.set_attn_processor(AttnAddedKVProcessor()		)
						pipe_a.unet.set_attn_processor(AttnAddedKVProcessor()		)

						self._test_if(_lowerCamelCase							,	_lowerCamelCase							,	_lowerCamelCase							,	_lowerCamelCase		)

						pipe_a.remove_all_hooks()
						pipe_a.remove_all_hooks()

						# img2img

						__magic_name__   :				List[str]						  =				IFImgaImgPipeline(**pipe_a.components		)
						__magic_name__   :				Tuple						  =				IFImgaImgSuperResolutionPipeline(**pipe_a.components		)

						pipe_a.enable_model_cpu_offload()
						pipe_a.enable_model_cpu_offload()

						pipe_a.unet.set_attn_processor(AttnAddedKVProcessor()		)
						pipe_a.unet.set_attn_processor(AttnAddedKVProcessor()		)

						self._test_if_imgaimg(_lowerCamelCase							,	_lowerCamelCase							,	_lowerCamelCase							,	_lowerCamelCase		)

						pipe_a.remove_all_hooks()
						pipe_a.remove_all_hooks()

						# inpainting

						__magic_name__   :				int						  =				IFInpaintingPipeline(**pipe_a.components		)
						__magic_name__   :				Optional[int]						  =				IFInpaintingSuperResolutionPipeline(**pipe_a.components		)

						pipe_a.enable_model_cpu_offload()
						pipe_a.enable_model_cpu_offload()

						pipe_a.unet.set_attn_processor(AttnAddedKVProcessor()		)
						pipe_a.unet.set_attn_processor(AttnAddedKVProcessor()		)

						self._test_if_inpainting(_lowerCamelCase							,	_lowerCamelCase							,	_lowerCamelCase							,	_lowerCamelCase		)






	def 					lowercase (      self							,	lowerCamelCase							,	lowerCamelCase							,	lowerCamelCase							,	lowerCamelCase		) ->  Union[str, Any]:




						"""simple docstring"""






						_start_torch_memory_measurement()

						__magic_name__   :				Any						  =				torch.Generator(device='''cpu'''		).manual_seed(0		)
						__magic_name__   :				Optional[int]						  =				pipe_a(
						    prompt_embeds=_lowerCamelCase							,	negative_prompt_embeds=_lowerCamelCase							,	num_inference_steps=2							,	generator=_lowerCamelCase							,	output_type='''np'''							,	)

						__magic_name__   :				List[Any]						  =				output.images[0]

						assert image.shape == (64, 64, 3)

						__magic_name__   :				List[str]						  =				torch.cuda.max_memory_allocated()
						assert mem_bytes < 13 * 10**9

						__magic_name__   :				int						  =				load_numpy(
						    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/if/test_if.npy'''		)
						assert_mean_pixel_difference(_lowerCamelCase							,	_lowerCamelCase		)

						# pipeline 2

						_start_torch_memory_measurement()

						__magic_name__   :				Any						  =				torch.Generator(device='''cpu'''		).manual_seed(0		)

						__magic_name__   :				Optional[int]						  =				floats_tensor((1, 3, 64, 64)							,	rng=random.Random(0		)		).to(_lowerCamelCase		)

						__magic_name__   :				Optional[Any]						  =				pipe_a(
						    prompt_embeds=_lowerCamelCase							,	negative_prompt_embeds=_lowerCamelCase							,	image=_lowerCamelCase							,	generator=_lowerCamelCase							,	num_inference_steps=2							,	output_type='''np'''							,	)

						__magic_name__   :				Any						  =				output.images[0]

						assert image.shape == (256, 256, 3)

						__magic_name__   :				Any						  =				torch.cuda.max_memory_allocated()
						assert mem_bytes < 4 * 10**9

						__magic_name__   :				List[Any]						  =				load_numpy(
						    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/if/test_if_superresolution_stage_II.npy'''		)
						assert_mean_pixel_difference(_lowerCamelCase							,	_lowerCamelCase		)






	def 					lowercase (      self							,	lowerCamelCase							,	lowerCamelCase							,	lowerCamelCase							,	lowerCamelCase		) ->  str:




						"""simple docstring"""






						_start_torch_memory_measurement()

						__magic_name__   :				Union[str, Any]						  =				floats_tensor((1, 3, 64, 64)							,	rng=random.Random(0		)		).to(_lowerCamelCase		)

						__magic_name__   :				str						  =				torch.Generator(device='''cpu'''		).manual_seed(0		)

						__magic_name__   :				str						  =				pipe_a(
						    prompt_embeds=_lowerCamelCase							,	negative_prompt_embeds=_lowerCamelCase							,	image=_lowerCamelCase							,	num_inference_steps=2							,	generator=_lowerCamelCase							,	output_type='''np'''							,	)

						__magic_name__   :				str						  =				output.images[0]

						assert image.shape == (64, 64, 3)

						__magic_name__   :				List[str]						  =				torch.cuda.max_memory_allocated()
						assert mem_bytes < 10 * 10**9

						__magic_name__   :				Any						  =				load_numpy(
						    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/if/test_if_img2img.npy'''		)
						assert_mean_pixel_difference(_lowerCamelCase							,	_lowerCamelCase		)

						# pipeline 2

						_start_torch_memory_measurement()

						__magic_name__   :				Tuple						  =				torch.Generator(device='''cpu'''		).manual_seed(0		)

						__magic_name__   :				str						  =				floats_tensor((1, 3, 256, 256)							,	rng=random.Random(0		)		).to(_lowerCamelCase		)
						__magic_name__   :				Dict						  =				floats_tensor((1, 3, 64, 64)							,	rng=random.Random(0		)		).to(_lowerCamelCase		)

						__magic_name__   :				str						  =				pipe_a(
						    prompt_embeds=_lowerCamelCase							,	negative_prompt_embeds=_lowerCamelCase							,	image=_lowerCamelCase							,	original_image=_lowerCamelCase							,	generator=_lowerCamelCase							,	num_inference_steps=2							,	output_type='''np'''							,	)

						__magic_name__   :				Dict						  =				output.images[0]

						assert image.shape == (256, 256, 3)

						__magic_name__   :				List[Any]						  =				torch.cuda.max_memory_allocated()
						assert mem_bytes < 4 * 10**9

						__magic_name__   :				Optional[int]						  =				load_numpy(
						    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/if/test_if_img2img_superresolution_stage_II.npy'''		)
						assert_mean_pixel_difference(_lowerCamelCase							,	_lowerCamelCase		)






	def 					lowercase (      self							,	lowerCamelCase							,	lowerCamelCase							,	lowerCamelCase							,	lowerCamelCase		) ->  Tuple:




						"""simple docstring"""






						_start_torch_memory_measurement()

						__magic_name__   :				Tuple						  =				floats_tensor((1, 3, 64, 64)							,	rng=random.Random(0		)		).to(_lowerCamelCase		)
						__magic_name__   :				List[str]						  =				floats_tensor((1, 3, 64, 64)							,	rng=random.Random(1		)		).to(_lowerCamelCase		)

						__magic_name__   :				Dict						  =				torch.Generator(device='''cpu'''		).manual_seed(0		)
						__magic_name__   :				Dict						  =				pipe_a(
						    prompt_embeds=_lowerCamelCase							,	negative_prompt_embeds=_lowerCamelCase							,	image=_lowerCamelCase							,	mask_image=_lowerCamelCase							,	num_inference_steps=2							,	generator=_lowerCamelCase							,	output_type='''np'''							,	)

						__magic_name__   :				str						  =				output.images[0]

						assert image.shape == (64, 64, 3)

						__magic_name__   :				List[Any]						  =				torch.cuda.max_memory_allocated()
						assert mem_bytes < 10 * 10**9

						__magic_name__   :				Union[str, Any]						  =				load_numpy(
						    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/if/test_if_inpainting.npy'''		)
						assert_mean_pixel_difference(_lowerCamelCase							,	_lowerCamelCase		)

						# pipeline 2

						_start_torch_memory_measurement()

						__magic_name__   :				List[str]						  =				torch.Generator(device='''cpu'''		).manual_seed(0		)

						__magic_name__   :				List[Any]						  =				floats_tensor((1, 3, 64, 64)							,	rng=random.Random(0		)		).to(_lowerCamelCase		)
						__magic_name__   :				Optional[Any]						  =				floats_tensor((1, 3, 256, 256)							,	rng=random.Random(0		)		).to(_lowerCamelCase		)
						__magic_name__   :				Optional[int]						  =				floats_tensor((1, 3, 256, 256)							,	rng=random.Random(1		)		).to(_lowerCamelCase		)

						__magic_name__   :				Dict						  =				pipe_a(
						    prompt_embeds=_lowerCamelCase							,	negative_prompt_embeds=_lowerCamelCase							,	image=_lowerCamelCase							,	mask_image=_lowerCamelCase							,	original_image=_lowerCamelCase							,	generator=_lowerCamelCase							,	num_inference_steps=2							,	output_type='''np'''							,	)

						__magic_name__   :				Optional[Any]						  =				output.images[0]

						assert image.shape == (256, 256, 3)

						__magic_name__   :				Optional[Any]						  =				torch.cuda.max_memory_allocated()
						assert mem_bytes < 4 * 10**9

						__magic_name__   :				List[str]						  =				load_numpy(
						    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/if/test_if_inpainting_superresolution_stage_II.npy'''		)
						assert_mean_pixel_difference(_lowerCamelCase							,	_lowerCamelCase		)







def 					lowerCAmelCase    (      )    ->List[Any]:






					"""simple docstring"""






					torch.cuda.empty_cache()
					torch.cuda.reset_max_memory_allocated()
					torch.cuda.reset_peak_memory_stats()




'''simple docstring'''

import json
import os
import tempfile
from unittest.mock import patch

import torch
from torch.utils.data import DataLoader, TensorDataset

from accelerate import DistributedType, infer_auto_device_map, init_empty_weights
from accelerate.accelerator import Accelerator
from accelerate.state import GradientState, PartialState
from accelerate.test_utils import require_bnb, require_multi_gpu, slow
from accelerate.test_utils.testing import AccelerateTestCase, require_cuda
from accelerate.utils import patch_environment



def snake_case_      (					)			->					str:
   lowerCAmelCase_			=      torch.nn.Linear(2	,   4)
   lowerCAmelCase_			=      torch.optim.AdamW(model.parameters()	,   lr=1.0)
   lowerCAmelCase_			=      torch.optim.lr_scheduler.OneCycleLR(__snake_case	,   max_lr=0.0_1	,   steps_per_epoch=2	,   epochs=1)
   lowerCAmelCase_			=      DataLoader(TensorDataset(torch.tensor([1, 2, 3])))
   lowerCAmelCase_			=      DataLoader(TensorDataset(torch.tensor([4, 5, 6])))

   return model, optimizer, scheduler, train_dl, valid_dl



def snake_case_      (					__snake_case  :							Any)			->					List[Any]:
   return (model.weight.abs().sum() + model.bias.abs().sum()).item()



def snake_case_      (					__snake_case  :							Optional[int])			->					Any:
   lowerCAmelCase_			=      torch.nn.Linear(*tuple(model.weight.T.shape)).state_dict()
   model.load_state_dict(__snake_case)






class     __UpperCAmelCase       ( __a		):





  @require_cuda
  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      Accelerator()
     assert PartialState._shared_state["_cpu"] is False
     assert PartialState._shared_state["device"].type == "cuda"
     with self.assertRaises(_lowerCamelCase ):
        lowerCAmelCase_			=      Accelerator(cpu=_lowerCamelCase )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      Accelerator()
     lowerCAmelCase_			=      GradientState()
     assert state.num_steps == 1
     lowerCAmelCase_			=      4
     assert state.num_steps == 4

     assert state.sync_gradients is True
     lowerCAmelCase_			=      False
     assert state.sync_gradients is False
     GradientState._reset_state()





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      Accelerator()
     lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_			=      create_components()

     (
         (
         lowerCAmelCase_
     )	,(
         lowerCAmelCase_
     )	,(
         lowerCAmelCase_
     )	,(
         lowerCAmelCase_
     )	,(
         lowerCAmelCase_
     )	,
     )			=      accelerator.prepare(_lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase )

     self.assertTrue(prepared_model in accelerator._models )
     self.assertTrue(prepared_optimizer in accelerator._optimizers )
     self.assertTrue(prepared_scheduler in accelerator._schedulers )
     self.assertTrue(prepared_train_dl in accelerator._dataloaders )
     self.assertTrue(prepared_valid_dl in accelerator._dataloaders )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      Accelerator()
     lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_			=      create_components()
     accelerator.prepare(_lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase )
     accelerator.free_memory()

     self.assertTrue(len(accelerator._models ) == 0 )
     self.assertTrue(len(accelerator._optimizers ) == 0 )
     self.assertTrue(len(accelerator._schedulers ) == 0 )
     self.assertTrue(len(accelerator._dataloaders ) == 0 )





  def 						UpperCAmelCase_	(	self ):
     PartialState._reset_state()

     # Mock torch.cuda.set_device to avoid an exception as the device doesn't exist
     def noop(*_lowerCamelCase    ,     **_lowerCamelCase ):
        pass

     with patch('''torch.cuda.set_device'''    ,     _lowerCamelCase ), patch_environment(ACCELERATE_TORCH_DEVICE='''cuda:64''' ):
        lowerCAmelCase_			=      Accelerator()
        self.assertEqual(str(accelerator.state.device )    ,     '''cuda:64''' )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      Accelerator()
     lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_			=      create_components()
     accelerator.prepare(_lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase )

     lowerCAmelCase_			=      get_signature(_lowerCamelCase )

     with tempfile.TemporaryDirectory() as tmpdirname:
        accelerator.save_state(_lowerCamelCase )

        # make sure random weights don't match
        load_random_weights(_lowerCamelCase )
        self.assertTrue(abs(model_signature - get_signature(_lowerCamelCase ) ) > 1E-3 )

        # make sure loaded weights match
        accelerator.load_state(_lowerCamelCase )
        self.assertTrue(abs(model_signature - get_signature(_lowerCamelCase ) ) < 1E-3 )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      Accelerator()
     lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_			=      create_components()
     accelerator.prepare(_lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase )

     lowerCAmelCase_			=      get_signature(_lowerCamelCase )

     # saving hook
     def save_config(_lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase ):
        lowerCAmelCase_			=      {'''class_name''': models[0].__class__.__name__}

        with open(os.path.join(_lowerCamelCase    ,     '''data.json''' )    ,     '''w''' ) as f:
           json.dump(_lowerCamelCase    ,     _lowerCamelCase )

        # loading hook
     def load_config(_lowerCamelCase    ,     _lowerCamelCase ):
        with open(os.path.join(_lowerCamelCase    ,     '''data.json''' )    ,     '''r''' ) as f:
           lowerCAmelCase_			=      json.load(_lowerCamelCase )

        lowerCAmelCase_			=      config['''class_name''']

     lowerCAmelCase_			=      accelerator.register_save_state_pre_hook(_lowerCamelCase )
     lowerCAmelCase_			=      accelerator.register_load_state_pre_hook(_lowerCamelCase )

     with tempfile.TemporaryDirectory() as tmpdirname:
        accelerator.save_state(_lowerCamelCase )

        # make sure random weights don't match with hooks
        load_random_weights(_lowerCamelCase )
        self.assertTrue(abs(model_signature - get_signature(_lowerCamelCase ) ) > 1E-3 )

        # random class name to verify correct one is loaded
        lowerCAmelCase_			=      '''random'''

        # make sure loaded weights match with hooks
        accelerator.load_state(_lowerCamelCase )
        self.assertTrue(abs(model_signature - get_signature(_lowerCamelCase ) ) < 1E-3 )

        # mode.class_name is loaded from config
        self.assertTrue(model.class_name == model.__class__.__name__ )

     # remove hooks
     save_hook.remove()
     load_hook.remove()

     with tempfile.TemporaryDirectory() as tmpdirname:
        accelerator.save_state(_lowerCamelCase )

        # make sure random weights don't match with hooks removed
        load_random_weights(_lowerCamelCase )
        self.assertTrue(abs(model_signature - get_signature(_lowerCamelCase ) ) > 1E-3 )

        # random class name to verify correct one is loaded
        lowerCAmelCase_			=      '''random'''

        # make sure loaded weights match with hooks removed
        accelerator.load_state(_lowerCamelCase )
        self.assertTrue(abs(model_signature - get_signature(_lowerCamelCase ) ) < 1E-3 )

        # mode.class_name is NOT loaded from config
        self.assertTrue(model.class_name != model.__class__.__name__ )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      Accelerator()
     lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_			=      create_components()
     lowerCAmelCase_			=      None

     # This should work
     lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_			=      accelerator.prepare(
         _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase )
     self.assertTrue(dummy_obj is None )





  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      Accelerator()
     lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_			=      create_components()
     lowerCAmelCase_			=      [1, 2, 3]

     # This should work
     lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_	,lowerCAmelCase_			=      accelerator.prepare(
         _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase    ,     _lowerCamelCase )
     self.assertEqual(
         getattr(_lowerCamelCase    ,     '''_is_accelerate_prepared'''    ,     _lowerCamelCase )    ,     _lowerCamelCase    ,     '''Dummy object should have `_is_accelerate_prepared` set to `True`'''    ,     )
     self.assertEqual(
         getattr(_lowerCamelCase    ,     '''_is_accelerate_prepared'''    ,     _lowerCamelCase )    ,     _lowerCamelCase    ,     '''Model is missing `_is_accelerator_prepared` or is set to `False`'''    ,     )
     self.assertEqual(
         getattr(_lowerCamelCase    ,     '''_is_accelerate_prepared'''    ,     _lowerCamelCase )    ,     _lowerCamelCase    ,     '''Optimizer is missing `_is_accelerator_prepared` or is set to `False`'''    ,     )
     self.assertEqual(
         getattr(_lowerCamelCase    ,     '''_is_accelerate_prepared'''    ,     _lowerCamelCase )    ,     _lowerCamelCase    ,     '''Scheduler is missing `_is_accelerator_prepared` or is set to `False`'''    ,     )
     self.assertEqual(
         getattr(_lowerCamelCase    ,     '''_is_accelerate_prepared'''    ,     _lowerCamelCase )    ,     _lowerCamelCase    ,     '''Train Dataloader is missing `_is_accelerator_prepared` or is set to `False`'''    ,     )
     self.assertEqual(
         getattr(_lowerCamelCase    ,     '''_is_accelerate_prepared'''    ,     _lowerCamelCase )    ,     _lowerCamelCase    ,     '''Valid Dataloader is missing `_is_accelerator_prepared` or is set to `False`'''    ,     )





  @slow
  @require_bnb
  def 						UpperCAmelCase_	(	self ):
     from transformers import AutoModelForCausalLM

     lowerCAmelCase_			=      AutoModelForCausalLM.from_pretrained(
         '''EleutherAI/gpt-neo-125m'''    ,     load_in_abit=_lowerCamelCase    ,     device_map={'''''': 0}    ,     )
     lowerCAmelCase_			=      Accelerator()

     # This should work
     lowerCAmelCase_			=      accelerator.prepare(_lowerCamelCase )





  @slow
  @require_bnb
  def 						UpperCAmelCase_	(	self ):
     from transformers import AutoModelForCausalLM

     lowerCAmelCase_			=      Accelerator()

     with init_empty_weights():
        lowerCAmelCase_			=      AutoModelForCausalLM.from_pretrained(
            '''EleutherAI/gpt-neo-125m'''    ,     )
        model.tie_weights()
        lowerCAmelCase_			=      infer_auto_device_map(_lowerCamelCase )
        lowerCAmelCase_			=      '''cpu'''

     lowerCAmelCase_			=      AutoModelForCausalLM.from_pretrained(
         '''EleutherAI/gpt-neo-125m'''    ,     device_map=_lowerCamelCase    ,     load_in_abit=_lowerCamelCase    ,     llm_inta_enable_fpaa_cpu_offload=_lowerCamelCase )

     # This should not work and get value error
     with self.assertRaises(_lowerCamelCase ):
        lowerCAmelCase_			=      accelerator.prepare(_lowerCamelCase )





  @slow
  @require_bnb
  @require_multi_gpu
  def 						UpperCAmelCase_	(	self ):
     from transformers import AutoModelForCausalLM

     lowerCAmelCase_			=      {'''distributed_type''': DistributedType.MULTI_GPU}

     with init_empty_weights():
        lowerCAmelCase_			=      AutoModelForCausalLM.from_pretrained(
            '''EleutherAI/gpt-neo-125m'''    ,     )
        model.tie_weights()
        lowerCAmelCase_			=      infer_auto_device_map(_lowerCamelCase )
        lowerCAmelCase_			=      1

     lowerCAmelCase_			=      AutoModelForCausalLM.from_pretrained(
         '''EleutherAI/gpt-neo-125m'''    ,     load_in_abit=_lowerCamelCase    ,     device_map=_lowerCamelCase    ,     )
     lowerCAmelCase_			=      Accelerator()

     # This should not work and get value error
     with self.assertRaises(_lowerCamelCase ):
        lowerCAmelCase_			=      accelerator.prepare(_lowerCamelCase )

     PartialState._reset_state()





  @slow
  @require_bnb
  @require_multi_gpu
  def 						UpperCAmelCase_	(	self ):
     from transformers import AutoModelForCausalLM

     with init_empty_weights():
        lowerCAmelCase_			=      AutoModelForCausalLM.from_pretrained(
            '''EleutherAI/gpt-neo-125m'''    ,     )
        lowerCAmelCase_			=      infer_auto_device_map(_lowerCamelCase )
        lowerCAmelCase_			=      1

     lowerCAmelCase_			=      AutoModelForCausalLM.from_pretrained(
         '''EleutherAI/gpt-neo-125m'''    ,     load_in_abit=_lowerCamelCase    ,     device_map=_lowerCamelCase    ,     )
     lowerCAmelCase_			=      Accelerator()

     # This should work
     lowerCAmelCase_			=      accelerator.prepare(_lowerCamelCase )







  @require_cuda
  def 						UpperCAmelCase_	(	self ):
     lowerCAmelCase_			=      torch.nn.Linear(10    ,     10 )
     lowerCAmelCase_			=      torch.optim.SGD(model.parameters()    ,     lr=0.01 )
     lowerCAmelCase_			=      Accelerator(cpu=_lowerCamelCase )
     lowerCAmelCase_			=      accelerator.prepare(_lowerCamelCase )





