
"""simple docstring"""

import unittest

import numpy as np
import torch

from diffusers import DDIMPipeline, DDIMScheduler, UNetaDModel
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, slow, torch_device

from ..pipeline_params import UNCONDITIONAL_IMAGE_GENERATION_BATCH_PARAMS, UNCONDITIONAL_IMAGE_GENERATION_PARAMS
from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()
class      _UpperCAmelCase							(		lowercase__     ,			unittest.TestCase							):
      __SCREAMING_SNAKE_CASE   :							str							      =	DDIMPipeline
      __SCREAMING_SNAKE_CASE   :							List[Any]							      =	UNCONDITIONAL_IMAGE_GENERATION_PARAMS
      __SCREAMING_SNAKE_CASE   :							Any							      =	PipelineTesterMixin.required_optional_params - {
          "num_images_per_prompt",
          "latents",
          "callback",
          "callback_steps",
      }
      __SCREAMING_SNAKE_CASE   :							Union[str, Any]							      =	UNCONDITIONAL_IMAGE_GENERATION_BATCH_PARAMS
      __SCREAMING_SNAKE_CASE   :							int							      =	False




      def     a_      (							self							)						->					Dict:
          torch.manual_seed(0							)
          UpperCAmelCase									=					UNetaDModel(
              block_out_channels=(3_2, 6_4)   ,       layers_per_block=2   ,       sample_size=3_2   ,       in_channels=3   ,       out_channels=3   ,       down_block_types=('DownBlock2D', 'AttnDownBlock2D')   ,       up_block_types=('AttnUpBlock2D', 'UpBlock2D')   ,       )
          UpperCAmelCase									=					DDIMScheduler()
          UpperCAmelCase									=					{'''unet''': unet, '''scheduler''': scheduler}
          return components




      def     a_      (							self   ,       lowercase_   ,       lowercase_=0							)						->					Optional[Any]:
          if str(UpperCAmelCase__							).startswith('mps'							):
              UpperCAmelCase									=					torch.manual_seed(UpperCAmelCase__							)
          else:
              UpperCAmelCase									=					torch.Generator(device=UpperCAmelCase__							).manual_seed(UpperCAmelCase__							)
          UpperCAmelCase									=					{
              '''batch_size''': 1,
              '''generator''': generator,
              '''num_inference_steps''': 2,
              '''output_type''': '''numpy''',
          }
          return inputs




      def     a_      (							self							)						->					Any:
          UpperCAmelCase									=					'''cpu'''

          UpperCAmelCase									=					self.get_dummy_components()
          UpperCAmelCase									=					self.pipeline_class(**UpperCAmelCase__							)
          pipe.to(UpperCAmelCase__							)
          pipe.set_progress_bar_config(disable=UpperCAmelCase__							)

          UpperCAmelCase									=					self.get_dummy_inputs(UpperCAmelCase__							)
          UpperCAmelCase									=					pipe(**UpperCAmelCase__							).images
          UpperCAmelCase									=					image[0, -3:, -3:, -1]

          self.assertEqual(image.shape   ,       (1, 3_2, 3_2, 3)							)
          UpperCAmelCase									=					np.array(
              [1.000E00, 5.717E-01, 4.717E-01, 1.000E00, 0.000E00, 1.000E00, 3.000E-04, 0.000E00, 9.000E-04]							)
          UpperCAmelCase									=					np.abs(image_slice.flatten() - expected_slice							).max()
          self.assertLessEqual(UpperCAmelCase__   ,       1E-3							)




      def     a_      (							self							)						->					int:
          super().test_dict_tuple_outputs_equivalent(expected_max_difference=3E-3							)




      def     a_      (							self							)						->					str:
          super().test_save_load_local(expected_max_difference=3E-3							)




      def     a_      (							self							)						->					List[Any]:
          super().test_save_load_optional_components(expected_max_difference=3E-3							)




      def     a_      (							self							)						->					Any:
          super().test_inference_batch_single_identical(expected_max_diff=3E-3							)

@slow
@require_torch_gpu
class      _UpperCAmelCase							(		unittest.TestCase							):




      def     a_      (							self							)						->					Tuple:
          UpperCAmelCase									=					'''google/ddpm-cifar10-32'''

          UpperCAmelCase									=					UNetaDModel.from_pretrained(UpperCAmelCase__							)
          UpperCAmelCase									=					DDIMScheduler()

          UpperCAmelCase									=					DDIMPipeline(unet=UpperCAmelCase__   ,       scheduler=UpperCAmelCase__							)
          ddim.to(UpperCAmelCase__							)
          ddim.set_progress_bar_config(disable=UpperCAmelCase__							)

          UpperCAmelCase									=					torch.manual_seed(0							)
          UpperCAmelCase									=					ddim(generator=UpperCAmelCase__   ,       eta=0.0   ,       output_type='numpy'							).images

          UpperCAmelCase									=					image[0, -3:, -3:, -1]

          assert image.shape == (1, 3_2, 3_2, 3)
          UpperCAmelCase									=					np.array([0.1_7_2_3, 0.1_6_1_7, 0.1_6_0_0, 0.1_6_2_6, 0.1_4_9_7, 0.1_5_1_3, 0.1_5_0_5, 0.1_4_4_2, 0.1_4_5_3]							)

          assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2




      def     a_      (							self							)						->					Tuple:
          UpperCAmelCase									=					'''google/ddpm-ema-bedroom-256'''

          UpperCAmelCase									=					UNetaDModel.from_pretrained(UpperCAmelCase__							)
          UpperCAmelCase									=					DDIMScheduler.from_pretrained(UpperCAmelCase__							)

          UpperCAmelCase									=					DDIMPipeline(unet=UpperCAmelCase__   ,       scheduler=UpperCAmelCase__							)
          ddpm.to(UpperCAmelCase__							)
          ddpm.set_progress_bar_config(disable=UpperCAmelCase__							)

          UpperCAmelCase									=					torch.manual_seed(0							)
          UpperCAmelCase									=					ddpm(generator=UpperCAmelCase__   ,       output_type='numpy'							).images

          UpperCAmelCase									=					image[0, -3:, -3:, -1]

          assert image.shape == (1, 2_5_6, 2_5_6, 3)
          UpperCAmelCase									=					np.array([0.0_0_6_0, 0.0_2_0_1, 0.0_3_4_4, 0.0_0_2_4, 0.0_0_1_8, 0.0_0_0_2, 0.0_0_2_2, 0.0_0_0_0, 0.0_0_6_9]							)

          assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2






"""simple docstring"""

import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu
class      _UpperCAmelCase							(		nn.Module							):




      def __init__(							self							)						->					Any:
          super().__init__()
          UpperCAmelCase									=					nn.Linear(3   ,       4							)
          UpperCAmelCase									=					nn.BatchNormad(4							)
          UpperCAmelCase									=					nn.Linear(4   ,       5							)




      def     a_      (							self   ,       lowercase_							)						->					Any:
          return self.lineara(self.batchnorm(self.lineara(lowercase_							)							)							)

class      _UpperCAmelCase							(		SCREAMING_SNAKE_CASE_							):




      def     a_      (							self   ,       lowercase_   ,       *lowercase_   ,       **lowercase_							)						->					List[str]:
          return (args[0] + 1,) + args[1:], kwargs

class      _UpperCAmelCase							(		SCREAMING_SNAKE_CASE_							):




      def     a_      (							self   ,       lowercase_   ,       lowercase_							)						->					Union[str, Any]:
          return output + 1

class      _UpperCAmelCase							(		unittest.TestCase							):




      def     a_      (							self							)						->					Optional[int]:
          UpperCAmelCase									=					ModelForTest()
          UpperCAmelCase									=					ModelHook()

          add_hook_to_module(lowercase_   ,       lowercase_							)
          self.assertEqual(test_model._hf_hook   ,       lowercase_							)
          self.assertTrue(hasattr(lowercase_   ,       '_old_forward'							)							)

          # Check adding the hook did not change the name or the signature
          self.assertEqual(test_model.forward.__name__   ,       'forward'							)
          self.assertListEqual(list(inspect.signature(test_model.forward							).parameters							)   ,       ['x']							)

          remove_hook_from_module(lowercase_							)
          self.assertFalse(hasattr(lowercase_   ,       '_hf_hook'							)							)
          self.assertFalse(hasattr(lowercase_   ,       '_old_forward'							)							)




      def     a_      (							self							)						->					Any:
          UpperCAmelCase									=					ModelForTest()
          UpperCAmelCase									=					ModelHook()

          add_hook_to_module(lowercase_   ,       lowercase_							)
          add_hook_to_module(lowercase_   ,       lowercase_   ,       append=lowercase_							)

          self.assertEqual(isinstance(test_model._hf_hook   ,       lowercase_							)   ,       lowercase_							)
          self.assertEqual(len(test_model._hf_hook.hooks							)   ,       2							)
          self.assertTrue(hasattr(lowercase_   ,       '_old_forward'							)							)

          # Check adding the hook did not change the name or the signature
          self.assertEqual(test_model.forward.__name__   ,       'forward'							)
          self.assertListEqual(list(inspect.signature(test_model.forward							).parameters							)   ,       ['x']							)

          remove_hook_from_module(lowercase_							)
          self.assertFalse(hasattr(lowercase_   ,       '_hf_hook'							)							)
          self.assertFalse(hasattr(lowercase_   ,       '_old_forward'							)							)




      def     a_      (							self							)						->					Any:
          UpperCAmelCase									=					ModelForTest()
          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					test_model(x + 1							)
          UpperCAmelCase									=					test_model(x + 2							)

          UpperCAmelCase									=					PreForwardHook()
          add_hook_to_module(lowercase_   ,       lowercase_							)
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertTrue(torch.allclose(lowercase_   ,       lowercase_   ,       atol=1E-5							)							)

          # Attaching a hook to a model when it already has one replaces, does not chain
          UpperCAmelCase									=					PreForwardHook()
          add_hook_to_module(lowercase_   ,       lowercase_							)
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertTrue(torch.allclose(lowercase_   ,       lowercase_   ,       atol=1E-5							)							)

          # You need to use the sequential hook to chain two or more hooks
          UpperCAmelCase									=					SequentialHook(PreForwardHook()   ,       PreForwardHook()							)
          add_hook_to_module(lowercase_   ,       lowercase_							)

          UpperCAmelCase									=					test_model(lowercase_							)
          assert torch.allclose(lowercase_   ,       lowercase_   ,       atol=1E-5							)




      def     a_      (							self							)						->					List[str]:
          UpperCAmelCase									=					ModelForTest()
          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					test_model(lowercase_							)

          UpperCAmelCase									=					PostForwardHook()
          add_hook_to_module(lowercase_   ,       lowercase_							)
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertTrue(torch.allclose(lowercase_   ,       output + 1   ,       atol=1E-5							)							)

          # Attaching a hook to a model when it already has one replaces, does not chain
          UpperCAmelCase									=					PostForwardHook()
          add_hook_to_module(lowercase_   ,       lowercase_							)
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertTrue(torch.allclose(lowercase_   ,       output + 1   ,       atol=1E-5							)							)

          # You need to use the sequential hook to chain two or more hooks
          UpperCAmelCase									=					SequentialHook(PostForwardHook()   ,       PostForwardHook()							)
          add_hook_to_module(lowercase_   ,       lowercase_							)

          UpperCAmelCase									=					test_model(lowercase_							)
          assert torch.allclose(lowercase_   ,       output + 2   ,       atol=1E-5							)




      def     a_      (							self							)						->					Tuple:
          UpperCAmelCase									=					ModelForTest()
          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					test_model(lowercase_							)

          UpperCAmelCase									=					PostForwardHook()
          add_hook_to_module(lowercase_   ,       lowercase_							)
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertTrue(torch.allclose(lowercase_   ,       output + 1							)							)
          self.assertTrue(outputa.requires_grad							)

          UpperCAmelCase									=					True
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertFalse(outputa.requires_grad							)




      @require_multi_gpu
      def     a_      (							self							)						->					Optional[int]:
          UpperCAmelCase									=					ModelForTest()
          # Everything is on CPU
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # This will move each submodule on different devices
          add_hook_to_module(model.lineara   ,       AlignDevicesHook(execution_device=0							)							)
          add_hook_to_module(model.batchnorm   ,       AlignDevicesHook(execution_device=0							)							)
          add_hook_to_module(model.lineara   ,       AlignDevicesHook(execution_device=1							)							)

          self.assertEqual(model.lineara.weight.device   ,       torch.device(0							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device(0							)							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       torch.device(0							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device(1							)							)

          # We can still make a forward pass. The input does not need to be on any particular device
          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       torch.device(1							)							)

          # We can add a general hook to put back output on same device as input.
          add_hook_to_module(lowercase_   ,       AlignDevicesHook(io_same_device=lowercase_							)							)
          UpperCAmelCase									=					torch.randn(2   ,       3							).to(0							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       torch.device(0							)							)




      def     a_      (							self							)						->					Dict:
          UpperCAmelCase									=					ModelForTest()

          # Everything is on CPU
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # This will move each submodule on different devices
          UpperCAmelCase									=					{'execution_device': 0 if torch.cuda.is_available() else 'cpu', 'offload': True}

          add_hook_to_module(model.lineara   ,       AlignDevicesHook(**lowercase_							)							)
          add_hook_to_module(model.batchnorm   ,       AlignDevicesHook(**lowercase_							)							)
          add_hook_to_module(model.lineara   ,       AlignDevicesHook(**lowercase_							)							)

          # Parameters have been offloaded, so on the meta device
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          # Buffers are not included in the offload by default, so are on the execution device
          UpperCAmelCase									=					torch.device(hook_kwargs['execution_device']							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       lowercase_							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_module(model.lineara							)
          remove_hook_from_module(model.batchnorm							)
          remove_hook_from_module(model.lineara							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # Now test with buffers included in the offload
          UpperCAmelCase									=					{
              'execution_device': 0 if torch.cuda.is_available() else 'cpu',
              'offload': True,
              'offload_buffers': True,
          }

          add_hook_to_module(model.lineara   ,       AlignDevicesHook(**lowercase_							)							)
          add_hook_to_module(model.batchnorm   ,       AlignDevicesHook(**lowercase_							)							)
          add_hook_to_module(model.lineara   ,       AlignDevicesHook(**lowercase_							)							)

          # Parameters have been offloaded, so on the meta device, buffers included
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       torch.device('meta'							)							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_module(model.lineara							)
          remove_hook_from_module(model.batchnorm							)
          remove_hook_from_module(model.lineara							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)




      def     a_      (							self							)						->					Optional[int]:
          UpperCAmelCase									=					ModelForTest()

          # Everything is on CPU
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # This will move each submodule on different devices
          UpperCAmelCase									=					0 if torch.cuda.is_available() else 'cpu'
          attach_align_device_hook(lowercase_   ,       execution_device=lowercase_   ,       offload=lowercase_							)

          # Parameters have been offloaded, so on the meta device
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          # Buffers are not included in the offload by default, so are on the execution device
          UpperCAmelCase									=					torch.device(lowercase_							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       lowercase_							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_submodules(lowercase_							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # Now test with buffers included in the offload
          attach_align_device_hook(lowercase_   ,       execution_device=lowercase_   ,       offload=lowercase_   ,       offload_buffers=lowercase_							)

          # Parameters have been offloaded, so on the meta device, buffers included
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       torch.device('meta'							)							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_submodules(lowercase_							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)




      def     a_      (							self							)						->					Dict:
          UpperCAmelCase									=					ModelForTest()

          # Everything is on CPU
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # This will move each submodule on different devices
          UpperCAmelCase									=					0 if torch.cuda.is_available() else 'cpu'
          attach_align_device_hook(
              lowercase_   ,       execution_device=lowercase_   ,       offload=lowercase_   ,       weights_map=model.state_dict()							)

          # Parameters have been offloaded, so on the meta device
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          # Buffers are not included in the offload by default, so are on the execution device
          UpperCAmelCase									=					torch.device(lowercase_							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       lowercase_							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_submodules(lowercase_							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # Now test with buffers included in the offload
          attach_align_device_hook(
              lowercase_   ,       execution_device=lowercase_   ,       offload=lowercase_   ,       weights_map=model.state_dict()   ,       offload_buffers=lowercase_   ,       )

          # Parameters have been offloaded, so on the meta device, buffers included
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       torch.device('meta'							)							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_submodules(lowercase_							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)



