
"""simple docstring"""

import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(42)

SCREAMING_SNAKE_CASE_    					=						'''bert-base-cased'''
SCREAMING_SNAKE_CASE_    					=						'''fp16'''
SCREAMING_SNAKE_CASE_    					=						'''bf16'''
SCREAMING_SNAKE_CASE_    					=						[FPaa, BFaa]
@require_fsdp
@require_cuda
class      _UpperCAmelCase							(		SCREAMING_SNAKE_CASE_							):




      def     a_      (							self							)						->					Optional[Any]:
          super().setUp()

          UpperCAmelCase									=					dict(
              ACCELERATE_USE_FSDP='true'   ,       MASTER_ADDR='localhost'   ,       MASTER_PORT='10999'   ,       RANK='0'   ,       LOCAL_RANK='0'   ,       WORLD_SIZE='1'   ,       )




      def     a_      (							self							)						->					str:
          from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

          for i, strategy in enumerate(lowercase_							):
              UpperCAmelCase									=					self.dist_env.copy()
              UpperCAmelCase									=					F"{i + 1}"
              UpperCAmelCase									=					strategy
              with mockenv_context(**lowercase_							):
                  UpperCAmelCase									=					FullyShardedDataParallelPlugin()
                  self.assertEqual(fsdp_plugin.sharding_strategy   ,       ShardingStrategy(i + 1							)							)




      def     a_      (							self							)						->					List[str]:
          from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

          for i, prefetch_policy in enumerate(lowercase_							):
              UpperCAmelCase									=					self.dist_env.copy()
              UpperCAmelCase									=					prefetch_policy
              with mockenv_context(**lowercase_							):
                  UpperCAmelCase									=					FullyShardedDataParallelPlugin()
                  if prefetch_policy == "NO_PREFETCH":
                      self.assertIsNone(fsdp_plugin.backward_prefetch							)
                  else:
                      self.assertEqual(fsdp_plugin.backward_prefetch   ,       BackwardPrefetch(i + 1							)							)




      def     a_      (							self							)						->					int:
          from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

          for i, state_dict_type in enumerate(lowercase_							):
              UpperCAmelCase									=					self.dist_env.copy()
              UpperCAmelCase									=					state_dict_type
              with mockenv_context(**lowercase_							):
                  UpperCAmelCase									=					FullyShardedDataParallelPlugin()
                  self.assertEqual(fsdp_plugin.state_dict_type   ,       StateDictType(i + 1							)							)
                  if state_dict_type == "FULL_STATE_DICT":
                      self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu							)
                      self.assertTrue(fsdp_plugin.state_dict_config.ranka_only							)




      def     a_      (							self							)						->					Optional[int]:
          UpperCAmelCase									=					AutoModel.from_pretrained(lowercase_							)
          for policy in FSDP_AUTO_WRAP_POLICY:
              UpperCAmelCase									=					self.dist_env.copy()
              UpperCAmelCase									=					policy
              if policy == "TRANSFORMER_BASED_WRAP":
                  UpperCAmelCase									=					'BertLayer'
              elif policy == "SIZE_BASED_WRAP":
                  UpperCAmelCase									=					'2000'
              with mockenv_context(**lowercase_							):
                  UpperCAmelCase									=					FullyShardedDataParallelPlugin()
                  fsdp_plugin.set_auto_wrap_policy(lowercase_							)
                  if policy == "NO_WRAP":
                      self.assertIsNone(fsdp_plugin.auto_wrap_policy							)
                  else:
                      self.assertIsNotNone(fsdp_plugin.auto_wrap_policy							)

          UpperCAmelCase									=					self.dist_env.copy()
          UpperCAmelCase									=					'TRANSFORMER_BASED_WRAP'
          UpperCAmelCase									=					'T5Layer'
          with mockenv_context(**lowercase_							):
              UpperCAmelCase									=					FullyShardedDataParallelPlugin()
              with self.assertRaises(lowercase_							) as cm:
                  fsdp_plugin.set_auto_wrap_policy(lowercase_							)
              self.assertTrue('Could not find the transformer layer class to wrap in the model.' in str(cm.exception							)							)

          UpperCAmelCase									=					self.dist_env.copy()
          UpperCAmelCase									=					'SIZE_BASED_WRAP'
          UpperCAmelCase									=					'0'
          with mockenv_context(**lowercase_							):
              UpperCAmelCase									=					FullyShardedDataParallelPlugin()
              fsdp_plugin.set_auto_wrap_policy(lowercase_							)
              self.assertIsNone(fsdp_plugin.auto_wrap_policy							)




      def     a_      (							self							)						->					List[str]:
          from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
          from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

          for mp_dtype in dtypes:
              UpperCAmelCase									=					self.dist_env.copy()
              UpperCAmelCase									=					mp_dtype
              with mockenv_context(**lowercase_							):
                  UpperCAmelCase									=					Accelerator()
                  if mp_dtype == "fp16":
                      UpperCAmelCase									=					torch.floataa
                  elif mp_dtype == "bf16":
                      UpperCAmelCase									=					torch.bfloataa
                  UpperCAmelCase									=					MixedPrecision(param_dtype=lowercase_   ,       reduce_dtype=lowercase_   ,       buffer_dtype=lowercase_							)
                  self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy   ,       lowercase_							)
                  if mp_dtype == FPaa:
                      self.assertTrue(isinstance(accelerator.scaler   ,       lowercase_							)							)
                  elif mp_dtype == BFaa:
                      self.assertIsNone(accelerator.scaler							)
                  AcceleratorState._reset_state(lowercase_							)




      def     a_      (							self							)						->					int:
          from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

          for flag in [True, False]:
              UpperCAmelCase									=					self.dist_env.copy()
              UpperCAmelCase									=					str(lowercase_							).lower()
              with mockenv_context(**lowercase_							):
                  UpperCAmelCase									=					FullyShardedDataParallelPlugin()
                  self.assertEqual(fsdp_plugin.cpu_offload   ,       CPUOffload(offload_params=lowercase_							)							)

@require_fsdp
@require_multi_gpu
@slow
class      _UpperCAmelCase							(		SCREAMING_SNAKE_CASE_							):




      def     a_      (							self							)						->					int:
          super().setUp()
          UpperCAmelCase									=					0.8_2
          UpperCAmelCase									=					[
              'fsdp_shard_grad_op_transformer_based_wrap',
              'fsdp_full_shard_transformer_based_wrap',
          ]
          UpperCAmelCase									=					{
              'multi_gpu_fp16': 3_2_0_0,
              'fsdp_shard_grad_op_transformer_based_wrap_fp16': 2_0_0_0,
              'fsdp_full_shard_transformer_based_wrap_fp16': 1_9_0_0,
              # Disabling below test as it overwhelms the RAM memory usage
              # on CI self-hosted runner leading to tests getting killed.
              # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
          }
          UpperCAmelCase									=					1_6_0
          UpperCAmelCase									=					1_6_0

          UpperCAmelCase									=					inspect.getfile(accelerate.test_utils							)
          UpperCAmelCase									=					os.path.sep.join(mod_file.split(os.path.sep							)[:-1] + ['scripts', 'external_deps']							)




      def     a_      (							self							)						->					List[Any]:
          UpperCAmelCase									=					os.path.join(self.test_scripts_folder   ,       'test_performance.py'							)
          UpperCAmelCase									=					['accelerate', 'launch', '--num_processes=2', '--num_machines=1', '--machine_rank=0', '--use_fsdp']
          for config in self.performance_configs:
              UpperCAmelCase									=					cmd.copy()
              for i, strategy in enumerate(lowercase_							):
                  if strategy.lower() in config:
                      cmd_config.append(F"--fsdp_sharding_strategy={i+1}"							)
                      break

              if "fp32" in config:
                  cmd_config.append('--mixed_precision=no'							)
              else:
                  cmd_config.append('--mixed_precision=fp16'							)

              if "cpu_offload" in config:
                  cmd_config.append('--fsdp_offload_params=True'							)

              for policy in FSDP_AUTO_WRAP_POLICY:
                  if policy.lower() in config:
                      cmd_config.append(F"--fsdp_auto_wrap_policy={policy}"							)
                      break

              if policy == "TRANSFORMER_BASED_WRAP":
                  cmd_config.append('--fsdp_transformer_layer_cls_to_wrap=BertLayer'							)
              elif policy == "SIZE_BASED_WRAP":
                  cmd_config.append('--fsdp_min_num_params=2000'							)

              cmd_config.extend(
                  [
                      self.test_file_path,
                      F"--output_dir={self.tmpdir}",
                      F"--performance_lower_bound={self.performance_lower_bound}",
                  ]							)
              with patch_environment(omp_num_threads=1							):
                  execute_subprocess_async(lowercase_   ,       env=os.environ.copy()							)




      def     a_      (							self							)						->					str:
          UpperCAmelCase									=					os.path.join(self.test_scripts_folder   ,       'test_checkpointing.py'							)
          UpperCAmelCase									=					[
              'accelerate',
              'launch',
              '--num_processes=2',
              '--num_machines=1',
              '--machine_rank=0',
              '--use_fsdp',
              '--mixed_precision=fp16',
              '--fsdp_transformer_layer_cls_to_wrap=BertLayer',
          ]

          for i, strategy in enumerate(lowercase_							):
              UpperCAmelCase									=					cmd.copy()
              cmd_config.append(F"--fsdp_sharding_strategy={i+1}"							)
              if strategy != "FULL_SHARD":
                  continue
              UpperCAmelCase									=					len(lowercase_							)
              for state_dict_type in FSDP_STATE_DICT_TYPE:
                  UpperCAmelCase									=					cmd_config[:state_dict_config_index]
                  cmd_config.append(F"--fsdp_state_dict_type={state_dict_type}"							)
                  cmd_config.extend(
                      [
                          self.test_file_path,
                          F"--output_dir={self.tmpdir}",
                          '--partial_train_epoch=1',
                      ]							)
                  with patch_environment(omp_num_threads=1							):
                      execute_subprocess_async(lowercase_   ,       env=os.environ.copy()							)

                  UpperCAmelCase									=					cmd_config[:-1]
                  UpperCAmelCase									=					os.path.join(self.tmpdir   ,       'epoch_0'							)
                  cmd_config.extend(
                      [
                          F"--resume_from_checkpoint={resume_from_checkpoint}",
                      ]							)
                  with patch_environment(omp_num_threads=1							):
                      execute_subprocess_async(lowercase_   ,       env=os.environ.copy()							)




      def     a_      (							self							)						->					Any:
          UpperCAmelCase									=					os.path.join(self.test_scripts_folder   ,       'test_peak_memory_usage.py'							)
          UpperCAmelCase									=					[
              'accelerate',
              'launch',
              '--num_processes=2',
              '--num_machines=1',
              '--machine_rank=0',
          ]
          for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
              UpperCAmelCase									=					cmd.copy()
              if "fp16" in spec:
                  cmd_config.extend(['--mixed_precision=fp16']							)
              else:
                  cmd_config.extend(['--mixed_precision=no']							)

              if "multi_gpu" in spec:
                  continue
              else:
                  cmd_config.extend(['--use_fsdp']							)
                  for i, strategy in enumerate(lowercase_							):
                      if strategy.lower() in spec:
                          cmd_config.append(F"--fsdp_sharding_strategy={i+1}"							)
                          break

                  if "cpu_offload" in spec:
                      cmd_config.append('--fsdp_offload_params=True'							)

                  for policy in FSDP_AUTO_WRAP_POLICY:
                      if policy.lower() in spec:
                          cmd_config.append(F"--fsdp_auto_wrap_policy={policy}"							)
                          break

                  if policy == "TRANSFORMER_BASED_WRAP":
                      cmd_config.append('--fsdp_transformer_layer_cls_to_wrap=BertLayer'							)
                  elif policy == "SIZE_BASED_WRAP":
                      cmd_config.append('--fsdp_min_num_params=2000'							)

              cmd_config.extend(
                  [
                      self.test_file_path,
                      F"--output_dir={self.tmpdir}",
                      F"--peak_memory_upper_bound={peak_mem_upper_bound}",
                      F"--n_train={self.n_train}",
                      F"--n_val={self.n_val}",
                  ]							)
              with patch_environment(omp_num_threads=1							):
                  execute_subprocess_async(lowercase_   ,       env=os.environ.copy()							)





"""simple docstring"""

import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu
class      _UpperCAmelCase							(		nn.Module							):




      def __init__(							self							)						->					Any:
          super().__init__()
          UpperCAmelCase									=					nn.Linear(3   ,       4							)
          UpperCAmelCase									=					nn.BatchNormad(4							)
          UpperCAmelCase									=					nn.Linear(4   ,       5							)




      def     a_      (							self   ,       lowercase_							)						->					Any:
          return self.lineara(self.batchnorm(self.lineara(lowercase_							)							)							)

class      _UpperCAmelCase							(		SCREAMING_SNAKE_CASE_							):




      def     a_      (							self   ,       lowercase_   ,       *lowercase_   ,       **lowercase_							)						->					List[str]:
          return (args[0] + 1,) + args[1:], kwargs

class      _UpperCAmelCase							(		SCREAMING_SNAKE_CASE_							):




      def     a_      (							self   ,       lowercase_   ,       lowercase_							)						->					Union[str, Any]:
          return output + 1

class      _UpperCAmelCase							(		unittest.TestCase							):




      def     a_      (							self							)						->					Optional[int]:
          UpperCAmelCase									=					ModelForTest()
          UpperCAmelCase									=					ModelHook()

          add_hook_to_module(lowercase_   ,       lowercase_							)
          self.assertEqual(test_model._hf_hook   ,       lowercase_							)
          self.assertTrue(hasattr(lowercase_   ,       '_old_forward'							)							)

          # Check adding the hook did not change the name or the signature
          self.assertEqual(test_model.forward.__name__   ,       'forward'							)
          self.assertListEqual(list(inspect.signature(test_model.forward							).parameters							)   ,       ['x']							)

          remove_hook_from_module(lowercase_							)
          self.assertFalse(hasattr(lowercase_   ,       '_hf_hook'							)							)
          self.assertFalse(hasattr(lowercase_   ,       '_old_forward'							)							)




      def     a_      (							self							)						->					Any:
          UpperCAmelCase									=					ModelForTest()
          UpperCAmelCase									=					ModelHook()

          add_hook_to_module(lowercase_   ,       lowercase_							)
          add_hook_to_module(lowercase_   ,       lowercase_   ,       append=lowercase_							)

          self.assertEqual(isinstance(test_model._hf_hook   ,       lowercase_							)   ,       lowercase_							)
          self.assertEqual(len(test_model._hf_hook.hooks							)   ,       2							)
          self.assertTrue(hasattr(lowercase_   ,       '_old_forward'							)							)

          # Check adding the hook did not change the name or the signature
          self.assertEqual(test_model.forward.__name__   ,       'forward'							)
          self.assertListEqual(list(inspect.signature(test_model.forward							).parameters							)   ,       ['x']							)

          remove_hook_from_module(lowercase_							)
          self.assertFalse(hasattr(lowercase_   ,       '_hf_hook'							)							)
          self.assertFalse(hasattr(lowercase_   ,       '_old_forward'							)							)




      def     a_      (							self							)						->					Any:
          UpperCAmelCase									=					ModelForTest()
          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					test_model(x + 1							)
          UpperCAmelCase									=					test_model(x + 2							)

          UpperCAmelCase									=					PreForwardHook()
          add_hook_to_module(lowercase_   ,       lowercase_							)
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertTrue(torch.allclose(lowercase_   ,       lowercase_   ,       atol=1E-5							)							)

          # Attaching a hook to a model when it already has one replaces, does not chain
          UpperCAmelCase									=					PreForwardHook()
          add_hook_to_module(lowercase_   ,       lowercase_							)
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertTrue(torch.allclose(lowercase_   ,       lowercase_   ,       atol=1E-5							)							)

          # You need to use the sequential hook to chain two or more hooks
          UpperCAmelCase									=					SequentialHook(PreForwardHook()   ,       PreForwardHook()							)
          add_hook_to_module(lowercase_   ,       lowercase_							)

          UpperCAmelCase									=					test_model(lowercase_							)
          assert torch.allclose(lowercase_   ,       lowercase_   ,       atol=1E-5							)




      def     a_      (							self							)						->					List[str]:
          UpperCAmelCase									=					ModelForTest()
          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					test_model(lowercase_							)

          UpperCAmelCase									=					PostForwardHook()
          add_hook_to_module(lowercase_   ,       lowercase_							)
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertTrue(torch.allclose(lowercase_   ,       output + 1   ,       atol=1E-5							)							)

          # Attaching a hook to a model when it already has one replaces, does not chain
          UpperCAmelCase									=					PostForwardHook()
          add_hook_to_module(lowercase_   ,       lowercase_							)
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertTrue(torch.allclose(lowercase_   ,       output + 1   ,       atol=1E-5							)							)

          # You need to use the sequential hook to chain two or more hooks
          UpperCAmelCase									=					SequentialHook(PostForwardHook()   ,       PostForwardHook()							)
          add_hook_to_module(lowercase_   ,       lowercase_							)

          UpperCAmelCase									=					test_model(lowercase_							)
          assert torch.allclose(lowercase_   ,       output + 2   ,       atol=1E-5							)




      def     a_      (							self							)						->					Tuple:
          UpperCAmelCase									=					ModelForTest()
          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					test_model(lowercase_							)

          UpperCAmelCase									=					PostForwardHook()
          add_hook_to_module(lowercase_   ,       lowercase_							)
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertTrue(torch.allclose(lowercase_   ,       output + 1							)							)
          self.assertTrue(outputa.requires_grad							)

          UpperCAmelCase									=					True
          UpperCAmelCase									=					test_model(lowercase_							)
          self.assertFalse(outputa.requires_grad							)




      @require_multi_gpu
      def     a_      (							self							)						->					Optional[int]:
          UpperCAmelCase									=					ModelForTest()
          # Everything is on CPU
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # This will move each submodule on different devices
          add_hook_to_module(model.lineara   ,       AlignDevicesHook(execution_device=0							)							)
          add_hook_to_module(model.batchnorm   ,       AlignDevicesHook(execution_device=0							)							)
          add_hook_to_module(model.lineara   ,       AlignDevicesHook(execution_device=1							)							)

          self.assertEqual(model.lineara.weight.device   ,       torch.device(0							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device(0							)							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       torch.device(0							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device(1							)							)

          # We can still make a forward pass. The input does not need to be on any particular device
          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       torch.device(1							)							)

          # We can add a general hook to put back output on same device as input.
          add_hook_to_module(lowercase_   ,       AlignDevicesHook(io_same_device=lowercase_							)							)
          UpperCAmelCase									=					torch.randn(2   ,       3							).to(0							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       torch.device(0							)							)




      def     a_      (							self							)						->					Dict:
          UpperCAmelCase									=					ModelForTest()

          # Everything is on CPU
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # This will move each submodule on different devices
          UpperCAmelCase									=					{'execution_device': 0 if torch.cuda.is_available() else 'cpu', 'offload': True}

          add_hook_to_module(model.lineara   ,       AlignDevicesHook(**lowercase_							)							)
          add_hook_to_module(model.batchnorm   ,       AlignDevicesHook(**lowercase_							)							)
          add_hook_to_module(model.lineara   ,       AlignDevicesHook(**lowercase_							)							)

          # Parameters have been offloaded, so on the meta device
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          # Buffers are not included in the offload by default, so are on the execution device
          UpperCAmelCase									=					torch.device(hook_kwargs['execution_device']							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       lowercase_							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_module(model.lineara							)
          remove_hook_from_module(model.batchnorm							)
          remove_hook_from_module(model.lineara							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # Now test with buffers included in the offload
          UpperCAmelCase									=					{
              'execution_device': 0 if torch.cuda.is_available() else 'cpu',
              'offload': True,
              'offload_buffers': True,
          }

          add_hook_to_module(model.lineara   ,       AlignDevicesHook(**lowercase_							)							)
          add_hook_to_module(model.batchnorm   ,       AlignDevicesHook(**lowercase_							)							)
          add_hook_to_module(model.lineara   ,       AlignDevicesHook(**lowercase_							)							)

          # Parameters have been offloaded, so on the meta device, buffers included
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       torch.device('meta'							)							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_module(model.lineara							)
          remove_hook_from_module(model.batchnorm							)
          remove_hook_from_module(model.lineara							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)




      def     a_      (							self							)						->					Optional[int]:
          UpperCAmelCase									=					ModelForTest()

          # Everything is on CPU
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # This will move each submodule on different devices
          UpperCAmelCase									=					0 if torch.cuda.is_available() else 'cpu'
          attach_align_device_hook(lowercase_   ,       execution_device=lowercase_   ,       offload=lowercase_							)

          # Parameters have been offloaded, so on the meta device
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          # Buffers are not included in the offload by default, so are on the execution device
          UpperCAmelCase									=					torch.device(lowercase_							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       lowercase_							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_submodules(lowercase_							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # Now test with buffers included in the offload
          attach_align_device_hook(lowercase_   ,       execution_device=lowercase_   ,       offload=lowercase_   ,       offload_buffers=lowercase_							)

          # Parameters have been offloaded, so on the meta device, buffers included
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       torch.device('meta'							)							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_submodules(lowercase_							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)




      def     a_      (							self							)						->					Dict:
          UpperCAmelCase									=					ModelForTest()

          # Everything is on CPU
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # This will move each submodule on different devices
          UpperCAmelCase									=					0 if torch.cuda.is_available() else 'cpu'
          attach_align_device_hook(
              lowercase_   ,       execution_device=lowercase_   ,       offload=lowercase_   ,       weights_map=model.state_dict()							)

          # Parameters have been offloaded, so on the meta device
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          # Buffers are not included in the offload by default, so are on the execution device
          UpperCAmelCase									=					torch.device(lowercase_							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       lowercase_							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_submodules(lowercase_							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)

          # Now test with buffers included in the offload
          attach_align_device_hook(
              lowercase_   ,       execution_device=lowercase_   ,       offload=lowercase_   ,       weights_map=model.state_dict()   ,       offload_buffers=lowercase_   ,       )

          # Parameters have been offloaded, so on the meta device, buffers included
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('meta'							)							)
          self.assertEqual(model.batchnorm.running_mean.device   ,       torch.device('meta'							)							)

          UpperCAmelCase									=					torch.randn(2   ,       3							)
          UpperCAmelCase									=					model(lowercase_							)
          self.assertEqual(output.device   ,       lowercase_							)

          # Removing hooks loads back the weights in the model.
          remove_hook_from_submodules(lowercase_							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.batchnorm.weight.device   ,       torch.device('cpu'							)							)
          self.assertEqual(model.lineara.weight.device   ,       torch.device('cpu'							)							)



