





"""simple docstring"""



import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu






class 		__snake_case (		nn.Module					):





 """simple docstring"""


 def __init__(   self  ):



     '''simple docstring'''




     super().__init__()
     __A   :						Tuple									=					nn.Linear(3 ,						4  )
     __A   :						List[Any]									=					nn.BatchNormad(4  )
     __A   :						int									=					nn.Linear(4 ,						5  )







 def  UpperCamelCase__(   self ,						__lowerCamelCase  ):



     '''simple docstring'''




     return self.lineara(self.batchnorm(self.lineara(__lowerCamelCase  )  )  )






class 		__snake_case (		SCREAMING_SNAKE_CASE__					):





 """simple docstring"""


 def  UpperCamelCase__(   self ,						__lowerCamelCase ,						*__lowerCamelCase ,						**__lowerCamelCase  ):



     '''simple docstring'''




     return (args[0] + 1,) + args[1:], kwargs






class 		__snake_case (		SCREAMING_SNAKE_CASE__					):





 """simple docstring"""


 def  UpperCamelCase__(   self ,						__lowerCamelCase ,						__lowerCamelCase  ):



     '''simple docstring'''




     return output + 1






class 		__snake_case (		unittest.TestCase					):





 """simple docstring"""


 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     __A   :						List[str]									=					ModelForTest()
     __A   :						Union[str, Any]									=					ModelHook()

     add_hook_to_module(__lowerCamelCase ,						__lowerCamelCase  )
     self.assertEqual(test_model._hf_hook ,						__lowerCamelCase  )
     self.assertTrue(hasattr(__lowerCamelCase ,						'''_old_forward'''  )  )

     # Check adding the hook did not change the name or the signature
     self.assertEqual(test_model.forward.__name__ ,						'''forward'''  )
     self.assertListEqual(list(inspect.signature(test_model.forward  ).parameters  ) ,						['''x''']  )

     remove_hook_from_module(__lowerCamelCase  )
     self.assertFalse(hasattr(__lowerCamelCase ,						'''_hf_hook'''  )  )
     self.assertFalse(hasattr(__lowerCamelCase ,						'''_old_forward'''  )  )

 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     __A   :						List[str]									=					ModelForTest()
     __A   :						Optional[Any]									=					ModelHook()

     add_hook_to_module(__lowerCamelCase ,						__lowerCamelCase  )
     add_hook_to_module(__lowerCamelCase ,						__lowerCamelCase ,						append=__lowerCamelCase  )

     self.assertEqual(isinstance(test_model._hf_hook ,						__lowerCamelCase  ) ,						__lowerCamelCase  )
     self.assertEqual(len(test_model._hf_hook.hooks  ) ,						2  )
     self.assertTrue(hasattr(__lowerCamelCase ,						'''_old_forward'''  )  )

     # Check adding the hook did not change the name or the signature
     self.assertEqual(test_model.forward.__name__ ,						'''forward'''  )
     self.assertListEqual(list(inspect.signature(test_model.forward  ).parameters  ) ,						['''x''']  )

     remove_hook_from_module(__lowerCamelCase  )
     self.assertFalse(hasattr(__lowerCamelCase ,						'''_hf_hook'''  )  )
     self.assertFalse(hasattr(__lowerCamelCase ,						'''_old_forward'''  )  )

 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     __A   :						Any									=					ModelForTest()
     __A   :						Union[str, Any]									=					torch.randn(2 ,						3  )
     __A   :						Optional[int]									=					test_model(x + 1  )
     __A   :						List[Any]									=					test_model(x + 2  )

     __A   :						Dict									=					PreForwardHook()
     add_hook_to_module(__lowerCamelCase ,						__lowerCamelCase  )
     __A   :						str									=					test_model(__lowerCamelCase  )
     self.assertTrue(torch.allclose(__lowerCamelCase ,						__lowerCamelCase ,						atol=1e-5  )  )

     # Attaching a hook to a model when it already has one replaces, does not chain
     __A   :						int									=					PreForwardHook()
     add_hook_to_module(__lowerCamelCase ,						__lowerCamelCase  )
     __A   :						Optional[Any]									=					test_model(__lowerCamelCase  )
     self.assertTrue(torch.allclose(__lowerCamelCase ,						__lowerCamelCase ,						atol=1e-5  )  )

     # You need to use the sequential hook to chain two or more hooks
     __A   :						List[str]									=					SequentialHook(PreForwardHook() ,						PreForwardHook()  )
     add_hook_to_module(__lowerCamelCase ,						__lowerCamelCase  )

     __A   :						Dict									=					test_model(__lowerCamelCase  )
     assert torch.allclose(__lowerCamelCase ,						__lowerCamelCase ,						atol=1e-5  )

 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     __A   :						str									=					ModelForTest()
     __A   :						Optional[Any]									=					torch.randn(2 ,						3  )
     __A   :						Dict									=					test_model(__lowerCamelCase  )

     __A   :						Any									=					PostForwardHook()
     add_hook_to_module(__lowerCamelCase ,						__lowerCamelCase  )
     __A   :						Tuple									=					test_model(__lowerCamelCase  )
     self.assertTrue(torch.allclose(__lowerCamelCase ,						output + 1 ,						atol=1e-5  )  )

     # Attaching a hook to a model when it already has one replaces, does not chain
     __A   :						List[Any]									=					PostForwardHook()
     add_hook_to_module(__lowerCamelCase ,						__lowerCamelCase  )
     __A   :						str									=					test_model(__lowerCamelCase  )
     self.assertTrue(torch.allclose(__lowerCamelCase ,						output + 1 ,						atol=1e-5  )  )

     # You need to use the sequential hook to chain two or more hooks
     __A   :						Union[str, Any]									=					SequentialHook(PostForwardHook() ,						PostForwardHook()  )
     add_hook_to_module(__lowerCamelCase ,						__lowerCamelCase  )

     __A   :						Optional[Any]									=					test_model(__lowerCamelCase  )
     assert torch.allclose(__lowerCamelCase ,						output + 2 ,						atol=1e-5  )

 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     __A   :						Any									=					ModelForTest()
     __A   :						Tuple									=					torch.randn(2 ,						3  )
     __A   :						int									=					test_model(__lowerCamelCase  )

     __A   :						Optional[Any]									=					PostForwardHook()
     add_hook_to_module(__lowerCamelCase ,						__lowerCamelCase  )
     __A   :						Optional[int]									=					test_model(__lowerCamelCase  )
     self.assertTrue(torch.allclose(__lowerCamelCase ,						output + 1  )  )
     self.assertTrue(outputa.requires_grad  )

     __A   :						Tuple									=					True
     __A   :						Dict									=					test_model(__lowerCamelCase  )
     self.assertFalse(outputa.requires_grad  )

 @require_multi_gpu
 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     __A   :						Union[str, Any]									=					ModelForTest()
     # Everything is on CPU
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )

     # This will move each submodule on different devices
     add_hook_to_module(model.lineara ,						AlignDevicesHook(execution_device=0  )  )
     add_hook_to_module(model.batchnorm ,						AlignDevicesHook(execution_device=0  )  )
     add_hook_to_module(model.lineara ,						AlignDevicesHook(execution_device=1  )  )

     self.assertEqual(model.lineara.weight.device ,						torch.device(0  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device(0  )  )
     self.assertEqual(model.batchnorm.running_mean.device ,						torch.device(0  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device(1  )  )

     # We can still make a forward pass. The input does not need to be on any particular device
     __A   :						str									=					torch.randn(2 ,						3  )
     __A   :						Tuple									=					model(__lowerCamelCase  )
     self.assertEqual(output.device ,						torch.device(1  )  )

     # We can add a general hook to put back output on same device as input.
     add_hook_to_module(__lowerCamelCase ,						AlignDevicesHook(io_same_device=__lowerCamelCase  )  )
     __A   :						Union[str, Any]									=					torch.randn(2 ,						3  ).to(0  )
     __A   :						str									=					model(__lowerCamelCase  )
     self.assertEqual(output.device ,						torch.device(0  )  )

 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     __A   :						Dict									=					ModelForTest()

     # Everything is on CPU
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )

     # This will move each submodule on different devices
     __A   :						List[Any]									=					{'''execution_device''': 0 if torch.cuda.is_available() else '''cpu''', '''offload''': True}

     add_hook_to_module(model.lineara ,						AlignDevicesHook(**__lowerCamelCase  )  )
     add_hook_to_module(model.batchnorm ,						AlignDevicesHook(**__lowerCamelCase  )  )
     add_hook_to_module(model.lineara ,						AlignDevicesHook(**__lowerCamelCase  )  )

     # Parameters have been offloaded, so on the meta device
     self.assertEqual(model.lineara.weight.device ,						torch.device('''meta'''  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device('''meta'''  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''meta'''  )  )
     # Buffers are not included in the offload by default, so are on the execution device
     __A   :						Optional[Any]									=					torch.device(hook_kwargs['''execution_device''']  )
     self.assertEqual(model.batchnorm.running_mean.device ,						__lowerCamelCase  )

     __A   :						Union[str, Any]									=					torch.randn(2 ,						3  )
     __A   :						int									=					model(__lowerCamelCase  )
     self.assertEqual(output.device ,						__lowerCamelCase  )

     # Removing hooks loads back the weights in the model.
     remove_hook_from_module(model.lineara  )
     remove_hook_from_module(model.batchnorm  )
     remove_hook_from_module(model.lineara  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )

     # Now test with buffers included in the offload
     __A   :						Any									=					{
         '''execution_device''': 0 if torch.cuda.is_available() else '''cpu''',
         '''offload''': True,
         '''offload_buffers''': True,
     }

     add_hook_to_module(model.lineara ,						AlignDevicesHook(**__lowerCamelCase  )  )
     add_hook_to_module(model.batchnorm ,						AlignDevicesHook(**__lowerCamelCase  )  )
     add_hook_to_module(model.lineara ,						AlignDevicesHook(**__lowerCamelCase  )  )

     # Parameters have been offloaded, so on the meta device, buffers included
     self.assertEqual(model.lineara.weight.device ,						torch.device('''meta'''  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device('''meta'''  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''meta'''  )  )
     self.assertEqual(model.batchnorm.running_mean.device ,						torch.device('''meta'''  )  )

     __A   :						List[Any]									=					torch.randn(2 ,						3  )
     __A   :						Tuple									=					model(__lowerCamelCase  )
     self.assertEqual(output.device ,						__lowerCamelCase  )

     # Removing hooks loads back the weights in the model.
     remove_hook_from_module(model.lineara  )
     remove_hook_from_module(model.batchnorm  )
     remove_hook_from_module(model.lineara  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )

 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     __A   :						str									=					ModelForTest()

     # Everything is on CPU
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )

     # This will move each submodule on different devices
     __A   :						str									=					0 if torch.cuda.is_available() else '''cpu'''
     attach_align_device_hook(__lowerCamelCase ,						execution_device=__lowerCamelCase ,						offload=__lowerCamelCase  )

     # Parameters have been offloaded, so on the meta device
     self.assertEqual(model.lineara.weight.device ,						torch.device('''meta'''  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device('''meta'''  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''meta'''  )  )
     # Buffers are not included in the offload by default, so are on the execution device
     __A   :						int									=					torch.device(__lowerCamelCase  )
     self.assertEqual(model.batchnorm.running_mean.device ,						__lowerCamelCase  )

     __A   :						Any									=					torch.randn(2 ,						3  )
     __A   :						Any									=					model(__lowerCamelCase  )
     self.assertEqual(output.device ,						__lowerCamelCase  )

     # Removing hooks loads back the weights in the model.
     remove_hook_from_submodules(__lowerCamelCase  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )

     # Now test with buffers included in the offload
     attach_align_device_hook(__lowerCamelCase ,						execution_device=__lowerCamelCase ,						offload=__lowerCamelCase ,						offload_buffers=__lowerCamelCase  )

     # Parameters have been offloaded, so on the meta device, buffers included
     self.assertEqual(model.lineara.weight.device ,						torch.device('''meta'''  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device('''meta'''  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''meta'''  )  )
     self.assertEqual(model.batchnorm.running_mean.device ,						torch.device('''meta'''  )  )

     __A   :						List[str]									=					torch.randn(2 ,						3  )
     __A   :						Tuple									=					model(__lowerCamelCase  )
     self.assertEqual(output.device ,						__lowerCamelCase  )

     # Removing hooks loads back the weights in the model.
     remove_hook_from_submodules(__lowerCamelCase  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )







 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     __A   :						List[Any]									=					ModelForTest()

     # Everything is on CPU
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )

     # This will move each submodule on different devices
     __A   :						Optional[int]									=					0 if torch.cuda.is_available() else '''cpu'''
     attach_align_device_hook(
         __lowerCamelCase ,						execution_device=__lowerCamelCase ,						offload=__lowerCamelCase ,						weights_map=model.state_dict()  )

     # Parameters have been offloaded, so on the meta device
     self.assertEqual(model.lineara.weight.device ,						torch.device('''meta'''  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device('''meta'''  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''meta'''  )  )
     # Buffers are not included in the offload by default, so are on the execution device
     __A   :						Optional[Any]									=					torch.device(__lowerCamelCase  )
     self.assertEqual(model.batchnorm.running_mean.device ,						__lowerCamelCase  )

     __A   :						str									=					torch.randn(2 ,						3  )
     __A   :						Union[str, Any]									=					model(__lowerCamelCase  )
     self.assertEqual(output.device ,						__lowerCamelCase  )

     # Removing hooks loads back the weights in the model.
     remove_hook_from_submodules(__lowerCamelCase  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )

     # Now test with buffers included in the offload
     attach_align_device_hook(
         __lowerCamelCase ,						execution_device=__lowerCamelCase ,						offload=__lowerCamelCase ,						weights_map=model.state_dict() ,						offload_buffers=__lowerCamelCase ,						)

     # Parameters have been offloaded, so on the meta device, buffers included
     self.assertEqual(model.lineara.weight.device ,						torch.device('''meta'''  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device('''meta'''  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''meta'''  )  )
     self.assertEqual(model.batchnorm.running_mean.device ,						torch.device('''meta'''  )  )

     __A   :						Optional[Any]									=					torch.randn(2 ,						3  )
     __A   :						str									=					model(__lowerCamelCase  )
     self.assertEqual(output.device ,						__lowerCamelCase  )

     # Removing hooks loads back the weights in the model.
     remove_hook_from_submodules(__lowerCamelCase  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.batchnorm.weight.device ,						torch.device('''cpu'''  )  )
     self.assertEqual(model.lineara.weight.device ,						torch.device('''cpu'''  )  )











"""simple docstring"""



def 		__lowercase					(       )       ->Tuple:



    '''simple docstring'''







    __A   :						str									=					[]
    __A   :						List[Any]									=					1

    while len(snake_case_						) < 1e6:
        constant.append(str(snake_case_						)						)
        i += 1

    __A   :						Any									=					''''''.join(snake_case_						)

    return (
        int(constant[0]						)
        * int(constant[9]						)
        * int(constant[99]						)
        * int(constant[999]						)
        * int(constant[9999]						)
        * int(constant[99999]						)
        * int(constant[999999]						)
    )


if __name__ == "__main__":
       print(solution())




