



"""simple docstring"""






import json
import os
import tempfile
from unittest.mock import patch

import torch
from torch.utils.data import DataLoader, TensorDataset

from accelerate import DistributedType, infer_auto_device_map, init_empty_weights
from accelerate.accelerator import Accelerator
from accelerate.state import GradientState, PartialState
from accelerate.test_utils import require_bnb, require_multi_gpu, slow
from accelerate.test_utils.testing import AccelerateTestCase, require_cuda
from accelerate.utils import patch_environment






def 					snake_case__       (		):
   """simple docstring"""
   lowerCamelCase__		:		Union[str, Any]		     =torch.nn.Linear(2							,		4     )
   lowerCamelCase__		:		List[str]		     =torch.optim.AdamW(model.parameters()							,		lr=1.0     )
   lowerCamelCase__		:		int		     =torch.optim.lr_scheduler.OneCycleLR(__lowerCamelCase							,		max_lr=0.01							,		steps_per_epoch=2							,		epochs=1     )
   lowerCamelCase__		:		Dict		     =DataLoader(TensorDataset(torch.tensor([1, 2, 3]     )     )     )
   lowerCamelCase__		:		Optional[int]		     =DataLoader(TensorDataset(torch.tensor([4, 5, 6]     )     )     )

   return model, optimizer, scheduler, train_dl, valid_dl






def 					snake_case__       (		__lowerCamelCase    :   List[str]     ):
   """simple docstring"""
   return (model.weight.abs().sum() + model.bias.abs().sum()).item()






def 					snake_case__       (		__lowerCamelCase    :   Union[str, Any]     ):
   """simple docstring"""
   lowerCamelCase__		:		List[Any]		     =torch.nn.Linear(*tuple(model.weight.T.shape     )     ).state_dict()
   model.load_state_dict(__lowerCamelCase     )

class 	__SCREAMING_SNAKE_CASE     (				lowerCAmelCase_       ):


     '''simple docstring'''





     @require_cuda
     def 		snake_case	(	self							:  Any					)-> Optional[int]:
        lowerCamelCase__		:		str		     =Accelerator()
        assert PartialState._shared_state["_cpu"] is False
        assert PartialState._shared_state["device"].type == "cuda"
        with self.assertRaises(lowerCamelCase					):
           lowerCamelCase__		:		Union[str, Any]		     =Accelerator(cpu=lowerCamelCase					)



     def 		snake_case	(	self							:  Dict					)-> Tuple:
        lowerCamelCase__		:		List[Any]		     =Accelerator()
        lowerCamelCase__		:		Union[str, Any]		     =GradientState()
        assert state.num_steps == 1
        lowerCamelCase__		:		Tuple		     =4
        assert state.num_steps == 4

        assert state.sync_gradients is True
        lowerCamelCase__		:		Any		     =False
        assert state.sync_gradients is False
        GradientState._reset_state()



     def 		snake_case	(	self							:  Any					)-> Optional[int]:
        lowerCamelCase__		:		Dict		     =Accelerator()
        lowerCamelCase__		:		int		     =create_components()

        (
            lowerCamelCase__       
        )		:		Optional[int]		     =accelerator.prepare(lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase					)

        self.assertTrue(prepared_model in accelerator._models					)
        self.assertTrue(prepared_optimizer in accelerator._optimizers					)
        self.assertTrue(prepared_scheduler in accelerator._schedulers					)
        self.assertTrue(prepared_train_dl in accelerator._dataloaders					)
        self.assertTrue(prepared_valid_dl in accelerator._dataloaders					)



     def 		snake_case	(	self							:  str					)-> Optional[Any]:
        lowerCamelCase__		:		Dict		     =Accelerator()
        lowerCamelCase__		:		Dict		     =create_components()
        accelerator.prepare(lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase					)
        accelerator.free_memory()

        self.assertTrue(len(accelerator._models					) == 0					)
        self.assertTrue(len(accelerator._optimizers					) == 0					)
        self.assertTrue(len(accelerator._schedulers					) == 0					)
        self.assertTrue(len(accelerator._dataloaders					) == 0					)



     def 		snake_case	(	self							:  int					)-> Dict:
        PartialState._reset_state()

        # Mock torch.cuda.set_device to avoid an exception as the device doesn't exist
        def noop(*lowerCamelCase							:  List[Any],  **lowerCamelCase							:  Tuple					):
           pass

        with patch('''torch.cuda.set_device''',  lowerCamelCase					), patch_environment(ACCELERATE_TORCH_DEVICE='''cuda:64'''					):
           lowerCamelCase__		:		Optional[int]		     =Accelerator()
           self.assertEqual(str(accelerator.state.device					),  '''cuda:64'''					)



     def 		snake_case	(	self							:  str					)-> Any:
        lowerCamelCase__		:		Union[str, Any]		     =Accelerator()
        lowerCamelCase__		:		int		     =create_components()
        accelerator.prepare(lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase					)

        lowerCamelCase__		:		Optional[Any]		     =get_signature(lowerCamelCase					)

        with tempfile.TemporaryDirectory() as tmpdirname:
           accelerator.save_state(lowerCamelCase					)

           # make sure random weights don't match
           load_random_weights(lowerCamelCase					)
           self.assertTrue(abs(model_signature - get_signature(lowerCamelCase					)					) > 1E-3					)

           # make sure loaded weights match
           accelerator.load_state(lowerCamelCase					)
           self.assertTrue(abs(model_signature - get_signature(lowerCamelCase					)					) < 1E-3					)



     def 		snake_case	(	self							:  List[Any]					)-> str:
        lowerCamelCase__		:		List[str]		     =Accelerator()
        lowerCamelCase__		:		Tuple		     =create_components()
        accelerator.prepare(lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase					)

        lowerCamelCase__		:		Tuple		     =get_signature(lowerCamelCase					)

        # saving hook
        def save_config(lowerCamelCase							:  List[Any],  lowerCamelCase							:  List[Any],  lowerCamelCase							:  Any					):
           lowerCamelCase__		:		Any		     ={'''class_name''': models[0].__class__.__name__}

           with open(os.path.join(lowerCamelCase,  '''data.json'''					),  '''w'''					) as f:
              json.dump(lowerCamelCase,  lowerCamelCase					)

        # loading hook
        def load_config(lowerCamelCase							:  str,  lowerCamelCase							:  Any					):
           with open(os.path.join(lowerCamelCase,  '''data.json'''					),  '''r'''					) as f:
              lowerCamelCase__		:		Tuple		     =json.load(lowerCamelCase					)

           lowerCamelCase__		:		List[Any]		     =config['''class_name''']

        lowerCamelCase__		:		Tuple		     =accelerator.register_save_state_pre_hook(lowerCamelCase					)
        lowerCamelCase__		:		Optional[Any]		     =accelerator.register_load_state_pre_hook(lowerCamelCase					)

        with tempfile.TemporaryDirectory() as tmpdirname:
           accelerator.save_state(lowerCamelCase					)

           # make sure random weights don't match with hooks
           load_random_weights(lowerCamelCase					)
           self.assertTrue(abs(model_signature - get_signature(lowerCamelCase					)					) > 1E-3					)

           # random class name to verify correct one is loaded
           lowerCamelCase__		:		Union[str, Any]		     ='''random'''

           # make sure loaded weights match with hooks
           accelerator.load_state(lowerCamelCase					)
           self.assertTrue(abs(model_signature - get_signature(lowerCamelCase					)					) < 1E-3					)

           # mode.class_name is loaded from config
           self.assertTrue(model.class_name == model.__class__.__name__					)

        # remove hooks
        save_hook.remove()
        load_hook.remove()

        with tempfile.TemporaryDirectory() as tmpdirname:
           accelerator.save_state(lowerCamelCase					)

           # make sure random weights don't match with hooks removed
           load_random_weights(lowerCamelCase					)
           self.assertTrue(abs(model_signature - get_signature(lowerCamelCase					)					) > 1E-3					)

           # random class name to verify correct one is loaded
           lowerCamelCase__		:		List[Any]		     ='''random'''

           # make sure loaded weights match with hooks removed
           accelerator.load_state(lowerCamelCase					)
           self.assertTrue(abs(model_signature - get_signature(lowerCamelCase					)					) < 1E-3					)

           # mode.class_name is NOT loaded from config
           self.assertTrue(model.class_name != model.__class__.__name__					)



     def 		snake_case	(	self							:  Optional[int]					)-> Tuple:
        lowerCamelCase__		:		str		     =Accelerator()
        lowerCamelCase__		:		Any		     =create_components()
        lowerCamelCase__		:		Tuple		     =None

        # This should work
        lowerCamelCase__		:		Optional[Any]		     =accelerator.prepare(
            lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase					)
        self.assertTrue(dummy_obj is None					)



     def 		snake_case	(	self							:  Dict					)-> Tuple:
        lowerCamelCase__		:		List[str]		     =Accelerator()
        lowerCamelCase__		:		Tuple		     =create_components()
        lowerCamelCase__		:		List[str]		     =[1, 2, 3]

        # This should work
        lowerCamelCase__		:		List[str]		     =accelerator.prepare(
            lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  lowerCamelCase					)
        self.assertEqual(
            getattr(lowerCamelCase,  '''_is_accelerate_prepared''',  lowerCamelCase					),  lowerCamelCase,  '''Dummy object should have `_is_accelerate_prepared` set to `True`''',  )
        self.assertEqual(
            getattr(lowerCamelCase,  '''_is_accelerate_prepared''',  lowerCamelCase					),  lowerCamelCase,  '''Model is missing `_is_accelerator_prepared` or is set to `False`''',  )
        self.assertEqual(
            getattr(lowerCamelCase,  '''_is_accelerate_prepared''',  lowerCamelCase					),  lowerCamelCase,  '''Optimizer is missing `_is_accelerator_prepared` or is set to `False`''',  )
        self.assertEqual(
            getattr(lowerCamelCase,  '''_is_accelerate_prepared''',  lowerCamelCase					),  lowerCamelCase,  '''Scheduler is missing `_is_accelerator_prepared` or is set to `False`''',  )
        self.assertEqual(
            getattr(lowerCamelCase,  '''_is_accelerate_prepared''',  lowerCamelCase					),  lowerCamelCase,  '''Train Dataloader is missing `_is_accelerator_prepared` or is set to `False`''',  )
        self.assertEqual(
            getattr(lowerCamelCase,  '''_is_accelerate_prepared''',  lowerCamelCase					),  lowerCamelCase,  '''Valid Dataloader is missing `_is_accelerator_prepared` or is set to `False`''',  )



     @slow
     @require_bnb
     def 		snake_case	(	self							:  Tuple					)-> List[Any]:
        from transformers import AutoModelForCausalLM

        lowerCamelCase__		:		Dict		     =AutoModelForCausalLM.from_pretrained(
            '''EleutherAI/gpt-neo-125m''',  load_in_abit=lowerCamelCase,  device_map={'''''': 0},  )
        lowerCamelCase__		:		Optional[Any]		     =Accelerator()

        # This should work
        lowerCamelCase__		:		Tuple		     =accelerator.prepare(lowerCamelCase					)



     @slow
     @require_bnb
     def 		snake_case	(	self							:  Dict					)-> List[Any]:
        from transformers import AutoModelForCausalLM

        lowerCamelCase__		:		int		     =Accelerator()

        with init_empty_weights():
           lowerCamelCase__		:		Optional[int]		     =AutoModelForCausalLM.from_pretrained(
               '''EleutherAI/gpt-neo-125m''',  )
           model.tie_weights()
           lowerCamelCase__		:		List[Any]		     =infer_auto_device_map(lowerCamelCase					)
           lowerCamelCase__		:		Any		     ='''cpu'''

        lowerCamelCase__		:		Any		     =AutoModelForCausalLM.from_pretrained(
            '''EleutherAI/gpt-neo-125m''',  device_map=lowerCamelCase,  load_in_abit=lowerCamelCase,  llm_inta_enable_fpaa_cpu_offload=lowerCamelCase					)

        # This should not work and get value error
        with self.assertRaises(lowerCamelCase					):
           lowerCamelCase__		:		Optional[Any]		     =accelerator.prepare(lowerCamelCase					)



     @slow
     @require_bnb
     @require_multi_gpu
     def 		snake_case	(	self							:  Any					)-> Dict:
        from transformers import AutoModelForCausalLM

        lowerCamelCase__		:		Dict		     ={'''distributed_type''': DistributedType.MULTI_GPU}

        with init_empty_weights():
           lowerCamelCase__		:		Tuple		     =AutoModelForCausalLM.from_pretrained(
               '''EleutherAI/gpt-neo-125m''',  )
           model.tie_weights()
           lowerCamelCase__		:		Union[str, Any]		     =infer_auto_device_map(lowerCamelCase					)
           lowerCamelCase__		:		int		     =1

        lowerCamelCase__		:		int		     =AutoModelForCausalLM.from_pretrained(
            '''EleutherAI/gpt-neo-125m''',  load_in_abit=lowerCamelCase,  device_map=lowerCamelCase,  )
        lowerCamelCase__		:		List[Any]		     =Accelerator()

        # This should not work and get value error
        with self.assertRaises(lowerCamelCase					):
           lowerCamelCase__		:		int		     =accelerator.prepare(lowerCamelCase					)

        PartialState._reset_state()



     @slow
     @require_bnb
     @require_multi_gpu
     def 		snake_case	(	self							:  Any					)-> List[str]:
        from transformers import AutoModelForCausalLM

        with init_empty_weights():
           lowerCamelCase__		:		List[str]		     =AutoModelForCausalLM.from_pretrained(
               '''EleutherAI/gpt-neo-125m''',  )
           lowerCamelCase__		:		Optional[Any]		     =infer_auto_device_map(lowerCamelCase					)
           lowerCamelCase__		:		Union[str, Any]		     =1

        lowerCamelCase__		:		Any		     =AutoModelForCausalLM.from_pretrained(
            '''EleutherAI/gpt-neo-125m''',  load_in_abit=lowerCamelCase,  device_map=lowerCamelCase,  )
        lowerCamelCase__		:		Any		     =Accelerator()

        # This should work
        lowerCamelCase__		:		Optional[Any]		     =accelerator.prepare(lowerCamelCase					)




     @require_cuda
     def 		snake_case	(	self							:  Any					)-> List[Any]:
        lowerCamelCase__		:		Dict		     =torch.nn.Linear(10,  10					)
        lowerCamelCase__		:		str		     =torch.optim.SGD(model.parameters(),  lr=0.01					)
        lowerCamelCase__		:		Dict		     =Accelerator(cpu=lowerCamelCase					)
        lowerCamelCase__		:		Optional[Any]		     =accelerator.prepare(lowerCamelCase					)









"""simple docstring"""






def 					snake_case__       (		__lowerCamelCase    :   int							,		__lowerCamelCase    :   int     ):
			"""simple docstring"""
			return int((input_a, input_a).count(0     ) != 0     )






def 					snake_case__       (		):
			"""simple docstring"""
			assert nand_gate(0							,		0     ) == 1
			assert nand_gate(0							,		1     ) == 1
			assert nand_gate(1							,		0     ) == 1
			assert nand_gate(1							,		1     ) == 0


if __name__ == "__main__":
		print(nand_gate(0, 0))
		print(nand_gate(0, 1))
		print(nand_gate(1, 0))
		print(nand_gate(1, 1))




