





import math
import random
from typing import Any

from .hill_climbing import SearchProblem





def       lowercase_      (		lowercase__				,  lowercase__ = True				,  lowercase__ = math.inf				,  lowercase__ = -math.inf				,  lowercase__ = math.inf				,  lowercase__ = -math.inf				,  lowercase__ = False				,  lowercase__ = 100				,  lowercase__ = 0.0_1				,  lowercase__ = 1				,  )   ->Any:
						_snake_case:   List[str]  					=		False
						_snake_case:   Union[str, Any]  					=		search_prob
						_snake_case:   Union[str, Any]  					=		start_temperate
						_snake_case:   int  					=		[]
						_snake_case:   Dict  					=		0
						_snake_case:   Union[str, Any]  					=		None

						while not search_end:
												_snake_case:   Optional[Any]  					=		current_state.score()
												if best_state is None or current_score > best_state.score():
																		_snake_case:   List[str]  					=		current_state
												scores.append(lowercase__						)
												iterations += 1
												_snake_case:   Optional[int]  					=		None
												_snake_case:   str  					=		current_state.get_neighbors()
												while (
												    next_state is None and neighbors
												):  # till we do not find a neighbor that we can move to
																		_snake_case:   Dict  					=		random.randint(0				,  len(lowercase__						) - 1						)  # picking a random neighbor
																		_snake_case:   List[Any]  					=		neighbors.pop(lowercase__						)
																		_snake_case:   str  					=		picked_neighbor.score() - current_score

																		if (
																		    picked_neighbor.x > max_x
																		    or picked_neighbor.x < min_x
																		    or picked_neighbor.y > max_y
																		    or picked_neighbor.y < min_y
																		):
																								continue  # neighbor outside our bounds

																		if not find_max:
																								_snake_case:   List[str]  					=		change * -1  # in case we are finding minimum
																		if change > 0:  # improves the solution
																								_snake_case:   Dict  					=		picked_neighbor
																		else:
																								_snake_case:   Union[str, Any]  					=		(math.e) ** (
																								    change / current_temp
																								)  # probability generation function
																								if random.random() < probability:  # random number within probability
																														_snake_case:   Union[str, Any]  					=		picked_neighbor
												_snake_case:   List[Any]  					=		current_temp - (current_temp * rate_of_decrease)

												if current_temp < threshold_temp or next_state is None:
																		# temperature below threshold, or could not find a suitable neighbor
																		_snake_case:   List[str]  					=		True
												else:
																		_snake_case:   List[str]  					=		next_state

						if visualization:
												from matplotlib import pyplot as plt

												plt.plot(range(lowercase__						)				,  lowercase__						)
												plt.xlabel('Iterations'						)
												plt.ylabel('Function values'						)
												plt.show()
						return best_state


if __name__ == "__main__":





		def       lowercase_      (		lowercase__				,  lowercase__						)   ->List[str]:
								return (x**2) + (y**2)

		# starting the problem with initial coordinates (12, 47)
		A   : Dict             =			SearchProblem(x=12, y=47, step_size=1, function_to_optimize=test_fa)
		A   : Union[str, Any]             =			simulated_annealing(
		    prob, find_max=False, max_x=100, min_x=5, max_y=50, min_y=-5, visualization=True
		)
		print(
		    'The minimum score for f(x, y) = x^2 + y^2 with the domain 100 > x > 5 '
		    F'and 50 > y > - 5 found via hill climbing: {local_min.score()}'
		)

		# starting the problem with initial coordinates (12, 47)
		A   : str             =			SearchProblem(x=12, y=47, step_size=1, function_to_optimize=test_fa)
		A   : List[Any]             =			simulated_annealing(
		    prob, find_max=True, max_x=100, min_x=5, max_y=50, min_y=-5, visualization=True
		)
		print(
		    'The maximum score for f(x, y) = x^2 + y^2 with the domain 100 > x > 5 '
		    F'and 50 > y > - 5 found via hill climbing: {local_min.score()}'
		)





		def       lowercase_      (		lowercase__				,  lowercase__						)   ->str:
								return (3 * x**2) - (6 * y)

		A   : Any             =			SearchProblem(x=3, y=4, step_size=1, function_to_optimize=test_fa)
		A   : List[str]             =			simulated_annealing(prob, find_max=False, visualization=True)
		print(
		    'The minimum score for f(x, y) = 3*x^2 - 6*y found via hill climbing: '
		    F'{local_min.score()}'
		)

		A   : Any             =			SearchProblem(x=3, y=4, step_size=1, function_to_optimize=test_fa)
		A   : List[Any]             =			simulated_annealing(prob, find_max=True, visualization=True)



		print(
		    'The maximum score for f(x, y) = 3*x^2 - 6*y found via hill climbing: '
		    F'{local_min.score()}'
		)












'''simple docstring'''






import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu







class 			lowerCamelCase					(						nn.Module  ):


					def __init__(					self	:  Optional[int]       ):





											'''simple docstring'''





											super().__init__()
											_snake_case:   int  					=		nn.Linear(3       ,     4       )
											_snake_case:   Any  					=		nn.BatchNormad(4       )
											_snake_case:   Union[str, Any]  					=		nn.Linear(4       ,     5       )




					def    SCREAMING_SNAKE_CASE_		(					self	:  Tuple       ,     __snake_case	:  List[Any]       ):





											'''simple docstring'''





											return self.lineara(self.batchnorm(self.lineara(__snake_case       )       )       )







class 			lowerCamelCase					(						__UpperCAmelCase  ):


					def    SCREAMING_SNAKE_CASE_		(					self	:  Tuple       ,     __snake_case	:  Optional[Any]       ,     *__snake_case	:  Optional[Any]       ,     **__snake_case	:  List[Any]       ):





											'''simple docstring'''





											return (args[0] + 1,) + args[1:], kwargs







class 			lowerCamelCase					(						__UpperCAmelCase  ):


					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ,     __snake_case	:  Union[str, Any]       ,     __snake_case	:  Tuple       ):





											'''simple docstring'''





											return output + 1







class 			lowerCamelCase					(						unittest.TestCase  ):


					def    SCREAMING_SNAKE_CASE_		(					self	:  Union[str, Any]       ):





											'''simple docstring'''





											_snake_case:   str  					=		ModelForTest()
											_snake_case:   List[str]  					=		ModelHook()

											add_hook_to_module(__snake_case       ,     __snake_case       )
											self.assertEqual(test_model._hf_hook       ,     __snake_case       )
											self.assertTrue(hasattr(__snake_case       ,     '_old_forward'       )       )

											# Check adding the hook did not change the name or the signature
											self.assertEqual(test_model.forward.__name__       ,     'forward'       )
											self.assertListEqual(list(inspect.signature(test_model.forward       ).parameters       )       ,     ['x']       )

											remove_hook_from_module(__snake_case       )
											self.assertFalse(hasattr(__snake_case       ,     '_hf_hook'       )       )
											self.assertFalse(hasattr(__snake_case       ,     '_old_forward'       )       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[Any]       ):





											'''simple docstring'''





											_snake_case:   Optional[Any]  					=		ModelForTest()
											_snake_case:   Union[str, Any]  					=		ModelHook()

											add_hook_to_module(__snake_case       ,     __snake_case       )
											add_hook_to_module(__snake_case       ,     __snake_case       ,     append=__snake_case       )

											self.assertEqual(isinstance(test_model._hf_hook       ,     __snake_case       )       ,     __snake_case       )
											self.assertEqual(len(test_model._hf_hook.hooks       )       ,     2       )
											self.assertTrue(hasattr(__snake_case       ,     '_old_forward'       )       )

											# Check adding the hook did not change the name or the signature
											self.assertEqual(test_model.forward.__name__       ,     'forward'       )
											self.assertListEqual(list(inspect.signature(test_model.forward       ).parameters       )       ,     ['x']       )

											remove_hook_from_module(__snake_case       )
											self.assertFalse(hasattr(__snake_case       ,     '_hf_hook'       )       )
											self.assertFalse(hasattr(__snake_case       ,     '_old_forward'       )       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[Any]       ):





											'''simple docstring'''





											_snake_case:   Dict  					=		ModelForTest()
											_snake_case:   Tuple  					=		torch.randn(2       ,     3       )
											_snake_case:   List[Any]  					=		test_model(x + 1       )
											_snake_case:   int  					=		test_model(x + 2       )

											_snake_case:   List[str]  					=		PreForwardHook()
											add_hook_to_module(__snake_case       ,     __snake_case       )
											_snake_case:   List[Any]  					=		test_model(__snake_case       )
											self.assertTrue(torch.allclose(__snake_case       ,     __snake_case       ,     atol=1e-5       )       )

											# Attaching a hook to a model when it already has one replaces, does not chain
											_snake_case:   Any  					=		PreForwardHook()
											add_hook_to_module(__snake_case       ,     __snake_case       )
											_snake_case:   Dict  					=		test_model(__snake_case       )
											self.assertTrue(torch.allclose(__snake_case       ,     __snake_case       ,     atol=1e-5       )       )

											# You need to use the sequential hook to chain two or more hooks
											_snake_case:   Any  					=		SequentialHook(PreForwardHook()       ,     PreForwardHook()       )
											add_hook_to_module(__snake_case       ,     __snake_case       )

											_snake_case:   int  					=		test_model(__snake_case       )
											assert torch.allclose(__snake_case       ,     __snake_case       ,     atol=1e-5       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[str]       ):





											'''simple docstring'''





											_snake_case:   int  					=		ModelForTest()
											_snake_case:   str  					=		torch.randn(2       ,     3       )
											_snake_case:   int  					=		test_model(__snake_case       )

											_snake_case:   Union[str, Any]  					=		PostForwardHook()
											add_hook_to_module(__snake_case       ,     __snake_case       )
											_snake_case:   int  					=		test_model(__snake_case       )
											self.assertTrue(torch.allclose(__snake_case       ,     output + 1       ,     atol=1e-5       )       )

											# Attaching a hook to a model when it already has one replaces, does not chain
											_snake_case:   Tuple  					=		PostForwardHook()
											add_hook_to_module(__snake_case       ,     __snake_case       )
											_snake_case:   Optional[int]  					=		test_model(__snake_case       )
											self.assertTrue(torch.allclose(__snake_case       ,     output + 1       ,     atol=1e-5       )       )

											# You need to use the sequential hook to chain two or more hooks
											_snake_case:   List[str]  					=		SequentialHook(PostForwardHook()       ,     PostForwardHook()       )
											add_hook_to_module(__snake_case       ,     __snake_case       )

											_snake_case:   List[Any]  					=		test_model(__snake_case       )
											assert torch.allclose(__snake_case       ,     output + 2       ,     atol=1e-5       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[Any]       ):





											'''simple docstring'''





											_snake_case:   Optional[Any]  					=		ModelForTest()
											_snake_case:   Dict  					=		torch.randn(2       ,     3       )
											_snake_case:   Tuple  					=		test_model(__snake_case       )

											_snake_case:   Tuple  					=		PostForwardHook()
											add_hook_to_module(__snake_case       ,     __snake_case       )
											_snake_case:   List[str]  					=		test_model(__snake_case       )
											self.assertTrue(torch.allclose(__snake_case       ,     output + 1       )       )
											self.assertTrue(outputa.requires_grad       )

											_snake_case:   Any  					=		True
											_snake_case:   str  					=		test_model(__snake_case       )
											self.assertFalse(outputa.requires_grad       )


					@require_multi_gpu
					def    SCREAMING_SNAKE_CASE_		(					self	:  Tuple       ):





											'''simple docstring'''





											_snake_case:   Any  					=		ModelForTest()
											# Everything is on CPU
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )

											# This will move each submodule on different devices
											add_hook_to_module(model.lineara       ,     AlignDevicesHook(execution_device=0       )       )
											add_hook_to_module(model.batchnorm       ,     AlignDevicesHook(execution_device=0       )       )
											add_hook_to_module(model.lineara       ,     AlignDevicesHook(execution_device=1       )       )

											self.assertEqual(model.lineara.weight.device       ,     torch.device(0       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device(0       )       )
											self.assertEqual(model.batchnorm.running_mean.device       ,     torch.device(0       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device(1       )       )

											# We can still make a forward pass. The input does not need to be on any particular device
											_snake_case:   int  					=		torch.randn(2       ,     3       )
											_snake_case:   Tuple  					=		model(__snake_case       )
											self.assertEqual(output.device       ,     torch.device(1       )       )

											# We can add a general hook to put back output on same device as input.
											add_hook_to_module(__snake_case       ,     AlignDevicesHook(io_same_device=__snake_case       )       )
											_snake_case:   Optional[Any]  					=		torch.randn(2       ,     3       ).to(0       )
											_snake_case:   Any  					=		model(__snake_case       )
											self.assertEqual(output.device       ,     torch.device(0       )       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Any       ):





											'''simple docstring'''





											_snake_case:   Union[str, Any]  					=		ModelForTest()

											# Everything is on CPU
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )

											# This will move each submodule on different devices
											_snake_case:   Optional[int]  					=		{'execution_device': 0 if torch.cuda.is_available() else 'cpu', 'offload': True}

											add_hook_to_module(model.lineara       ,     AlignDevicesHook(**__snake_case       )       )
											add_hook_to_module(model.batchnorm       ,     AlignDevicesHook(**__snake_case       )       )
											add_hook_to_module(model.lineara       ,     AlignDevicesHook(**__snake_case       )       )

											# Parameters have been offloaded, so on the meta device
											self.assertEqual(model.lineara.weight.device       ,     torch.device('meta'       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device('meta'       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('meta'       )       )
											# Buffers are not included in the offload by default, so are on the execution device
											_snake_case:   Optional[int]  					=		torch.device(hook_kwargs['execution_device']       )
											self.assertEqual(model.batchnorm.running_mean.device       ,     __snake_case       )

											_snake_case:   Tuple  					=		torch.randn(2       ,     3       )
											_snake_case:   Union[str, Any]  					=		model(__snake_case       )
											self.assertEqual(output.device       ,     __snake_case       )

											# Removing hooks loads back the weights in the model.
											remove_hook_from_module(model.lineara       )
											remove_hook_from_module(model.batchnorm       )
											remove_hook_from_module(model.lineara       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )

											# Now test with buffers included in the offload
											_snake_case:   Optional[Any]  					=		{
											    'execution_device': 0 if torch.cuda.is_available() else 'cpu',
											    'offload': True,
											    'offload_buffers': True,
											}

											add_hook_to_module(model.lineara       ,     AlignDevicesHook(**__snake_case       )       )
											add_hook_to_module(model.batchnorm       ,     AlignDevicesHook(**__snake_case       )       )
											add_hook_to_module(model.lineara       ,     AlignDevicesHook(**__snake_case       )       )

											# Parameters have been offloaded, so on the meta device, buffers included
											self.assertEqual(model.lineara.weight.device       ,     torch.device('meta'       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device('meta'       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('meta'       )       )
											self.assertEqual(model.batchnorm.running_mean.device       ,     torch.device('meta'       )       )

											_snake_case:   Optional[Any]  					=		torch.randn(2       ,     3       )
											_snake_case:   Union[str, Any]  					=		model(__snake_case       )
											self.assertEqual(output.device       ,     __snake_case       )

											# Removing hooks loads back the weights in the model.
											remove_hook_from_module(model.lineara       )
											remove_hook_from_module(model.batchnorm       )
											remove_hook_from_module(model.lineara       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Any       ):





											'''simple docstring'''





											_snake_case:   Union[str, Any]  					=		ModelForTest()

											# Everything is on CPU
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )

											# This will move each submodule on different devices
											_snake_case:   Optional[Any]  					=		0 if torch.cuda.is_available() else 'cpu'
											attach_align_device_hook(__snake_case       ,     execution_device=__snake_case       ,     offload=__snake_case       )

											# Parameters have been offloaded, so on the meta device
											self.assertEqual(model.lineara.weight.device       ,     torch.device('meta'       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device('meta'       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('meta'       )       )
											# Buffers are not included in the offload by default, so are on the execution device
											_snake_case:   List[str]  					=		torch.device(__snake_case       )
											self.assertEqual(model.batchnorm.running_mean.device       ,     __snake_case       )

											_snake_case:   Optional[Any]  					=		torch.randn(2       ,     3       )
											_snake_case:   List[str]  					=		model(__snake_case       )
											self.assertEqual(output.device       ,     __snake_case       )

											# Removing hooks loads back the weights in the model.
											remove_hook_from_submodules(__snake_case       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )

											# Now test with buffers included in the offload
											attach_align_device_hook(__snake_case       ,     execution_device=__snake_case       ,     offload=__snake_case       ,     offload_buffers=__snake_case       )

											# Parameters have been offloaded, so on the meta device, buffers included
											self.assertEqual(model.lineara.weight.device       ,     torch.device('meta'       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device('meta'       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('meta'       )       )
											self.assertEqual(model.batchnorm.running_mean.device       ,     torch.device('meta'       )       )

											_snake_case:   Any  					=		torch.randn(2       ,     3       )
											_snake_case:   List[Any]  					=		model(__snake_case       )
											self.assertEqual(output.device       ,     __snake_case       )

											# Removing hooks loads back the weights in the model.
											remove_hook_from_submodules(__snake_case       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )




					def    SCREAMING_SNAKE_CASE_		(					self	:  Any       ):





											'''simple docstring'''





											_snake_case:   Tuple  					=		ModelForTest()

											# Everything is on CPU
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )

											# This will move each submodule on different devices
											_snake_case:   Tuple  					=		0 if torch.cuda.is_available() else 'cpu'
											attach_align_device_hook(
											    __snake_case       ,     execution_device=__snake_case       ,     offload=__snake_case       ,     weights_map=model.state_dict()       )

											# Parameters have been offloaded, so on the meta device
											self.assertEqual(model.lineara.weight.device       ,     torch.device('meta'       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device('meta'       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('meta'       )       )
											# Buffers are not included in the offload by default, so are on the execution device
											_snake_case:   str  					=		torch.device(__snake_case       )
											self.assertEqual(model.batchnorm.running_mean.device       ,     __snake_case       )

											_snake_case:   Dict  					=		torch.randn(2       ,     3       )
											_snake_case:   Dict  					=		model(__snake_case       )
											self.assertEqual(output.device       ,     __snake_case       )

											# Removing hooks loads back the weights in the model.
											remove_hook_from_submodules(__snake_case       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )

											# Now test with buffers included in the offload
											attach_align_device_hook(
											    __snake_case       ,     execution_device=__snake_case       ,     offload=__snake_case       ,     weights_map=model.state_dict()       ,     offload_buffers=__snake_case       ,     )

											# Parameters have been offloaded, so on the meta device, buffers included
											self.assertEqual(model.lineara.weight.device       ,     torch.device('meta'       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device('meta'       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('meta'       )       )
											self.assertEqual(model.batchnorm.running_mean.device       ,     torch.device('meta'       )       )

											_snake_case:   List[Any]  					=		torch.randn(2       ,     3       )
											_snake_case:   List[str]  					=		model(__snake_case       )
											self.assertEqual(output.device       ,     __snake_case       )

											# Removing hooks loads back the weights in the model.
											remove_hook_from_submodules(__snake_case       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.batchnorm.weight.device       ,     torch.device('cpu'       )       )
											self.assertEqual(model.lineara.weight.device       ,     torch.device('cpu'       )       )





