




import inspect
import warnings

import torch

from .state import AcceleratorState, GradientState
from .utils import DistributedType, honor_type, is_tpu_available


if is_tpu_available(check_device=False):
				import torch_xla.core.xla_model as xm







def  lowerCAmelCase  (	UpperCAmelCase,	UpperCAmelCase       )  ->Optional[int]:






					"""simple docstring"""






					if isinstance(UpperCAmelCase,	(list, tuple)       ):
										return honor_type(UpperCAmelCase,	(move_to_device(UpperCAmelCase,	UpperCAmelCase       ) for t in state)       )
					elif isinstance(UpperCAmelCase,	UpperCAmelCase       ):
										return type(UpperCAmelCase       )({k: move_to_device(UpperCAmelCase,	UpperCAmelCase       ) for k, v in state.items()}       )
					elif isinstance(UpperCAmelCase,	torch.Tensor       ):
										return state.to(UpperCAmelCase       )
					return state





class 		A__				(   torch.optim.Optimizer ):






	def __init__(	self				,     lowerCamelCase				,     lowerCamelCase=True				,     lowerCamelCase=None						)      ->   str:




						"""simple docstring"""






						__magic_name__					:							Tuple		   =							optimizer
						__magic_name__					:							Union[str, Any]		   =							scaler
						__magic_name__					:							Union[str, Any]		   =							AcceleratorState()
						__magic_name__					:							str		   =							GradientState()
						__magic_name__					:							Any		   =							device_placement
						__magic_name__					:							Union[str, Any]		   =							False
						__magic_name__					:							Optional[Any]		   =							None

						# Handle device placement
						if device_placement:
											__magic_name__					:							List[Any]		   =							self.optimizer.state_dict()
											if self.accelerator_state.distributed_type == DistributedType.TPU:
																xm.send_cpu_data_to_device(lowerCamelCase				,     self.accelerator_state.device						)
											else:
																__magic_name__					:							Optional[Any]		   =							move_to_device(lowerCamelCase				,     self.accelerator_state.device						)
											self.optimizer.load_state_dict(lowerCamelCase						)






	@property
	def  lowercase      (	self						)      ->   int:




						"""simple docstring"""






						return self.optimizer.state






	@state.setter
	def  lowercase      (	self				,     lowerCamelCase						)      ->   List[Any]:




						"""simple docstring"""






						__magic_name__					:							Any		   =							state






	@property
	def  lowercase      (	self						)      ->   Tuple:




						"""simple docstring"""






						return self.optimizer.param_groups






	@param_groups.setter
	def  lowercase      (	self				,     lowerCamelCase						)      ->   Optional[int]:




						"""simple docstring"""






						__magic_name__					:							Optional[Any]		   =							param_groups






	@property
	def  lowercase      (	self						)      ->   Dict:




						"""simple docstring"""






						return self.optimizer.defaults






	@defaults.setter
	def  lowercase      (	self				,     lowerCamelCase						)      ->   Dict:




						"""simple docstring"""






						__magic_name__					:							Any		   =							defaults






	def  lowercase      (	self				,     lowerCamelCase						)      ->   str:




						"""simple docstring"""






						self.optimizer.add_param_group(lowerCamelCase						)






	def  lowercase      (	self				,     lowerCamelCase						)      ->   Tuple:




						"""simple docstring"""






						if self.accelerator_state.distributed_type == DistributedType.TPU and self.device_placement:
											xm.send_cpu_data_to_device(lowerCamelCase				,     self.accelerator_state.device						)
						self.optimizer.load_state_dict(lowerCamelCase						)






	def  lowercase      (	self						)      ->   Optional[int]:




						"""simple docstring"""






						return self.optimizer.state_dict()






	def  lowercase      (	self				,     lowerCamelCase=None						)      ->   Union[str, Any]:




						"""simple docstring"""






						if self.gradient_state.sync_gradients:
											__magic_name__					:							Optional[int]		   =							'''set_to_none''' in inspect.signature(self.optimizer.zero_grad						).parameters
											if accept_arg:
																if set_to_none is None:
																					__magic_name__					:							Union[str, Any]		   =							False
																self.optimizer.zero_grad(set_to_none=lowerCamelCase						)
											else:
																if set_to_none is not None:
																					raise ValueError('''`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer.'''						)
																self.optimizer.zero_grad()






	def  lowercase      (	self				,     lowerCamelCase=None						)      ->   str:




						"""simple docstring"""






						if self.gradient_state.sync_gradients:
											if self.accelerator_state.distributed_type == DistributedType.TPU:
																__magic_name__					:							Tuple		   =							{'''closure''': closure} if closure is not None else {}
																xm.optimizer_step(self.optimizer				,     optimizer_args=lowerCamelCase						)
											elif self.scaler is not None:
																__magic_name__					:							Optional[Any]		   =							False
																if self._last_scale is None:
																					# `get_scale` is an async operation requiring full synchronization
																					# on CPU and GPUs before finishing. As a result, we store away
																					# the prior one to reduce the call overhead
																					__magic_name__					:							Any		   =							self.scaler.get_scale()
																					__magic_name__					:							Tuple		   =							True
																self.scaler.step(self.optimizer				,     lowerCamelCase						)
																self.scaler.update()
																__magic_name__					:							Any		   =							self.scaler.get_scale()
																if not new_scale:
																					# If we reduced the loss scale, it means the optimizer step was skipped because of gradient overflow.
																					__magic_name__					:							Tuple		   =							scale_after < self._last_scale
																__magic_name__					:							Union[str, Any]		   =							scale_after

											else:
																self.optimizer.step(lowerCamelCase						)






	def  lowercase      (	self				,     lowerCamelCase						)      ->   Union[str, Any]:




						"""simple docstring"""






						for param_group in self.optimizer.param_groups:
											__magic_name__					:							List[Any]		   =							[parameters_map.get(lowerCamelCase				,     lowerCamelCase						) for p in param_group['''params''']]






	@property
	def  lowercase      (	self						)      ->   Tuple:




						"""simple docstring"""






						warnings.warn(
						    '''The `is_overflow` property is deprecated and will be removed in version 1.0 of Accelerate use '''
						    '''`optimizer.step_was_skipped` instead.'''				,     lowerCamelCase				,     )
						return self._is_overflow






	@property
	def  lowercase      (	self						)      ->   Union[str, Any]:




						"""simple docstring"""






						return self._is_overflow






	def __getstate__(	self						)      ->   Any:




						"""simple docstring"""






						return self.__dict__.copy()






	def __setstate__(	self				,     lowerCamelCase						)      ->   str:




						"""simple docstring"""






						self.__dict__.update(lowerCamelCase						)









import unittest

import numpy as np

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
				import tensorflow as tf

				from transformers.activations_tf import get_tf_activation





@require_tf
class 		A__				(   unittest.TestCase ):






	def  lowercase      (	self						)      ->   Dict:




						"""simple docstring"""






						__magic_name__					:							int		   =							tf.constant([-100, -1.0, -0.1, 0, 0.1, 1.0, 1_0_0.0]						)
						__magic_name__					:							List[str]		   =							get_tf_activation('''gelu'''						)
						__magic_name__					:							str		   =							get_tf_activation('''gelu_10'''						)

						__magic_name__					:							List[str]		   =							gelu(lowerCamelCase						)
						__magic_name__					:							Optional[Any]		   =							geluaa(lowerCamelCase						)

						__magic_name__					:							Optional[int]		   =							tf.where(y_gelu_aa < 1_0.0				,     1.0				,     0.0						)

						self.assertEqual(tf.math.reduce_max(lowerCamelCase						).numpy().item()				,     1_0.0						)
						self.assertTrue(np.allclose(y_gelu * clipped_mask				,     y_gelu_aa * clipped_mask						)						)






	def  lowercase      (	self						)      ->   Dict:




						"""simple docstring"""






						get_tf_activation('''gelu'''						)
						get_tf_activation('''gelu_10'''						)
						get_tf_activation('''gelu_fast'''						)
						get_tf_activation('''gelu_new'''						)
						get_tf_activation('''glu'''						)
						get_tf_activation('''mish'''						)
						get_tf_activation('''quick_gelu'''						)
						get_tf_activation('''relu'''						)
						get_tf_activation('''sigmoid'''						)
						get_tf_activation('''silu'''						)
						get_tf_activation('''swish'''						)
						get_tf_activation('''tanh'''						)
						with self.assertRaises(lowerCamelCase						):
											get_tf_activation('''bogus'''						)
						with self.assertRaises(lowerCamelCase						):
											get_tf_activation(lowerCamelCase						)



