




import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
	import tensorflow as tf
	from tensorflow.python.eager import context
	from tensorflow.python.framework import ops

	from transformers import GradientAccumulator, create_optimizer


@require_tf
class    SCREAMING_SNAKE_CASE_				(       unittest.TestCase   ):



	"""simple docstring"""


	def 					lowerCamelCase__     (					self		:						Union[str, Any]					,     lowerCAmelCase		:						Any					,     lowerCAmelCase		:						Optional[int]					,     lowerCAmelCase		:						Union[str, Any]	)	->       List[Any]:







						"""simple docstring"""







						self.assertEqual(len(lowerCAmelCase	)					,     len(lowerCAmelCase	)	)
						for a, b in zip(lowerCAmelCase					,     lowerCAmelCase	):
											self.assertAlmostEqual(lowerCAmelCase					,     lowerCAmelCase					,     delta=lowerCAmelCase	)






	def 					lowerCamelCase__     (					self		:						Dict	)	->       str:







						"""simple docstring"""







						__UpperCamelCase					:	Optional[Any]             =      GradientAccumulator()
						accumulator([tf.constant([1.0, 2.0]	)]	)
						accumulator([tf.constant([-2.0, 1.0]	)]	)
						accumulator([tf.constant([-1.0, 2.0]	)]	)
						with self.assertRaises(lowerCAmelCase	):
											accumulator([tf.constant([1.0, 1.0]	), tf.constant([2.0, 2.0]	)]	)
						self.assertEqual(accumulator.step					,     3	)
						self.assertEqual(len(accumulator.gradients	)					,     1	)
						self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist()					,     [-2.0, 5.0]					,     tol=1E-2	)
						accumulator.reset()
						self.assertEqual(accumulator.step					,     0	)
						self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist()					,     [0.0, 0.0]					,     tol=1E-2	)






	def 					lowerCamelCase__     (					self		:						List[str]	)	->       Optional[Any]:







						"""simple docstring"""







						__UpperCamelCase					:	int             =      None
						ops.enable_eager_execution_internal()
						__UpperCamelCase					:	List[Any]             =      tf.config.list_physical_devices("""CPU"""	)
						if len(lowerCAmelCase	) == 1:
											tf.config.set_logical_device_configuration(
											    physical_devices[0]					,     [tf.config.LogicalDeviceConfiguration(), tf.config.LogicalDeviceConfiguration()]	)
						__UpperCamelCase					:	List[str]             =      tf.config.list_logical_devices(device_type="""CPU"""	)
						__UpperCamelCase					:	Dict             =      tf.distribute.MirroredStrategy(devices=devices[:2]	)

						with strategy.scope():
											__UpperCamelCase					:	Dict             =      GradientAccumulator()
											__UpperCamelCase					:	Optional[Any]             =      tf.Variable([4.0, 3.0]	)
											__UpperCamelCase      ,			__UpperCamelCase					:	Optional[Any]             =      create_optimizer(5E-5					,     10					,     5	)
											__UpperCamelCase					:	Optional[int]             =      tf.Variable([0.0, 0.0]					,     trainable=lowerCAmelCase	)

						def accumulate_on_replica(lowerCAmelCase		:						int	):
											accumulator([gradient]	)

						def apply_on_replica():
											optimizer.apply_gradients(list(zip(accumulator.gradients					,     [variable]	)	)	)

						@tf.function
						def accumulate(lowerCAmelCase		:						Tuple					,     lowerCAmelCase		:						Tuple	):
											with strategy.scope():
																__UpperCamelCase					:	Union[str, Any]             =      strategy.experimental_local_results(lowerCAmelCase	)
																local_variables[0].assign(lowerCAmelCase	)
																local_variables[1].assign(lowerCAmelCase	)
																strategy.run(lowerCAmelCase					,     args=(gradient_placeholder,)	)

						@tf.function
						def apply_grad():
											with strategy.scope():
																strategy.run(lowerCAmelCase	)

						def _check_local_values(lowerCAmelCase		:						Dict					,     lowerCAmelCase		:						Dict	):
											__UpperCamelCase					:	Optional[Any]             =      strategy.experimental_local_results(accumulator._gradients[0]	)
											self.assertListAlmostEqual(values[0].value()					,     lowerCAmelCase					,     tol=1E-2	)
											self.assertListAlmostEqual(values[1].value()					,     lowerCAmelCase					,     tol=1E-2	)

						accumulate([1.0, 2.0]					,     [-1.0, 1.0]	)
						accumulate([3.0, -1.0]					,     [-1.0, -1.0]	)
						accumulate([-2.0, 2.0]					,     [3.0, -2.0]	)
						self.assertEqual(accumulator.step					,     3	)
						_check_local_values([2.0, 3.0]					,     [1.0, -2.0]	)
						apply_grad()
						self.assertListAlmostEqual(variable.value()					,     [4.0, 3.0]					,     tol=1E-2	)
						accumulator.reset()
						self.assertEqual(accumulator.step					,     0	)
						_check_local_values([0.0, 0.0]					,     [0.0, 0.0]	)












import math
from enum import Enum
from typing import Optional, Union

from torch.optim import Optimizer
from torch.optim.lr_scheduler import LambdaLR

from .utils import logging


a__		     =     logging.get_logger(__name__)


class    SCREAMING_SNAKE_CASE_				(       _UpperCamelCase   ):



	"""simple docstring"""


	__magic_name__   :       Dict      	=		'linear'
	__magic_name__   :       Dict      	=		'cosine'
	__magic_name__   :       Optional[int]      	=		'cosine_with_restarts'
	__magic_name__   :       List[str]      	=		'polynomial'
	__magic_name__   :       Any      	=		'constant'
	__magic_name__   :       Union[str, Any]      	=		'constant_with_warmup'
	__magic_name__   :       str      	=		'piecewise_constant'






def 				A__ (snake_case   :		Optimizer		,	snake_case   :		int = -1				)     ->						Optional[Any]:
					return LambdaLR(snake_case		,	lambda snake_case				: 1		,	last_epoch=snake_case				)





def 				A__ (snake_case   :		Optimizer		,	snake_case   :		int		,	snake_case   :		int = -1				)     ->						List[Any]:

					def lr_lambda(snake_case   :		int				):
										if current_step < num_warmup_steps:
															return float(snake_case				) / float(max(1.0		,	snake_case				)				)
										return 1.0

					return LambdaLR(snake_case		,	snake_case		,	last_epoch=snake_case				)





def 				A__ (snake_case   :		Optimizer		,	snake_case   :		str		,	snake_case   :		int = -1				)     ->						Union[str, Any]:
					__UpperCamelCase					:	Optional[Any]             =      {}
					__UpperCamelCase					:	int             =      step_rules.split(""","""				)
					for rule_str in rule_list[:-1]:
										__UpperCamelCase      ,			__UpperCamelCase					:	Tuple             =      rule_str.split(""":"""				)
										__UpperCamelCase					:	int             =      int(snake_case				)
										__UpperCamelCase					:	Union[str, Any]             =      float(snake_case				)
										__UpperCamelCase					:	Optional[int]             =      value
					__UpperCamelCase					:	Dict             =      float(rule_list[-1]				)

					def create_rules_function(snake_case   :		List[str]		,	snake_case   :		Any				):
										def rule_func(snake_case   :		int				) -> float:
															__UpperCamelCase					:	Any             =      sorted(rules_dict.keys()				)
															for i, sorted_step in enumerate(snake_case				):
																				if steps < sorted_step:
																									return rules_dict[sorted_steps[i]]
															return last_lr_multiple

										return rule_func

					__UpperCamelCase					:	Tuple             =      create_rules_function(snake_case		,	snake_case				)

					return LambdaLR(snake_case		,	snake_case		,	last_epoch=snake_case				)





def 				A__ (snake_case   :		int		,	snake_case   :		Optional[Any]		,	snake_case   :		Union[str, Any]		,	snake_case   :		str=-1				)     ->						str:

					def lr_lambda(snake_case   :		int				):
										if current_step < num_warmup_steps:
															return float(snake_case				) / float(max(1		,	snake_case				)				)
										return max(
										    0.0		,	float(num_training_steps - current_step				) / float(max(1		,	num_training_steps - num_warmup_steps				)				)				)

					return LambdaLR(snake_case		,	snake_case		,	snake_case				)





def 				A__ (snake_case   :		Optimizer		,	snake_case   :		int		,	snake_case   :		int		,	snake_case   :		float = 0.5		,	snake_case   :		int = -1				)     ->						List[str]:

					def lr_lambda(snake_case   :		Dict				):
										if current_step < num_warmup_steps:
															return float(snake_case				) / float(max(1		,	snake_case				)				)
										__UpperCamelCase					:	Dict             =      float(current_step - num_warmup_steps				) / float(max(1		,	num_training_steps - num_warmup_steps				)				)
										return max(0.0		,	0.5 * (1.0 + math.cos(math.pi * float(snake_case				) * 2.0 * progress				))				)

					return LambdaLR(snake_case		,	snake_case		,	snake_case				)





def 				A__ (snake_case   :		Optimizer		,	snake_case   :		int		,	snake_case   :		int		,	snake_case   :		int = 1		,	snake_case   :		int = -1				)     ->						Tuple:

					def lr_lambda(snake_case   :		Optional[int]				):
										if current_step < num_warmup_steps:
															return float(snake_case				) / float(max(1		,	snake_case				)				)
										__UpperCamelCase					:	List[str]             =      float(current_step - num_warmup_steps				) / float(max(1		,	num_training_steps - num_warmup_steps				)				)
										if progress >= 1.0:
															return 0.0
										return max(0.0		,	0.5 * (1.0 + math.cos(math.pi * ((float(snake_case				) * progress) % 1.0)				))				)

					return LambdaLR(snake_case		,	snake_case		,	snake_case				)





def 				A__ (snake_case   :		List[str]		,	snake_case   :		Optional[Any]		,	snake_case   :		Optional[int]		,	snake_case   :		str=1e-7		,	snake_case   :		List[str]=1.0		,	snake_case   :		Dict=-1				)     ->						Tuple:
					__UpperCamelCase					:	Tuple             =      optimizer.defaults["""lr"""]
					if not (lr_init > lr_end):
										raise ValueError(F'''lr_end ({lr_end}) must be be smaller than initial lr ({lr_init})'''				)

					def lr_lambda(snake_case   :		int				):
										if current_step < num_warmup_steps:
															return float(snake_case				) / float(max(1		,	snake_case				)				)
										elif current_step > num_training_steps:
															return lr_end / lr_init  # as LambdaLR multiplies by lr_init
										else:
															__UpperCamelCase					:	List[str]             =      lr_init - lr_end
															__UpperCamelCase					:	Any             =      num_training_steps - num_warmup_steps
															__UpperCamelCase					:	List[str]             =      1 - (current_step - num_warmup_steps) / decay_steps
															__UpperCamelCase					:	List[Any]             =      lr_range * pct_remaining**power + lr_end
															return decay / lr_init  # as LambdaLR multiplies by lr_init

					return LambdaLR(snake_case		,	snake_case		,	snake_case				)


a__		     =     {
    SchedulerType.LINEAR: get_linear_schedule_with_warmup,
    SchedulerType.COSINE: get_cosine_schedule_with_warmup,
    SchedulerType.COSINE_WITH_RESTARTS: get_cosine_with_hard_restarts_schedule_with_warmup,
    SchedulerType.POLYNOMIAL: get_polynomial_decay_schedule_with_warmup,
    SchedulerType.CONSTANT: get_constant_schedule,
    SchedulerType.CONSTANT_WITH_WARMUP: get_constant_schedule_with_warmup,
    SchedulerType.PIECEWISE_CONSTANT: get_piecewise_constant_schedule,
}







def 				A__ (snake_case   :		Union[str, SchedulerType]		,	snake_case   :		Optimizer		,	snake_case   :		Optional[str] = None		,	snake_case   :		Optional[int] = None		,	snake_case   :		Optional[int] = None		,	snake_case   :		int = 1		,	snake_case   :		float = 1.0		,	snake_case   :		int = -1		,	)     ->						Dict:
					__UpperCamelCase					:	List[str]             =      SchedulerType(snake_case				)
					__UpperCamelCase					:	int             =      TYPE_TO_SCHEDULER_FUNCTION[name]
					if name == SchedulerType.CONSTANT:
										return schedule_func(snake_case		,	last_epoch=snake_case				)

					if name == SchedulerType.PIECEWISE_CONSTANT:
										return schedule_func(snake_case		,	step_rules=snake_case		,	last_epoch=snake_case				)

					# All other schedulers require `num_warmup_steps`
					if num_warmup_steps is None:
										raise ValueError(F'''{name} requires `num_warmup_steps`, please provide that argument.'''				)

					if name == SchedulerType.CONSTANT_WITH_WARMUP:
										return schedule_func(snake_case		,	num_warmup_steps=snake_case		,	last_epoch=snake_case				)

					# All other schedulers require `num_training_steps`
					if num_training_steps is None:
										raise ValueError(F'''{name} requires `num_training_steps`, please provide that argument.'''				)

					if name == SchedulerType.COSINE_WITH_RESTARTS:
										return schedule_func(
										    snake_case		,	num_warmup_steps=snake_case		,	num_training_steps=snake_case		,	num_cycles=snake_case		,	last_epoch=snake_case		,	)

					if name == SchedulerType.POLYNOMIAL:
										return schedule_func(
										    snake_case		,	num_warmup_steps=snake_case		,	num_training_steps=snake_case		,	power=snake_case		,	last_epoch=snake_case		,	)

					return schedule_func(
					    snake_case		,	num_warmup_steps=snake_case		,	num_training_steps=snake_case		,	last_epoch=snake_case				)






