





import re
from typing import Callable, List, Optional, Union

import tensorflow as tf


try:
					from tensorflow.keras.optimizers.legacy import Adam
except ImportError:
					from tensorflow.keras.optimizers import Adam


class   A      (tf.keras.optimizers.schedules.LearningRateSchedule   ):
			'''simple docstring'''




			def __init__(      self       :   Dict							,     __lowerCAmelCase       :   Dict							,     __lowerCAmelCase       :   int							,     __lowerCAmelCase       :   Tuple							,     __lowerCAmelCase       :   Optional[int] = 1.0							,     __lowerCAmelCase       :   Optional[Any] = None							,     )     ->     Optional[int]:







					"""simple docstring"""






					super().__init__()
					A__  				=       initial_learning_rate
					A__  				=       warmup_steps
					A__  				=       power
					A__  				=       decay_schedule_fn
					A__  				=       name






			def __call__(      self       :   List[Any]							,     __lowerCAmelCase       :   Optional[Any]       )     ->     Tuple:







					"""simple docstring"""






					with tf.name_scope(self.name or """WarmUp"""       ) as name:
							# Implements polynomial warmup. i.e., if global_step < warmup_steps, the
							# learning rate will be `global_step/num_warmup_steps * init_lr`.
							A__  				=       tf.cast(UpperCamelCase__							,     tf.floataa       )
							A__  				=       tf.cast(self.warmup_steps							,     tf.floataa       )
							A__  				=       global_step_float / warmup_steps_float
							A__  				=       self.initial_learning_rate * tf.math.pow(UpperCamelCase__							,     self.power       )
							return tf.cond(
							    global_step_float < warmup_steps_float							,     lambda: warmup_learning_rate							,     lambda: self.decay_schedule_fn(step - self.warmup_steps       )							,     name=UpperCamelCase__							,     )







			def        a_	(      self       :   Optional[int]       )     ->     Union[str, Any]:







					"""simple docstring"""






					return {
					    "initial_learning_rate": self.initial_learning_rate,
					    "decay_schedule_fn": self.decay_schedule_fn,
					    "warmup_steps": self.warmup_steps,
					    "power": self.power,
					    "name": self.name,
					}







def        __lowerCamelCase (							__a       :Optional[Any]   ,					__a       :Tuple   ,					__a       :Union[str, Any]   ,					__a       :Union[str, Any] = 0.0   ,					__a       :str = 0.9   ,					__a       :Any = 0.999   ,					__a       :List[Any] = 1E-8   ,					__a       :Optional[int] = None   ,					__a       :List[Any] = None   ,					__a       :List[Any] = 0.0   ,					__a       :List[str] = 1.0   ,					__a       :List[Any] = None   ,					)		->    Any:






		"""simple docstring"""

		A__  				=       tf.keras.optimizers.schedules.PolynomialDecay(
		    initial_learning_rate=UpperCAmelCase__   ,					decay_steps=num_train_steps - num_warmup_steps   ,					end_learning_rate=init_lr * min_lr_ratio   ,					power=UpperCAmelCase__   ,					)
		if num_warmup_steps:
				A__  				=       WarmUp(
				    initial_learning_rate=UpperCAmelCase__   ,					decay_schedule_fn=UpperCAmelCase__   ,					warmup_steps=UpperCAmelCase__   ,					)
		if weight_decay_rate > 0.0:
				A__  				=       AdamWeightDecay(
				    learning_rate=UpperCAmelCase__   ,					weight_decay_rate=UpperCAmelCase__   ,					beta_a=UpperCAmelCase__   ,					beta_a=UpperCAmelCase__   ,					epsilon=UpperCAmelCase__   ,					clipnorm=UpperCAmelCase__   ,					global_clipnorm=UpperCAmelCase__   ,					exclude_from_weight_decay=["""LayerNorm""", """layer_norm""", """bias"""]   ,					include_in_weight_decay=UpperCAmelCase__   ,					)
		else:
				A__  				=       tf.keras.optimizers.Adam(
				    learning_rate=UpperCAmelCase__   ,					beta_a=UpperCAmelCase__   ,					beta_a=UpperCAmelCase__   ,					epsilon=UpperCAmelCase__   ,					clipnorm=UpperCAmelCase__   ,					global_clipnorm=UpperCAmelCase__   ,					)
		# We return the optimizer and the LR scheduler in order to better track the
		# evolution of the LR independently of the optimizer.
		return optimizer, lr_schedule


class   A      (_snake_case   ):
			'''simple docstring'''




			def __init__(      self       :   Union[str, Any]							,     __lowerCAmelCase       :   Optional[Any] = 0.0_0_1							,     __lowerCAmelCase       :   int = 0.9							,     __lowerCAmelCase       :   Any = 0.9_9_9							,     __lowerCAmelCase       :   List[Any] = 1e-7							,     __lowerCAmelCase       :   Optional[Any] = False							,     __lowerCAmelCase       :   Union[str, Any] = 0.0							,     __lowerCAmelCase       :   str = None							,     __lowerCAmelCase       :   List[Any] = None							,     __lowerCAmelCase       :   Dict = "AdamWeightDecay"							,     **__lowerCAmelCase       :   List[str]							,     )     ->     Optional[Any]:







					"""simple docstring"""






					super().__init__(UpperCamelCase__							,     UpperCamelCase__							,     UpperCamelCase__							,     UpperCamelCase__							,     UpperCamelCase__							,     UpperCamelCase__							,     **UpperCamelCase__       )
					A__  				=       weight_decay_rate
					A__  				=       include_in_weight_decay
					A__  				=       exclude_from_weight_decay






			@classmethod
			def        a_	(      cls       :   Any							,     __lowerCAmelCase       :   Any       )     ->     Optional[Any]:







					"""simple docstring"""






					A__  				=       {"""WarmUp""": WarmUp}
					return super(UpperCamelCase__							,     cls       ).from_config(UpperCamelCase__							,     custom_objects=UpperCamelCase__       )






			def        a_	(      self       :   Optional[int]							,     __lowerCAmelCase       :   Dict							,     __lowerCAmelCase       :   Union[str, Any]							,     __lowerCAmelCase       :   Any       )     ->     Tuple:







					"""simple docstring"""






					super(UpperCamelCase__							,     self       )._prepare_local(UpperCamelCase__							,     UpperCamelCase__							,     UpperCamelCase__       )
					A__  				=       tf.constant(
					    self.weight_decay_rate							,     name="""adam_weight_decay_rate"""       )






			def        a_	(      self       :   str							,     __lowerCAmelCase       :   Union[str, Any]							,     __lowerCAmelCase       :   Any							,     __lowerCAmelCase       :   int       )     ->     Union[str, Any]:







					"""simple docstring"""






					A__  				=       self._do_use_weight_decay(var.name       )
					if do_decay:
							return var.assign_sub(
							    learning_rate * var * apply_state[(var.device, var.dtype.base_dtype)]["""weight_decay_rate"""]							,     use_locking=self._use_locking							,     )
					return tf.no_op()






			def        a_	(      self       :   Union[str, Any]							,     __lowerCAmelCase       :   Optional[Any]							,     __lowerCAmelCase       :   Any=None							,     **__lowerCAmelCase       :   Union[str, Any]       )     ->     Optional[int]:







					"""simple docstring"""






					A__				,      A__  				=       list(zip(*UpperCamelCase__       )       )
					return super(UpperCamelCase__							,     self       ).apply_gradients(zip(UpperCamelCase__							,     UpperCamelCase__       )							,     name=UpperCamelCase__							,     **UpperCamelCase__       )






			def        a_	(      self       :   Optional[Any]							,     __lowerCAmelCase       :   List[Any]							,     __lowerCAmelCase       :   Union[str, Any]							,     __lowerCAmelCase       :   List[str]       )     ->     int:







					"""simple docstring"""






					if apply_state is None:
							return self._decayed_lr_t[var_dtype], {}

					A__  				=       apply_state or {}
					A__  				=       apply_state.get((var_device, var_dtype)       )
					if coefficients is None:
							A__  				=       self._fallback_apply_state(UpperCamelCase__							,     UpperCamelCase__       )
							A__  				=       coefficients

					return coefficients["lr_t"], {"apply_state": apply_state}






			def        a_	(      self       :   Optional[Any]							,     __lowerCAmelCase       :   Any							,     __lowerCAmelCase       :   Optional[int]							,     __lowerCAmelCase       :   List[str]=None       )     ->     List[Any]:







					"""simple docstring"""






					A__				,      A__  				=       self._get_lr(var.device							,     var.dtype.base_dtype							,     UpperCamelCase__       )
					A__  				=       self._decay_weights_op(UpperCamelCase__							,     UpperCamelCase__							,     UpperCamelCase__       )
					with tf.control_dependencies([decay]       ):
							return super(UpperCamelCase__							,     self       )._resource_apply_dense(UpperCamelCase__							,     UpperCamelCase__							,     **UpperCamelCase__       )






			def        a_	(      self       :   Dict							,     __lowerCAmelCase       :   Union[str, Any]							,     __lowerCAmelCase       :   List[Any]							,     __lowerCAmelCase       :   str							,     __lowerCAmelCase       :   List[str]=None       )     ->     Any:







					"""simple docstring"""






					A__				,      A__  				=       self._get_lr(var.device							,     var.dtype.base_dtype							,     UpperCamelCase__       )
					A__  				=       self._decay_weights_op(UpperCamelCase__							,     UpperCamelCase__							,     UpperCamelCase__       )
					with tf.control_dependencies([decay]       ):
							return super(UpperCamelCase__							,     self       )._resource_apply_sparse(UpperCamelCase__							,     UpperCamelCase__							,     UpperCamelCase__							,     **UpperCamelCase__       )






			def        a_	(      self       :   str       )     ->     List[str]:







					"""simple docstring"""






					A__  				=       super().get_config()
					config.update({"""weight_decay_rate""": self.weight_decay_rate}       )
					return config







			def        a_	(      self       :   Optional[Any]							,     __lowerCAmelCase       :   Optional[Any]       )     ->     Any:







					"""simple docstring"""






					if self.weight_decay_rate == 0:
							return False

					if self._include_in_weight_decay:
							for r in self._include_in_weight_decay:
									if re.search(UpperCamelCase__							,     UpperCamelCase__       ) is not None:
											return True

					if self._exclude_from_weight_decay:
							for r in self._exclude_from_weight_decay:
									if re.search(UpperCamelCase__							,     UpperCamelCase__       ) is not None:
											return False
					return True


class   A      (_snake_case   ):
			'''simple docstring'''




			def __init__(      self       :   Union[str, Any]       )     ->     int:







					"""simple docstring"""






					A__  				=       []
					A__  				=       None






			@property
			def        a_	(      self       :   Optional[Any]       )     ->     Optional[Any]:







					"""simple docstring"""






					if self._accum_steps is None:
							A__  				=       tf.Variable(
							    tf.constant(0							,     dtype=tf.intaa       )							,     trainable=UpperCamelCase__							,     synchronization=tf.VariableSynchronization.ON_READ							,     aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA							,     )

					return self._accum_steps.value()






			@property
			def        a_	(      self       :   List[str]       )     ->     Any:







					"""simple docstring"""






					if not self._gradients:
							raise ValueError("""The accumulator should be called first to initialize the gradients"""       )
					return [gradient.value() if gradient is not None else gradient for gradient in self._gradients]






			def __call__(      self       :   Optional[Any]							,     __lowerCAmelCase       :   Optional[int]       )     ->     Dict:







					"""simple docstring"""






					if not self._gradients:
							A__  				=       self.step  # Create the step variable.
							self._gradients.extend(
							    [
							        tf.Variable(
							            tf.zeros_like(UpperCamelCase__       )							,     trainable=UpperCamelCase__							,     synchronization=tf.VariableSynchronization.ON_READ							,     aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA							,     )
							        if gradient is not None
							        else gradient
							        for gradient in gradients
							    ]       )
					if len(UpperCamelCase__       ) != len(self._gradients       ):
							raise ValueError(f'Expected {len(self._gradients       )} gradients, but got {len(UpperCamelCase__       )}'       )

					for accum_gradient, gradient in zip(self._gradients							,     UpperCamelCase__       ):
							if accum_gradient is not None and gradient is not None:
									accum_gradient.assign_add(UpperCamelCase__       )

					self._accum_steps.assign_add(1       )







			def        a_	(      self       :   int       )     ->     List[str]:







					"""simple docstring"""






					if not self._gradients:
							return
					self._accum_steps.assign(0       )
					for gradient in self._gradients:
							if gradient is not None:
									gradient.assign(tf.zeros_like(UpperCamelCase__       )       )










from typing import TYPE_CHECKING

from ...utils import _LazyModule


A   : Optional[Any]			=				{'''processing_wav2vec2_with_lm''': ['''Wav2Vec2ProcessorWithLM''']}


if TYPE_CHECKING:
					from .processing_wavaveca_with_lm import WavaVecaProcessorWithLM
else:
					import sys

					A   : List[str]			=				_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



