


import re
from typing import Callable, List, Optional, Union

import tensorflow as tf


try:
   from tensorflow.keras.optimizers.legacy import Adam
except ImportError:
   from tensorflow.keras.optimizers import Adam


class      __lowerCAmelCase  (			tf.keras.optimizers.schedules.LearningRateSchedule       ):






  """simple docstring"""





  def __init__(  self							:							Union[str, Any]     ,					_lowerCAmelCase							:							float     ,					_lowerCAmelCase							:							Callable     ,					_lowerCAmelCase							:							int     ,					_lowerCAmelCase							:							float = 1.0     ,					_lowerCAmelCase							:							str = None     ,					) ->			str:




     """simple docstring"""
     super().__init__()
     snake_case_            =       initial_learning_rate
     snake_case_            =       warmup_steps
     snake_case_            =       power
     snake_case_            =       decay_schedule_fn
     snake_case_            =       name



  def __call__(  self							:							Any     ,					_lowerCAmelCase							:							int	) ->			Tuple:




     """simple docstring"""
     with tf.name_scope(self.name or "WarmUp"	) as name:
        # Implements polynomial warmup. i.e., if global_step < warmup_steps, the
        # learning rate will be `global_step/num_warmup_steps * init_lr`.
        snake_case_            =       tf.cast(_lowerCAmelCase     ,					tf.floataa	)
        snake_case_            =       tf.cast(self.warmup_steps     ,					tf.floataa	)
        snake_case_            =       global_step_float / warmup_steps_float
        snake_case_            =       self.initial_learning_rate * tf.math.pow(_lowerCAmelCase     ,					self.power	)
        return tf.cond(
            global_step_float < warmup_steps_float     ,					lambda: warmup_learning_rate     ,					lambda: self.decay_schedule_fn(step - self.warmup_steps	)     ,					name=_lowerCAmelCase     ,					)



  def        lowerCAmelCase__	(  self							:							Optional[Any]	) ->			Dict:




     """simple docstring"""
     return {
         "initial_learning_rate": self.initial_learning_rate,
         "decay_schedule_fn": self.decay_schedule_fn,
         "warmup_steps": self.warmup_steps,
         "power": self.power,
         "name": self.name,
     }






def       _lowerCAmelCase    (				lowerCAmelCase_				:float		,    lowerCAmelCase_				:int		,    lowerCAmelCase_				:int		,    lowerCAmelCase_				:float = 0.0		,    lowerCAmelCase_				:float = 0.9		,    lowerCAmelCase_				:float = 0.9_9_9		,    lowerCAmelCase_				:float = 1e-8		,    lowerCAmelCase_				:Optional[float] = None		,    lowerCAmelCase_				:Optional[float] = None		,    lowerCAmelCase_				:float = 0.0		,    lowerCAmelCase_				:float = 1.0		,    lowerCAmelCase_				:Optional[List[str]] = None		,    )->Optional[Any]:
   '''simple docstring'''






   snake_case_            =       tf.keras.optimizers.schedules.PolynomialDecay(
       initial_learning_rate=lowerCAmelCase_		,    decay_steps=num_train_steps - num_warmup_steps		,    end_learning_rate=init_lr * min_lr_ratio		,    power=lowerCAmelCase_		,    )
   if num_warmup_steps:
      snake_case_            =       WarmUp(
          initial_learning_rate=lowerCAmelCase_		,    decay_schedule_fn=lowerCAmelCase_		,    warmup_steps=lowerCAmelCase_		,    )
   if weight_decay_rate > 0.0:
      snake_case_            =       AdamWeightDecay(
          learning_rate=lowerCAmelCase_		,    weight_decay_rate=lowerCAmelCase_		,    beta_a=lowerCAmelCase_		,    beta_a=lowerCAmelCase_		,    epsilon=lowerCAmelCase_		,    clipnorm=lowerCAmelCase_		,    global_clipnorm=lowerCAmelCase_		,    exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"]		,    include_in_weight_decay=lowerCAmelCase_		,    )
   else:
      snake_case_            =       tf.keras.optimizers.Adam(
          learning_rate=lowerCAmelCase_		,    beta_a=lowerCAmelCase_		,    beta_a=lowerCAmelCase_		,    epsilon=lowerCAmelCase_		,    clipnorm=lowerCAmelCase_		,    global_clipnorm=lowerCAmelCase_		,    )
   # We return the optimizer and the LR scheduler in order to better track the
   # evolution of the LR independently of the optimizer.
   return optimizer, lr_schedule


class      __lowerCAmelCase  (			a       ):






  """simple docstring"""





  def __init__(  self							:							Dict     ,					_lowerCAmelCase							:							Union[float, tf.keras.optimizers.schedules.LearningRateSchedule] = 0.001     ,					_lowerCAmelCase							:							float = 0.9     ,					_lowerCAmelCase							:							float = 0.999     ,					_lowerCAmelCase							:							float = 1e-7     ,					_lowerCAmelCase							:							bool = False     ,					_lowerCAmelCase							:							float = 0.0     ,					_lowerCAmelCase							:							Optional[List[str]] = None     ,					_lowerCAmelCase							:							Optional[List[str]] = None     ,					_lowerCAmelCase							:							str = "AdamWeightDecay"     ,					**_lowerCAmelCase							:							Union[str, Any]     ,					) ->			Optional[int]:




     """simple docstring"""
     super().__init__(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase     ,					**_lowerCAmelCase	)
     snake_case_            =       weight_decay_rate
     snake_case_            =       include_in_weight_decay
     snake_case_            =       exclude_from_weight_decay



  @classmethod
  def        lowerCAmelCase__	(  cls							:							Dict     ,					_lowerCAmelCase							:							Optional[Any]	) ->			Tuple:




     """simple docstring"""
     snake_case_            =       {"WarmUp": WarmUp}
     return super(_lowerCAmelCase     ,					cls	).from_config(_lowerCAmelCase     ,					custom_objects=_lowerCAmelCase	)



  def        lowerCAmelCase__	(  self							:							List[Any]     ,					_lowerCAmelCase							:							Union[str, Any]     ,					_lowerCAmelCase							:							int     ,					_lowerCAmelCase							:							Optional[int]	) ->			List[str]:




     """simple docstring"""
     super(_lowerCAmelCase     ,					self	)._prepare_local(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase	)
     snake_case_            =       tf.constant(
         self.weight_decay_rate     ,					name="adam_weight_decay_rate"	)



  def        lowerCAmelCase__	(  self							:							Optional[Any]     ,					_lowerCAmelCase							:							List[Any]     ,					_lowerCAmelCase							:							Optional[Any]     ,					_lowerCAmelCase							:							List[Any]	) ->			int:




     """simple docstring"""
     snake_case_            =       self._do_use_weight_decay(var.name	)
     if do_decay:
        return var.assign_sub(
            learning_rate * var * apply_state[(var.device, var.dtype.base_dtype)]["weight_decay_rate"]     ,					use_locking=self._use_locking     ,					)
     return tf.no_op()



  def        lowerCAmelCase__	(  self							:							List[str]     ,					_lowerCAmelCase							:							str     ,					_lowerCAmelCase							:							Optional[int]=None     ,					**_lowerCAmelCase							:							Dict	) ->			Union[str, Any]:




     """simple docstring"""
     snake_case_     ,     snake_case_            =       list(zip(*_lowerCAmelCase	)	)
     return super(_lowerCAmelCase     ,					self	).apply_gradients(zip(_lowerCAmelCase     ,					_lowerCAmelCase	)     ,					name=_lowerCAmelCase     ,					**_lowerCAmelCase	)



  def        lowerCAmelCase__	(  self							:							List[str]     ,					_lowerCAmelCase							:							Union[str, Any]     ,					_lowerCAmelCase							:							Optional[Any]     ,					_lowerCAmelCase							:							Optional[int]	) ->			int:




     """simple docstring"""
     if apply_state is None:
        return self._decayed_lr_t[var_dtype], {}

     snake_case_            =       apply_state or {}
     snake_case_            =       apply_state.get((var_device, var_dtype)	)
     if coefficients is None:
        snake_case_            =       self._fallback_apply_state(_lowerCAmelCase     ,					_lowerCAmelCase	)
        snake_case_            =       coefficients

     return coefficients["lr_t"], {"apply_state": apply_state}



  def        lowerCAmelCase__	(  self							:							int     ,					_lowerCAmelCase							:							List[Any]     ,					_lowerCAmelCase							:							Optional[int]     ,					_lowerCAmelCase							:							Tuple=None	) ->			List[str]:




     """simple docstring"""
     snake_case_     ,     snake_case_            =       self._get_lr(var.device     ,					var.dtype.base_dtype     ,					_lowerCAmelCase	)
     snake_case_            =       self._decay_weights_op(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase	)
     with tf.control_dependencies([decay]	):
        return super(_lowerCAmelCase     ,					self	)._resource_apply_dense(_lowerCAmelCase     ,					_lowerCAmelCase     ,					**_lowerCAmelCase	)



  def        lowerCAmelCase__	(  self							:							List[Any]     ,					_lowerCAmelCase							:							Any     ,					_lowerCAmelCase							:							str     ,					_lowerCAmelCase							:							Any     ,					_lowerCAmelCase							:							Tuple=None	) ->			List[str]:




     """simple docstring"""
     snake_case_     ,     snake_case_            =       self._get_lr(var.device     ,					var.dtype.base_dtype     ,					_lowerCAmelCase	)
     snake_case_            =       self._decay_weights_op(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase	)
     with tf.control_dependencies([decay]	):
        return super(_lowerCAmelCase     ,					self	)._resource_apply_sparse(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase     ,					**_lowerCAmelCase	)



  def        lowerCAmelCase__	(  self							:							Union[str, Any]	) ->			int:




     """simple docstring"""
     snake_case_            =       super().get_config()
     config.update({"weight_decay_rate": self.weight_decay_rate}	)
     return config



  def        lowerCAmelCase__	(  self							:							Union[str, Any]     ,					_lowerCAmelCase							:							List[str]	) ->			Tuple:




     """simple docstring"""
     if self.weight_decay_rate == 0:
        return False

     if self._include_in_weight_decay:
        for r in self._include_in_weight_decay:
           if re.search(_lowerCAmelCase     ,					_lowerCAmelCase	) is not None:
              return True

     if self._exclude_from_weight_decay:
        for r in self._exclude_from_weight_decay:
           if re.search(_lowerCAmelCase     ,					_lowerCAmelCase	) is not None:
              return False
     return True






class      __lowerCAmelCase  (			a       ):






  """simple docstring"""





  def __init__(  self							:							str	) ->			Dict:




     """simple docstring"""
     snake_case_            =       []
     snake_case_            =       None



  @property
  def        lowerCAmelCase__	(  self							:							Dict	) ->			List[Any]:




     """simple docstring"""
     if self._accum_steps is None:
        snake_case_            =       tf.Variable(
            tf.constant(0     ,					dtype=tf.intaa	)     ,					trainable=_lowerCAmelCase     ,					synchronization=tf.VariableSynchronization.ON_READ     ,					aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA     ,					)

     return self._accum_steps.value()



  @property
  def        lowerCAmelCase__	(  self							:							Tuple	) ->			List[str]:




     """simple docstring"""
     if not self._gradients:
        raise ValueError("The accumulator should be called first to initialize the gradients"	)
     return [gradient.value() if gradient is not None else gradient for gradient in self._gradients]



  def __call__(  self							:							List[Any]     ,					_lowerCAmelCase							:							Union[str, Any]	) ->			Optional[Any]:




     """simple docstring"""
     if not self._gradients:
        snake_case_            =       self.step  # Create the step variable.
        self._gradients.extend(
            [
                tf.Variable(
                    tf.zeros_like(_lowerCAmelCase	)     ,					trainable=_lowerCAmelCase     ,					synchronization=tf.VariableSynchronization.ON_READ     ,					aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA     ,					)
                if gradient is not None
                else gradient
                for gradient in gradients
            ]	)
     if len(_lowerCAmelCase	) != len(self._gradients	):
        raise ValueError(F'''Expected {len(self._gradients	)} gradients, but got {len(_lowerCAmelCase	)}'''	)

     for accum_gradient, gradient in zip(self._gradients     ,					_lowerCAmelCase	):
        if accum_gradient is not None and gradient is not None:
           accum_gradient.assign_add(_lowerCAmelCase	)

     self._accum_steps.assign_add(1	)



  def        lowerCAmelCase__	(  self							:							Union[str, Any]	) ->			Optional[Any]:




     """simple docstring"""
     if not self._gradients:
        return
     self._accum_steps.assign(0	)
     for gradient in self._gradients:
        if gradient is not None:
           gradient.assign(tf.zeros_like(_lowerCAmelCase	)	)








import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
   import torch_xla.core.xla_model as xm
   import torch_xla.debug.metrics as met


class      __lowerCAmelCase  (			a       ):






  """simple docstring"""





  def __init__(  self							:							Union[str, Any]     ,					*_lowerCAmelCase							:							Dict     ,					_lowerCAmelCase							:							List[Any]=None     ,					_lowerCAmelCase							:							str=None     ,					**_lowerCAmelCase							:							str	) ->			Dict:




     """simple docstring"""
     super().__init__(*_lowerCAmelCase     ,					**_lowerCAmelCase	)
     snake_case_            =       eval_examples
     snake_case_            =       post_process_function



  def        lowerCAmelCase__	(  self							:							Union[str, Any]     ,					_lowerCAmelCase							:							Optional[Dataset] = None     ,					_lowerCAmelCase							:							Any=None     ,					_lowerCAmelCase							:							Optional[List[str]] = None     ,					_lowerCAmelCase							:							str = "eval"     ,					**_lowerCAmelCase							:							Tuple     ,					) ->			Dict[str, float]:




     """simple docstring"""
     snake_case_            =       gen_kwargs.copy()
     snake_case_            =       (
         gen_kwargs["max_length"] if gen_kwargs.get("max_length"	) is not None else self.args.generation_max_length
     )
     snake_case_            =       (
         gen_kwargs["num_beams"] if gen_kwargs.get("num_beams"	) is not None else self.args.generation_num_beams
     )
     snake_case_            =       gen_kwargs

     snake_case_            =       self.eval_dataset if eval_dataset is None else eval_dataset
     snake_case_            =       self.get_eval_dataloader(_lowerCAmelCase	)
     snake_case_            =       self.eval_examples if eval_examples is None else eval_examples

     # Temporarily disable metric computation, we will do it in the loop here.
     snake_case_            =       self.compute_metrics
     snake_case_            =       None
     snake_case_            =       time.time()
     snake_case_            =       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
     try:
        snake_case_            =       eval_loop(
            _lowerCAmelCase     ,					description="Evaluation"     ,					prediction_loss_only=True if compute_metrics is None else None     ,					ignore_keys=_lowerCAmelCase     ,					metric_key_prefix=_lowerCAmelCase     ,					)
     finally:
        snake_case_            =       compute_metrics
     snake_case_            =       self.args.eval_batch_size * self.args.world_size
     if F'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
        start_time += output.metrics[F'''{metric_key_prefix}_jit_compilation_time''']
     output.metrics.update(
         speed_metrics(
             _lowerCAmelCase     ,					_lowerCAmelCase     ,					num_samples=output.num_samples     ,					num_steps=math.ceil(output.num_samples / total_batch_size	)     ,					)	)

     if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
        # Only the main node write the results by default
        snake_case_            =       self.post_process_function(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase	)
        snake_case_            =       self.compute_metrics(_lowerCAmelCase	)

        # Prefix all keys with metric_key_prefix + '_'
        for key in list(metrics.keys()	):
           if not key.startswith(F'''{metric_key_prefix}_'''	):
              snake_case_            =       metrics.pop(_lowerCAmelCase	)

        metrics.update(output.metrics	)
     else:
        snake_case_            =       output.metrics

     if self.args.should_log:
        # Only the main node log the results by default
        self.log(_lowerCAmelCase	)

     if self.args.tpu_metrics_debug or self.args.debug:
        # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
        xm.master_print(met.metrics_report()	)

     snake_case_            =       self.callback_handler.on_evaluate(self.args     ,					self.state     ,					self.control     ,					_lowerCAmelCase	)
     return metrics



  def        lowerCAmelCase__	(  self							:							Optional[Any]     ,					_lowerCAmelCase							:							Optional[Any]     ,					_lowerCAmelCase							:							Union[str, Any]     ,					_lowerCAmelCase							:							List[str]=None     ,					_lowerCAmelCase							:							str = "test"     ,					**_lowerCAmelCase							:							Optional[Any]	) ->			Union[str, Any]:




     """simple docstring"""
     snake_case_            =       gen_kwargs.copy()

     snake_case_            =       self.get_test_dataloader(_lowerCAmelCase	)

     # Temporarily disable metric computation, we will do it in the loop here.
     snake_case_            =       self.compute_metrics
     snake_case_            =       None
     snake_case_            =       time.time()
     snake_case_            =       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
     try:
        snake_case_            =       eval_loop(
            _lowerCAmelCase     ,					description="Prediction"     ,					prediction_loss_only=True if compute_metrics is None else None     ,					ignore_keys=_lowerCAmelCase     ,					metric_key_prefix=_lowerCAmelCase     ,					)
     finally:
        snake_case_            =       compute_metrics

     snake_case_            =       self.args.eval_batch_size * self.args.world_size
     if F'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
        start_time += output.metrics[F'''{metric_key_prefix}_jit_compilation_time''']
     output.metrics.update(
         speed_metrics(
             _lowerCAmelCase     ,					_lowerCAmelCase     ,					num_samples=output.num_samples     ,					num_steps=math.ceil(output.num_samples / total_batch_size	)     ,					)	)
     if self.post_process_function is None or self.compute_metrics is None:
        return output

     snake_case_            =       self.post_process_function(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase     ,					"predict"	)
     snake_case_            =       self.compute_metrics(_lowerCAmelCase	)

     # Prefix all keys with metric_key_prefix + '_'
     for key in list(metrics.keys()	):
        if not key.startswith(F'''{metric_key_prefix}_'''	):
           snake_case_            =       metrics.pop(_lowerCAmelCase	)
     metrics.update(output.metrics	)
     return PredictionOutput(predictions=predictions.predictions     ,					label_ids=predictions.label_ids     ,					metrics=_lowerCAmelCase	)




