

import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


_UpperCAmelCase     						=      logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
  import torch_xla.core.xla_model as xm
  import torch_xla.debug.metrics as met



class 							_UpperCAmelCase			( __lowercase      ):
  '''simple docstring'''







  def __init__(			self						:			List[str]		,    *UpperCamelCase__						:			str		,    UpperCamelCase__						:			Tuple=None		,    UpperCamelCase__						:			Tuple=None		,    UpperCamelCase__						:			int=None		,    **UpperCamelCase__						:			int			):
        super().__init__(*__lowerCAmelCase		,    **__lowerCAmelCase			)
        A 			=					eval_examples
        A 			=					post_process_function
        A 			=					quant_trainer_args
        A 			=					128  # default number of calibration samples







  def 						UpperCamelCase							(			self						:			int		,    UpperCamelCase__						:			Union[str, Any]=None			):
        if calib_dataset is None and self.calib_dataset is None:
              raise ValueError('Trainer: calibration requires an calib_dataset.'			)
        A 			=					calib_dataset if calib_dataset is not None else self.calib_dataset

        A 			=					self._remove_unused_columns(__lowerCAmelCase		,    description='Calibration'			)

        return DataLoader(
            __lowerCAmelCase		,    batch_size=self.args.eval_batch_size		,    collate_fn=self.data_collator		,    drop_last=self.args.dataloader_drop_last		,    num_workers=self.args.dataloader_num_workers		,    pin_memory=self.args.dataloader_pin_memory		,    shuffle=__lowerCAmelCase		,    )







  def 						UpperCamelCase							(			self						:			int		,    UpperCamelCase__						:			Dict=None			):
        A 			=					self.train_dataset if calib_dataset is None else calib_dataset
        A 			=					self.get_calib_dataloader(__lowerCAmelCase			)

        A 			=					self.model
        quant_trainer.configure_model(__lowerCAmelCase		,    self.quant_trainer_args		,    calib=__lowerCAmelCase			)
        model.eval()
        quant_trainer.enable_calibration(__lowerCAmelCase			)

        logger.info('***** Running calibration *****'			)
        logger.info(f'''  Num examples = {self.calib_num}'''			)
        logger.info(f'''  Batch size = {calib_dataloader.batch_size}'''			)

        for step, inputs in enumerate(__lowerCAmelCase			):
              # Prediction step
              A 			=					self.prediction_step(__lowerCAmelCase		,    __lowerCAmelCase		,    prediction_loss_only=__lowerCAmelCase			)
              if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
                    break

        quant_trainer.finish_calibration(__lowerCAmelCase		,    self.quant_trainer_args			)
        A 			=					model







  def 						UpperCamelCase							(			self						:			List[Any]		,    UpperCamelCase__						:			Optional[Any]=None		,    UpperCamelCase__						:			Optional[int]=None		,    UpperCamelCase__						:			List[str]=None		,    UpperCamelCase__						:			int = "eval"			):
        A 			=					self.eval_dataset if eval_dataset is None else eval_dataset
        A 			=					self.get_eval_dataloader(__lowerCAmelCase			)
        A 			=					self.eval_examples if eval_examples is None else eval_examples

        # Temporarily disable metric computation, we will do it in the loop here.
        A 			=					self.compute_metrics
        A 			=					None
        A 			=					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
        try:
              A 			=					eval_loop(
                  __lowerCAmelCase		,    description='Evaluation'		,    prediction_loss_only=True if compute_metrics is None else None		,    ignore_keys=__lowerCAmelCase		,    )
        finally:
              A 			=					compute_metrics

        if self.post_process_function is not None and self.compute_metrics is not None:
              A 			=					self.post_process_function(__lowerCAmelCase		,    __lowerCAmelCase		,    output.predictions			)
              A 			=					self.compute_metrics(__lowerCAmelCase			)

              # Prefix all keys with metric_key_prefix + '_'
              for key in list(metrics.keys()			):
                    if not key.startswith(f'''{metric_key_prefix}_'''			):
                          A 			=					metrics.pop(__lowerCAmelCase			)

              self.log(__lowerCAmelCase			)
        else:
              A 			=					{}

        if self.args.tpu_metrics_debug or self.args.debug:
              # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
              xm.master_print(met.metrics_report()			)

        A 			=					self.callback_handler.on_evaluate(self.args		,    self.state		,    self.control		,    __lowerCAmelCase			)
        return metrics







  def 						UpperCamelCase							(			self						:			List[Any]		,    UpperCamelCase__						:			Dict		,    UpperCamelCase__						:			str		,    UpperCamelCase__						:			List[Any]=None		,    UpperCamelCase__						:			int = "test"			):
        A 			=					self.get_test_dataloader(__lowerCAmelCase			)

        # Temporarily disable metric computation, we will do it in the loop here.
        A 			=					self.compute_metrics
        A 			=					None
        A 			=					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
        try:
              A 			=					eval_loop(
                  __lowerCAmelCase		,    description='Prediction'		,    prediction_loss_only=True if compute_metrics is None else None		,    ignore_keys=__lowerCAmelCase		,    )
        finally:
              A 			=					compute_metrics

        if self.post_process_function is None or self.compute_metrics is None:
              return output

        A 			=					self.post_process_function(__lowerCAmelCase		,    __lowerCAmelCase		,    output.predictions		,    'predict'			)
        A 			=					self.compute_metrics(__lowerCAmelCase			)

        # Prefix all keys with metric_key_prefix + '_'
        for key in list(metrics.keys()			):
              if not key.startswith(f'''{metric_key_prefix}_'''			):
                    A 			=					metrics.pop(__lowerCAmelCase			)

        return PredictionOutput(predictions=predictions.predictions		,    label_ids=predictions.label_ids		,    metrics=__lowerCAmelCase			)







  def 						UpperCamelCase							(			self						:			Any		,    UpperCamelCase__						:			Any="./"			):
        A 			=					self.eval_dataset
        A 			=					self.get_eval_dataloader(__lowerCAmelCase			)

        A 			=					next(iter(__lowerCAmelCase			)			)

        # saving device - to make it consistent
        A 			=					torch.device('cuda' if torch.cuda.is_available() else 'cpu'			)

        # convert to tuple
        A 			=					tuple(v.to(__lowerCAmelCase			) for k, v in batch.items()			)

        logger.info('Converting model to be onnx compatible'			)
        from pytorch_quantization.nn import TensorQuantizer

        A 			=					True

        A 			=					self.model.to(__lowerCAmelCase			)

        model.eval()
        model.float()

        A 			=					model.module if hasattr(__lowerCAmelCase		,    'module'			) else model
        quant_trainer.configure_model(__lowerCAmelCase		,    self.quant_trainer_args			)

        A 			=					os.path.join(__lowerCAmelCase		,    'model.onnx'			)
        logger.info(f'''exporting model to {output_model_file}'''			)

        A 			=					{0: '''batch_size''', 1: '''seq_len'''}

        torch.onnx.export(
            __lowerCAmelCase		,    __lowerCAmelCase		,    __lowerCAmelCase		,    export_params=__lowerCAmelCase		,    opset_version=13		,    do_constant_folding=__lowerCAmelCase		,    input_names=['input_ids', 'attention_mask', 'token_type_ids']		,    output_names=['output_start_logits', 'output_end_logits']		,    dynamic_axes={
                'input_ids': axes,
                'attention_mask': axes,
                'token_type_ids': axes,
                'output_start_logits': axes,
                'output_end_logits': axes,
            }		,    verbose=__lowerCAmelCase		,    )
        logger.info('onnx export finished'			)











import unittest

import numpy as np

from transformers.testing_utils import require_flax, require_tf, require_torch
from transformers.utils import (
    expand_dims,
    flatten_dict,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    reshape,
    squeeze,
    transpose,
)


if is_flax_available():
					import jax.numpy as jnp

if is_tf_available():
					import tensorflow as tf

if is_torch_available():
					import torch
class     lowerCamelCase_					(   unittest.TestCase			):







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[Any]   =   {
								    '''task_specific_params''': {
								        '''summarization''': {'''length_penalty''': 1.0, '''max_length''': 1_2_8, '''min_length''': 1_2, '''num_beams''': 4},
								        '''summarization_cnn''': {'''length_penalty''': 2.0, '''max_length''': 1_4_2, '''min_length''': 5_6, '''num_beams''': 4},
								        '''summarization_xsum''': {'''length_penalty''': 1.0, '''max_length''': 6_2, '''min_length''': 1_1, '''num_beams''': 6},
								    }
								}
								__magic_name__		:List[str]   =   {
								    '''task_specific_params.summarization.length_penalty''': 1.0,
								    '''task_specific_params.summarization.max_length''': 1_2_8,
								    '''task_specific_params.summarization.min_length''': 1_2,
								    '''task_specific_params.summarization.num_beams''': 4,
								    '''task_specific_params.summarization_cnn.length_penalty''': 2.0,
								    '''task_specific_params.summarization_cnn.max_length''': 1_4_2,
								    '''task_specific_params.summarization_cnn.min_length''': 5_6,
								    '''task_specific_params.summarization_cnn.num_beams''': 4,
								    '''task_specific_params.summarization_xsum.length_penalty''': 1.0,
								    '''task_specific_params.summarization_xsum.max_length''': 6_2,
								    '''task_specific_params.summarization_xsum.min_length''': 1_1,
								    '''task_specific_params.summarization_xsum.num_beams''': 6,
								}

								self.assertEqual(flatten_dict(__lowerCAmelCase  )							,       __lowerCAmelCase  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[Any]   =   np.random.randn(3							,       4  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase  )							,       x.transpose()  )  )

								__magic_name__		:List[Any]   =   np.random.randn(3							,       4							,       5  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  )							,       x.transpose((1, 2, 0)  )  )  )







				@require_torch
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[Any]   =   np.random.randn(3							,       4  )
								__magic_name__		:Tuple   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase  )							,       transpose(__lowerCAmelCase  ).numpy()  )  )

								__magic_name__		:int   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:Union[str, Any]   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  )							,       transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  ).numpy()  )  )







				@require_tf
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:int   =   np.random.randn(3							,       4  )
								__magic_name__		:Optional[Any]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase  )							,       transpose(__lowerCAmelCase  ).numpy()  )  )

								__magic_name__		:List[str]   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:Optional[Any]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  )							,       transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  ).numpy()  )  )







				@require_flax
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:int   =   np.random.randn(3							,       4  )
								__magic_name__		:Dict   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase  )							,       np.asarray(transpose(__lowerCAmelCase  )  )  )  )

								__magic_name__		:Dict   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:Dict   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  )							,       np.asarray(transpose(__lowerCAmelCase							,       axes=(1, 2, 0)  )  )  )  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Any   =   np.random.randn(3							,       4  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (4, 3)  )							,       np.reshape(__lowerCAmelCase							,       (4, 3)  )  )  )

								__magic_name__		:Union[str, Any]   =   np.random.randn(3							,       4							,       5  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (1_2, 5)  )							,       np.reshape(__lowerCAmelCase							,       (1_2, 5)  )  )  )







				@require_torch
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Dict   =   np.random.randn(3							,       4  )
								__magic_name__		:Tuple   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (4, 3)  )							,       reshape(__lowerCAmelCase							,       (4, 3)  ).numpy()  )  )

								__magic_name__		:Union[str, Any]   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:List[str]   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (1_2, 5)  )							,       reshape(__lowerCAmelCase							,       (1_2, 5)  ).numpy()  )  )







				@require_tf
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Dict   =   np.random.randn(3							,       4  )
								__magic_name__		:Union[str, Any]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (4, 3)  )							,       reshape(__lowerCAmelCase							,       (4, 3)  ).numpy()  )  )

								__magic_name__		:List[Any]   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:Optional[int]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (1_2, 5)  )							,       reshape(__lowerCAmelCase							,       (1_2, 5)  ).numpy()  )  )







				@require_flax
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[str]   =   np.random.randn(3							,       4  )
								__magic_name__		:Any   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (4, 3)  )							,       np.asarray(reshape(__lowerCAmelCase							,       (4, 3)  )  )  )  )

								__magic_name__		:List[Any]   =   np.random.randn(3							,       4							,       5  )
								__magic_name__		:List[str]   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(reshape(__lowerCAmelCase							,       (1_2, 5)  )							,       np.asarray(reshape(__lowerCAmelCase							,       (1_2, 5)  )  )  )  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[Any]   =   np.random.randn(1							,       3							,       4  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase  )							,       np.squeeze(__lowerCAmelCase  )  )  )

								__magic_name__		:Optional[Any]   =   np.random.randn(1							,       4							,       1							,       5  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase							,       axis=2  )							,       np.squeeze(__lowerCAmelCase							,       axis=2  )  )  )







				@require_torch
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Dict   =   np.random.randn(1							,       3							,       4  )
								__magic_name__		:List[Any]   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase  )							,       squeeze(__lowerCAmelCase  ).numpy()  )  )

								__magic_name__		:List[str]   =   np.random.randn(1							,       4							,       1							,       5  )
								__magic_name__		:str   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase							,       axis=2  )							,       squeeze(__lowerCAmelCase							,       axis=2  ).numpy()  )  )







				@require_tf
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:int   =   np.random.randn(1							,       3							,       4  )
								__magic_name__		:Tuple   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase  )							,       squeeze(__lowerCAmelCase  ).numpy()  )  )

								__magic_name__		:Tuple   =   np.random.randn(1							,       4							,       1							,       5  )
								__magic_name__		:Optional[int]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase							,       axis=2  )							,       squeeze(__lowerCAmelCase							,       axis=2  ).numpy()  )  )







				@require_flax
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Tuple   =   np.random.randn(1							,       3							,       4  )
								__magic_name__		:Optional[Any]   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase  )							,       np.asarray(squeeze(__lowerCAmelCase  )  )  )  )

								__magic_name__		:List[Any]   =   np.random.randn(1							,       4							,       1							,       5  )
								__magic_name__		:Optional[Any]   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(squeeze(__lowerCAmelCase							,       axis=2  )							,       np.asarray(squeeze(__lowerCAmelCase							,       axis=2  )  )  )  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Any   =   np.random.randn(3							,       4  )
								self.assertTrue(np.allclose(expand_dims(__lowerCAmelCase							,       axis=1  )							,       np.expand_dims(__lowerCAmelCase							,       axis=1  )  )  )







				@require_torch
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[Any]   =   np.random.randn(3							,       4  )
								__magic_name__		:Any   =   torch.tensor(__lowerCAmelCase  )
								self.assertTrue(np.allclose(expand_dims(__lowerCAmelCase							,       axis=1  )							,       expand_dims(__lowerCAmelCase							,       axis=1  ).numpy()  )  )







				@require_tf
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Union[str, Any]   =   np.random.randn(3							,       4  )
								__magic_name__		:Union[str, Any]   =   tf.constant(__lowerCAmelCase  )
								self.assertTrue(np.allclose(expand_dims(__lowerCAmelCase							,       axis=1  )							,       expand_dims(__lowerCAmelCase							,       axis=1  ).numpy()  )  )







				@require_flax
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[str]   =   np.random.randn(3							,       4  )
								__magic_name__		:Tuple   =   jnp.array(__lowerCAmelCase  )
								self.assertTrue(np.allclose(expand_dims(__lowerCAmelCase							,       axis=1  )							,       np.asarray(expand_dims(__lowerCAmelCase							,       axis=1  )  )  )  )






