

import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler, UNetaDConditionModel
from diffusers.pipelines.stable_diffusion_safe import StableDiffusionPipelineSafe as StableDiffusionPipeline
from diffusers.utils import floats_tensor, nightly, torch_device
from diffusers.utils.testing_utils import require_torch_gpu






class _UpperCamelCase				(				unittest.TestCase ):
			'''simple docstring'''







			def 				__UpperCamelCase			(      self				:			Union[str, Any]						)				->							int:

				"""simple docstring"""

				super().tearDown()
				gc.collect()
				torch.cuda.empty_cache()
			@property
			def 				__UpperCamelCase			(      self				:			Tuple						)				->							Optional[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   str					  =							1
				SCREAMING_SNAKE_CASE :   str					  =							3
				SCREAMING_SNAKE_CASE :   int					  =							(32, 32)

				SCREAMING_SNAKE_CASE :   str					  =							floats_tensor((batch_size, num_channels) + sizes   ,   rng=random.Random(0						)						).to(a						)
				return image
			@property
			def 				__UpperCamelCase			(      self				:			Optional[Any]						)				->							str:

				"""simple docstring"""

				torch.manual_seed(0						)
				SCREAMING_SNAKE_CASE :   Dict					  =							UNetaDConditionModel(
				    block_out_channels=(32, 64)   ,   layers_per_block=2   ,   sample_size=32   ,   in_channels=4   ,   out_channels=4   ,   down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")   ,   up_block_types=("CrossAttnUpBlock2D", "UpBlock2D")   ,   cross_attention_dim=32   ,   )
				return model
			@property
			def 				__UpperCamelCase			(      self				:			int						)				->							Any:

				"""simple docstring"""

				torch.manual_seed(0						)
				SCREAMING_SNAKE_CASE :   Tuple					  =							AutoencoderKL(
				    block_out_channels=[32, 64]   ,   in_channels=3   ,   out_channels=3   ,   down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"]   ,   up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"]   ,   latent_channels=4   ,   )
				return model
			@property
			def 				__UpperCamelCase			(      self				:			Tuple						)				->							str:

				"""simple docstring"""

				torch.manual_seed(0						)
				SCREAMING_SNAKE_CASE :   Tuple					  =							CLIPTextConfig(
				    bos_token_id=0   ,   eos_token_id=2   ,   hidden_size=32   ,   intermediate_size=37   ,   layer_norm_eps=1e-05   ,   num_attention_heads=4   ,   num_hidden_layers=5   ,   pad_token_id=1   ,   vocab_size=1000   ,   )
				return CLIPTextModel(a						)
			@property
			def 				__UpperCamelCase			(      self				:			List[str]						)				->							str:

				"""simple docstring"""

				def extract(*a				:			int   ,   **a				:			Union[str, Any]						):






					class _UpperCamelCase				:
								'''simple docstring'''







								def __init__(      self				:			Optional[Any]						)				->							str:

									"""simple docstring"""

									SCREAMING_SNAKE_CASE :   Union[str, Any]					  =							torch.ones([0]						)






								def 				__UpperCamelCase			(      self				:			str   ,   a				:			List[Any]						)				->							List[str]:

									"""simple docstring"""

									self.pixel_values.to(a						)
									return self

					return Out()

				return extract
			def 				__UpperCamelCase			(      self				:			List[Any]						)				->							Any:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   List[Any]					  =							"cpu"  # ensure determinism for the device-dependent torch.Generator
				SCREAMING_SNAKE_CASE :   int					  =							self.dummy_cond_unet
				SCREAMING_SNAKE_CASE :   Dict					  =							DDIMScheduler(
				    beta_start=0.0_0085   ,   beta_end=0.012   ,   beta_schedule="scaled_linear"   ,   clip_sample=a   ,   set_alpha_to_one=a   ,   )

				SCREAMING_SNAKE_CASE :   Tuple					  =							self.dummy_vae
				SCREAMING_SNAKE_CASE :   Dict					  =							self.dummy_text_encoder
				SCREAMING_SNAKE_CASE :   Dict					  =							CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"						)

				# make sure here that pndm scheduler skips prk
				SCREAMING_SNAKE_CASE :   int					  =							StableDiffusionPipeline(
				    unet=a   ,   scheduler=a   ,   vae=a   ,   text_encoder=a   ,   tokenizer=a   ,   safety_checker=a   ,   feature_extractor=self.dummy_extractor   ,   )
				SCREAMING_SNAKE_CASE :   List[str]					  =							sd_pipe.to(a						)
				sd_pipe.set_progress_bar_config(disable=a						)

				SCREAMING_SNAKE_CASE :   Union[str, Any]					  =							"A painting of a squirrel eating a burger"

				SCREAMING_SNAKE_CASE :   Any					  =							torch.Generator(device=a						).manual_seed(0						)
				SCREAMING_SNAKE_CASE :   Dict					  =							sd_pipe([prompt]   ,   generator=a   ,   guidance_scale=6.0   ,   num_inference_steps=2   ,   output_type="np"						)
				SCREAMING_SNAKE_CASE :   Dict					  =							output.images

				SCREAMING_SNAKE_CASE :   Optional[Any]					  =							torch.Generator(device=a						).manual_seed(0						)
				SCREAMING_SNAKE_CASE :   Any					  =							sd_pipe(
				    [prompt]   ,   generator=a   ,   guidance_scale=6.0   ,   num_inference_steps=2   ,   output_type="np"   ,   return_dict=a   ,   )[0]

				SCREAMING_SNAKE_CASE :   str					  =							image[0, -3:, -3:, -1]
				SCREAMING_SNAKE_CASE :   Dict					  =							image_from_tuple[0, -3:, -3:, -1]

				assert image.shape == (1, 64, 64, 3)
				SCREAMING_SNAKE_CASE :   Tuple					  =							np.array([0.5756, 0.6118, 0.5005, 0.5041, 0.5471, 0.4726, 0.4976, 0.4865, 0.4864]						)

				assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2
				assert np.abs(image_from_tuple_slice.flatten() - expected_slice						).max() < 1e-2
			def 				__UpperCamelCase			(      self				:			Any						)				->							Union[str, Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   List[Any]					  =							"cpu"  # ensure determinism for the device-dependent torch.Generator
				SCREAMING_SNAKE_CASE :   List[str]					  =							self.dummy_cond_unet
				SCREAMING_SNAKE_CASE :   Tuple					  =							PNDMScheduler(skip_prk_steps=a						)
				SCREAMING_SNAKE_CASE :   Dict					  =							self.dummy_vae
				SCREAMING_SNAKE_CASE :   Dict					  =							self.dummy_text_encoder
				SCREAMING_SNAKE_CASE :   Tuple					  =							CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"						)

				# make sure here that pndm scheduler skips prk
				SCREAMING_SNAKE_CASE :   List[str]					  =							StableDiffusionPipeline(
				    unet=a   ,   scheduler=a   ,   vae=a   ,   text_encoder=a   ,   tokenizer=a   ,   safety_checker=a   ,   feature_extractor=self.dummy_extractor   ,   )
				SCREAMING_SNAKE_CASE :   Optional[int]					  =							sd_pipe.to(a						)
				sd_pipe.set_progress_bar_config(disable=a						)

				SCREAMING_SNAKE_CASE :   List[str]					  =							"A painting of a squirrel eating a burger"
				SCREAMING_SNAKE_CASE :   Union[str, Any]					  =							torch.Generator(device=a						).manual_seed(0						)
				SCREAMING_SNAKE_CASE :   Union[str, Any]					  =							sd_pipe([prompt]   ,   generator=a   ,   guidance_scale=6.0   ,   num_inference_steps=2   ,   output_type="np"						)

				SCREAMING_SNAKE_CASE :   int					  =							output.images

				SCREAMING_SNAKE_CASE :   List[str]					  =							torch.Generator(device=a						).manual_seed(0						)
				SCREAMING_SNAKE_CASE :   List[str]					  =							sd_pipe(
				    [prompt]   ,   generator=a   ,   guidance_scale=6.0   ,   num_inference_steps=2   ,   output_type="np"   ,   return_dict=a   ,   )[0]

				SCREAMING_SNAKE_CASE :   str					  =							image[0, -3:, -3:, -1]
				SCREAMING_SNAKE_CASE :   str					  =							image_from_tuple[0, -3:, -3:, -1]

				assert image.shape == (1, 64, 64, 3)
				SCREAMING_SNAKE_CASE :   Any					  =							np.array([0.5125, 0.5716, 0.4828, 0.5060, 0.5650, 0.4768, 0.5185, 0.4895, 0.4993]						)

				assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2
				assert np.abs(image_from_tuple_slice.flatten() - expected_slice						).max() < 1e-2
			def 				__UpperCamelCase			(      self				:			Dict						)				->							List[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   Union[str, Any]					  =							StableDiffusionPipeline.from_pretrained(
				    "hf-internal-testing/tiny-stable-diffusion-lms-pipe"   ,   safety_checker=a						)
				assert isinstance(a   ,   a						)
				assert isinstance(pipe.scheduler   ,   a						)
				assert pipe.safety_checker is None

				SCREAMING_SNAKE_CASE :   Tuple					  =							pipe("example prompt"   ,   num_inference_steps=2						).images[0]
				assert image is not None

				# check that there's no error when saving a pipeline with one of the models being None
				with tempfile.TemporaryDirectory() as tmpdirname:
					pipe.save_pretrained(a						)
					SCREAMING_SNAKE_CASE :   Union[str, Any]					  =							StableDiffusionPipeline.from_pretrained(a						)

				# sanity check that the pipeline still works
				assert pipe.safety_checker is None
				SCREAMING_SNAKE_CASE :   Dict					  =							pipe("example prompt"   ,   num_inference_steps=2						).images[0]
				assert image is not None






			@unittest.skipIf(torch_device != "cuda"   ,   "This test requires a GPU"						)
			def 				__UpperCamelCase			(      self				:			Dict						)				->							Tuple:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   int					  =							self.dummy_cond_unet
				SCREAMING_SNAKE_CASE :   Dict					  =							PNDMScheduler(skip_prk_steps=a						)
				SCREAMING_SNAKE_CASE :   Dict					  =							self.dummy_vae
				SCREAMING_SNAKE_CASE :   Dict					  =							self.dummy_text_encoder
				SCREAMING_SNAKE_CASE :   Any					  =							CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"						)

				# put models in fp16
				SCREAMING_SNAKE_CASE :   Tuple					  =							unet.half()
				SCREAMING_SNAKE_CASE :   Tuple					  =							vae.half()
				SCREAMING_SNAKE_CASE :   Tuple					  =							bert.half()

				# make sure here that pndm scheduler skips prk
				SCREAMING_SNAKE_CASE :   str					  =							StableDiffusionPipeline(
				    unet=a   ,   scheduler=a   ,   vae=a   ,   text_encoder=a   ,   tokenizer=a   ,   safety_checker=a   ,   feature_extractor=self.dummy_extractor   ,   )
				SCREAMING_SNAKE_CASE :   List[Any]					  =							sd_pipe.to(a						)
				sd_pipe.set_progress_bar_config(disable=a						)

				SCREAMING_SNAKE_CASE :   List[Any]					  =							"A painting of a squirrel eating a burger"
				SCREAMING_SNAKE_CASE :   Tuple					  =							sd_pipe([prompt]   ,   num_inference_steps=2   ,   output_type="np"						).images

				assert image.shape == (1, 64, 64, 3)






@nightly
@require_torch_gpu
class _UpperCamelCase				(				unittest.TestCase ):
			'''simple docstring'''







			def 				__UpperCamelCase			(      self				:			Tuple						)				->							Optional[int]:

				"""simple docstring"""

				super().tearDown()
				gc.collect()
				torch.cuda.empty_cache()
			def 				__UpperCamelCase			(      self				:			Any						)				->							Optional[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   Dict					  =							StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5"   ,   safety_checker=a						)
				SCREAMING_SNAKE_CASE :   Tuple					  =							LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config						)
				SCREAMING_SNAKE_CASE :   Optional[Any]					  =							sd_pipe.to(a						)
				sd_pipe.set_progress_bar_config(disable=a						)

				SCREAMING_SNAKE_CASE :   Optional[int]					  =							(
				    "portrait of girl with smokey eyes makeup in abandoned hotel, grange clothes, redshift, wide high angle"
				    " coloured polaroid photograph with flash, kodak film, hyper real, stunning moody cinematography, with"
				    " anamorphic lenses, by maripol, fallen angels by wong kar - wai, style of suspiria and neon demon and"
				    " children from bahnhof zoo, detailed "
				)
				SCREAMING_SNAKE_CASE :   Union[str, Any]					  =							40_0366_0346
				SCREAMING_SNAKE_CASE :   Tuple					  =							7

				# without safety guidance (sld_guidance_scale = 0)
				SCREAMING_SNAKE_CASE :   Union[str, Any]					  =							torch.manual_seed(a						)
				SCREAMING_SNAKE_CASE :   Dict					  =							sd_pipe(
				    [prompt]   ,   generator=a   ,   guidance_scale=a   ,   num_inference_steps=50   ,   output_type="np"   ,   width=512   ,   height=512   ,   sld_guidance_scale=0   ,   )

				SCREAMING_SNAKE_CASE :   Union[str, Any]					  =							output.images
				SCREAMING_SNAKE_CASE :   Optional[int]					  =							image[0, -3:, -3:, -1]
				SCREAMING_SNAKE_CASE :   List[str]					  =							[0.2278, 0.2231, 0.2249, 0.2333, 0.2303, 0.1885, 0.2273, 0.2144, 0.2176]

				assert image.shape == (1, 512, 512, 3)

				assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2

				# without safety guidance (strong configuration)
				SCREAMING_SNAKE_CASE :   Optional[int]					  =							torch.manual_seed(a						)
				SCREAMING_SNAKE_CASE :   str					  =							sd_pipe(
				    [prompt]   ,   generator=a   ,   guidance_scale=a   ,   num_inference_steps=50   ,   output_type="np"   ,   width=512   ,   height=512   ,   sld_guidance_scale=2000   ,   sld_warmup_steps=7   ,   sld_threshold=0.025   ,   sld_momentum_scale=0.5   ,   sld_mom_beta=0.7   ,   )

				SCREAMING_SNAKE_CASE :   Union[str, Any]					  =							output.images
				SCREAMING_SNAKE_CASE :   Optional[int]					  =							image[0, -3:, -3:, -1]
				SCREAMING_SNAKE_CASE :   List[Any]					  =							[0.2383, 0.2276, 0.236, 0.2192, 0.2186, 0.2053, 0.1971, 0.1901, 0.1719]

				assert image.shape == (1, 512, 512, 3)

				assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2
			def 				__UpperCamelCase			(      self				:			str						)				->							Union[str, Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   Dict					  =							StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5"   ,   safety_checker=a						)
				SCREAMING_SNAKE_CASE :   Union[str, Any]					  =							LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config						)
				SCREAMING_SNAKE_CASE :   Any					  =							sd_pipe.to(a						)
				sd_pipe.set_progress_bar_config(disable=a						)

				SCREAMING_SNAKE_CASE :   Union[str, Any]					  =							"padme amidala taking a bath artwork, safe for work, no nudity"
				SCREAMING_SNAKE_CASE :   Optional[int]					  =							27_3497_1755
				SCREAMING_SNAKE_CASE :   Optional[Any]					  =							7

				SCREAMING_SNAKE_CASE :   str					  =							torch.manual_seed(a						)
				SCREAMING_SNAKE_CASE :   Dict					  =							sd_pipe(
				    [prompt]   ,   generator=a   ,   guidance_scale=a   ,   num_inference_steps=50   ,   output_type="np"   ,   width=512   ,   height=512   ,   sld_guidance_scale=0   ,   )

				SCREAMING_SNAKE_CASE :   List[Any]					  =							output.images
				SCREAMING_SNAKE_CASE :   str					  =							image[0, -3:, -3:, -1]
				SCREAMING_SNAKE_CASE :   Any					  =							[0.3502, 0.3622, 0.3396, 0.3642, 0.3478, 0.3318, 0.35, 0.3348, 0.3297]

				assert image.shape == (1, 512, 512, 3)

				assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2

				SCREAMING_SNAKE_CASE :   int					  =							torch.manual_seed(a						)
				SCREAMING_SNAKE_CASE :   List[str]					  =							sd_pipe(
				    [prompt]   ,   generator=a   ,   guidance_scale=a   ,   num_inference_steps=50   ,   output_type="np"   ,   width=512   ,   height=512   ,   sld_guidance_scale=2000   ,   sld_warmup_steps=7   ,   sld_threshold=0.025   ,   sld_momentum_scale=0.5   ,   sld_mom_beta=0.7   ,   )

				SCREAMING_SNAKE_CASE :   Dict					  =							output.images
				SCREAMING_SNAKE_CASE :   Any					  =							image[0, -3:, -3:, -1]
				SCREAMING_SNAKE_CASE :   Optional[Any]					  =							[0.5531, 0.5206, 0.4895, 0.5156, 0.5182, 0.4751, 0.4802, 0.4803, 0.4443]

				assert image.shape == (1, 512, 512, 3)

				assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2






			def 				__UpperCamelCase			(      self				:			Tuple						)				->							Optional[int]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE :   Optional[int]					  =							StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5"						)
				SCREAMING_SNAKE_CASE :   List[str]					  =							sd_pipe.to(a						)
				sd_pipe.set_progress_bar_config(disable=a						)

				SCREAMING_SNAKE_CASE :   Optional[Any]					  =							(
				    "the four horsewomen of the apocalypse, painting by tom of finland, gaston bussiere, craig mullins, j. c."
				    " leyendecker"
				)
				SCREAMING_SNAKE_CASE :   str					  =							10_4435_5234
				SCREAMING_SNAKE_CASE :   List[str]					  =							12

				SCREAMING_SNAKE_CASE :   Any					  =							torch.manual_seed(a						)
				SCREAMING_SNAKE_CASE :   Optional[int]					  =							sd_pipe(
				    [prompt]   ,   generator=a   ,   guidance_scale=a   ,   num_inference_steps=50   ,   output_type="np"   ,   width=512   ,   height=512   ,   sld_guidance_scale=0   ,   )

				SCREAMING_SNAKE_CASE :   Tuple					  =							output.images
				SCREAMING_SNAKE_CASE :   List[str]					  =							image[0, -3:, -3:, -1]
				SCREAMING_SNAKE_CASE :   List[Any]					  =							np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]						)

				assert image.shape == (1, 512, 512, 3)

				assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-7

				SCREAMING_SNAKE_CASE :   Optional[Any]					  =							torch.manual_seed(a						)
				SCREAMING_SNAKE_CASE :   Optional[Any]					  =							sd_pipe(
				    [prompt]   ,   generator=a   ,   guidance_scale=a   ,   num_inference_steps=50   ,   output_type="np"   ,   width=512   ,   height=512   ,   sld_guidance_scale=2000   ,   sld_warmup_steps=7   ,   sld_threshold=0.025   ,   sld_momentum_scale=0.5   ,   sld_mom_beta=0.7   ,   )

				SCREAMING_SNAKE_CASE :   Dict					  =							output.images
				SCREAMING_SNAKE_CASE :   Optional[Any]					  =							image[0, -3:, -3:, -1]
				SCREAMING_SNAKE_CASE :   Tuple					  =							np.array([0.5818, 0.6285, 0.6835, 0.6019, 0.625, 0.6754, 0.6096, 0.6334, 0.6561]						)
				assert image.shape == (1, 512, 512, 3)

				assert np.abs(image_slice.flatten() - expected_slice						).max() < 1e-2







import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


_SCREAMING_SNAKE_CASE   =  logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
    import torch_xla.core.xla_model as xm
    import torch_xla.debug.metrics as met






class 					SCREAMING_SNAKE_CASE_  (       _a ):
       """simple docstring"""


       def __init__(				self	:Dict,      *snake_case	:int,      snake_case	:Any=None,      snake_case	:str=None,      snake_case	:int=None,      **snake_case	:List[str]):



          """simple docstring"""

          super().__init__(*snake_case,      **snake_case)
          _lowercase              =eval_examples
          _lowercase              =post_process_function
          _lowercase              =quant_trainer_args
          _lowercase              =128  # default number of calibration samples





       def 				UpperCamelCase__						(				self	:Dict,      snake_case	:List[str]=None):



          """simple docstring"""

          if calib_dataset is None and self.calib_dataset is None:
             raise ValueError('Trainer: calibration requires an calib_dataset.')
          _lowercase              =calib_dataset if calib_dataset is not None else self.calib_dataset

          _lowercase              =self._remove_unused_columns(snake_case,      description='Calibration')

          return DataLoader(
              snake_case,      batch_size=self.args.eval_batch_size,      collate_fn=self.data_collator,      drop_last=self.args.dataloader_drop_last,      num_workers=self.args.dataloader_num_workers,      pin_memory=self.args.dataloader_pin_memory,      shuffle=snake_case,      )





       def 				UpperCamelCase__						(				self	:Any,      snake_case	:Union[str, Any]=None):



          """simple docstring"""

          _lowercase              =self.train_dataset if calib_dataset is None else calib_dataset
          _lowercase              =self.get_calib_dataloader(snake_case)

          _lowercase              =self.model
          quant_trainer.configure_model(snake_case,      self.quant_trainer_args,      calib=snake_case)
          model.eval()
          quant_trainer.enable_calibration(snake_case)

          logger.info('***** Running calibration *****')
          logger.info(f'''  Num examples = {self.calib_num}''')
          logger.info(f'''  Batch size = {calib_dataloader.batch_size}''')

          for step, inputs in enumerate(snake_case):
             # Prediction step
             _lowercase       ,       _lowercase       ,       _lowercase              =self.prediction_step(snake_case,      snake_case,      prediction_loss_only=snake_case)
             if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
                break

          quant_trainer.finish_calibration(snake_case,      self.quant_trainer_args)
          _lowercase              =model





       def 				UpperCamelCase__						(				self	:Optional[int],      snake_case	:Union[str, Any]=None,      snake_case	:Optional[int]=None,      snake_case	:Union[str, Any]=None,      snake_case	:str = "eval"):



          """simple docstring"""

          _lowercase              =self.eval_dataset if eval_dataset is None else eval_dataset
          _lowercase              =self.get_eval_dataloader(snake_case)
          _lowercase              =self.eval_examples if eval_examples is None else eval_examples

          # Temporarily disable metric computation, we will do it in the loop here.
          _lowercase              =self.compute_metrics
          _lowercase              =None
          _lowercase              =self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
          try:
             _lowercase              =eval_loop(
                 snake_case,      description='Evaluation',      prediction_loss_only=True if compute_metrics is None else None,      ignore_keys=snake_case,      )
          finally:
             _lowercase              =compute_metrics

          if self.post_process_function is not None and self.compute_metrics is not None:
             _lowercase              =self.post_process_function(snake_case,      snake_case,      output.predictions)
             _lowercase              =self.compute_metrics(snake_case)

             # Prefix all keys with metric_key_prefix + '_'
             for key in list(metrics.keys()):
                if not key.startswith(f'''{metric_key_prefix}_'''):
                   _lowercase              =metrics.pop(snake_case)

             self.log(snake_case)
          else:
             _lowercase              ={}

          if self.args.tpu_metrics_debug or self.args.debug:
             # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
             xm.master_print(met.metrics_report())

          _lowercase              =self.callback_handler.on_evaluate(self.args,      self.state,      self.control,      snake_case)
          return metrics





       def 				UpperCamelCase__						(				self	:Optional[int],      snake_case	:str,      snake_case	:Optional[Any],      snake_case	:Any=None,      snake_case	:str = "test"):



          """simple docstring"""

          _lowercase              =self.get_test_dataloader(snake_case)

          # Temporarily disable metric computation, we will do it in the loop here.
          _lowercase              =self.compute_metrics
          _lowercase              =None
          _lowercase              =self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
          try:
             _lowercase              =eval_loop(
                 snake_case,      description='Prediction',      prediction_loss_only=True if compute_metrics is None else None,      ignore_keys=snake_case,      )
          finally:
             _lowercase              =compute_metrics

          if self.post_process_function is None or self.compute_metrics is None:
             return output

          _lowercase              =self.post_process_function(snake_case,      snake_case,      output.predictions,      'predict')
          _lowercase              =self.compute_metrics(snake_case)

          # Prefix all keys with metric_key_prefix + '_'
          for key in list(metrics.keys()):
             if not key.startswith(f'''{metric_key_prefix}_'''):
                _lowercase              =metrics.pop(snake_case)

          return PredictionOutput(predictions=predictions.predictions,      label_ids=predictions.label_ids,      metrics=snake_case)





       def 				UpperCamelCase__						(				self	:List[str],      snake_case	:Dict="./"):



          """simple docstring"""

          _lowercase              =self.eval_dataset
          _lowercase              =self.get_eval_dataloader(snake_case)

          _lowercase              =next(iter(snake_case))

          # saving device - to make it consistent
          _lowercase              =torch.device('cuda' if torch.cuda.is_available() else 'cpu')

          # convert to tuple
          _lowercase              =tuple(v.to(snake_case) for k, v in batch.items())

          logger.info('Converting model to be onnx compatible')
          from pytorch_quantization.nn import TensorQuantizer

          _lowercase              =True

          _lowercase              =self.model.to(snake_case)

          model.eval()
          model.float()

          _lowercase              =model.module if hasattr(snake_case,      'module') else model
          quant_trainer.configure_model(snake_case,      self.quant_trainer_args)

          _lowercase              =os.path.join(snake_case,      'model.onnx')
          logger.info(f'''exporting model to {output_model_file}''')

          _lowercase              ={0: 'batch_size', 1: 'seq_len'}

          torch.onnx.export(
              snake_case,      snake_case,      snake_case,      export_params=snake_case,      opset_version=13,      do_constant_folding=snake_case,      input_names=['input_ids', 'attention_mask', 'token_type_ids'],      output_names=['output_start_logits', 'output_end_logits'],      dynamic_axes={
                  'input_ids': axes,
                  'attention_mask': axes,
                  'token_type_ids': axes,
                  'output_start_logits': axes,
                  'output_end_logits': axes,
              },      verbose=snake_case,      )
          logger.info('onnx export finished')


