





import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


_lowerCAmelCase								=  logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
      import torch_xla.core.xla_model as xm
      import torch_xla.debug.metrics as met

class  _UpperCAmelCase				(		_lowerCamelCase  ):
      def __init__(		self   ,							*a__   ,							a__=None   ,							a__=None   ,							a__=None   ,							**a__							):
         super().__init__(*a__   ,							**a__							)
         A_  :					List[str]         =						eval_examples
         A_  :					Optional[Any]         =						post_process_function
         A_  :					int         =						quant_trainer_args
         A_  :					int         =						128  # default number of calibration samples
      def 		_lowerCamelCase   (		self   ,							a__=None							):
         if calib_dataset is None and self.calib_dataset is None:
            raise ValueError("""Trainer: calibration requires an calib_dataset."""							)
         A_  :					Union[str, Any]         =						calib_dataset if calib_dataset is not None else self.calib_dataset

         A_  :					Tuple         =						self._remove_unused_columns(a__   ,							description="""Calibration"""							)

         return DataLoader(
             a__   ,							batch_size=self.args.eval_batch_size   ,							collate_fn=self.data_collator   ,							drop_last=self.args.dataloader_drop_last   ,							num_workers=self.args.dataloader_num_workers   ,							pin_memory=self.args.dataloader_pin_memory   ,							shuffle=a__   ,							)
      def 		_lowerCamelCase   (		self   ,							a__=None							):
         A_  :					List[str]         =						self.train_dataset if calib_dataset is None else calib_dataset
         A_  :					Dict         =						self.get_calib_dataloader(a__							)

         A_  :					Union[str, Any]         =						self.model
         quant_trainer.configure_model(a__   ,							self.quant_trainer_args   ,							calib=a__							)
         model.eval()
         quant_trainer.enable_calibration(a__							)

         logger.info("""***** Running calibration *****"""							)
         logger.info(F"""  Num examples = {self.calib_num}"""							)
         logger.info(F"""  Batch size = {calib_dataloader.batch_size}"""							)

         for step, inputs in enumerate(a__							):
            # Prediction step
            A_  :					List[Any]         =						self.prediction_step(a__   ,							a__   ,							prediction_loss_only=a__							)
            if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
               break

         quant_trainer.finish_calibration(a__   ,							self.quant_trainer_args							)
         A_  :					Any         =						model
      def 		_lowerCamelCase   (		self   ,							a__=None   ,							a__=None   ,							a__=None   ,							a__ = "eval"							):
         A_  :					Optional[Any]         =						self.eval_dataset if eval_dataset is None else eval_dataset
         A_  :					Any         =						self.get_eval_dataloader(a__							)
         A_  :					str         =						self.eval_examples if eval_examples is None else eval_examples

         # Temporarily disable metric computation, we will do it in the loop here.
         A_  :					List[Any]         =						self.compute_metrics
         A_  :					List[Any]         =						None
         A_  :					Any         =						self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
         try:
            A_  :					List[Any]         =						eval_loop(
                a__   ,							description="""Evaluation"""   ,							prediction_loss_only=True if compute_metrics is None else None   ,							ignore_keys=a__   ,							)
         finally:
            A_  :					Any         =						compute_metrics

         if self.post_process_function is not None and self.compute_metrics is not None:
            A_  :					Optional[int]         =						self.post_process_function(a__   ,							a__   ,							output.predictions							)
            A_  :					Any         =						self.compute_metrics(a__							)

            # Prefix all keys with metric_key_prefix + '_'
            for key in list(metrics.keys()							):
               if not key.startswith(F"""{metric_key_prefix}_"""							):
                  A_  :					int         =						metrics.pop(a__							)

            self.log(a__							)
         else:
            A_  :					Any         =						{}

         if self.args.tpu_metrics_debug or self.args.debug:
            # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
            xm.master_print(met.metrics_report()							)

         A_  :					Optional[int]         =						self.callback_handler.on_evaluate(self.args   ,							self.state   ,							self.control   ,							a__							)
         return metrics
      def 		_lowerCamelCase   (		self   ,							a__   ,							a__   ,							a__=None   ,							a__ = "test"							):
         A_  :					List[str]         =						self.get_test_dataloader(a__							)

         # Temporarily disable metric computation, we will do it in the loop here.
         A_  :					List[str]         =						self.compute_metrics
         A_  :					Tuple         =						None
         A_  :					Dict         =						self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
         try:
            A_  :					int         =						eval_loop(
                a__   ,							description="""Prediction"""   ,							prediction_loss_only=True if compute_metrics is None else None   ,							ignore_keys=a__   ,							)
         finally:
            A_  :					Tuple         =						compute_metrics

         if self.post_process_function is None or self.compute_metrics is None:
            return output

         A_  :					str         =						self.post_process_function(a__   ,							a__   ,							output.predictions   ,							"""predict"""							)
         A_  :					List[str]         =						self.compute_metrics(a__							)

         # Prefix all keys with metric_key_prefix + '_'
         for key in list(metrics.keys()							):
            if not key.startswith(F"""{metric_key_prefix}_"""							):
               A_  :					Tuple         =						metrics.pop(a__							)

         return PredictionOutput(predictions=predictions.predictions   ,							label_ids=predictions.label_ids   ,							metrics=a__							)






      def 		_lowerCamelCase   (		self   ,							a__="./"							):
         A_  :					Optional[Any]         =						self.eval_dataset
         A_  :					Tuple         =						self.get_eval_dataloader(a__							)

         A_  :					Dict         =						next(iter(a__							)							)

         # saving device - to make it consistent
         A_  :					List[Any]         =						torch.device("""cuda""" if torch.cuda.is_available() else """cpu"""							)

         # convert to tuple
         A_  :					int         =						tuple(v.to(a__							) for k, v in batch.items()							)

         logger.info("""Converting model to be onnx compatible"""							)
         from pytorch_quantization.nn import TensorQuantizer

         A_  :					str         =						True

         A_  :					Dict         =						self.model.to(a__							)

         model.eval()
         model.float()

         A_  :					Optional[Any]         =						model.module if hasattr(a__   ,							"""module"""							) else model
         quant_trainer.configure_model(a__   ,							self.quant_trainer_args							)

         A_  :					int         =						os.path.join(a__   ,							"""model.onnx"""							)
         logger.info(F"""exporting model to {output_model_file}"""							)

         A_  :					Optional[Any]         =						{0: 'batch_size', 1: 'seq_len'}

         torch.onnx.export(
             a__   ,							a__   ,							a__   ,							export_params=a__   ,							opset_version=13   ,							do_constant_folding=a__   ,							input_names=["""input_ids""", """attention_mask""", """token_type_ids"""]   ,							output_names=["""output_start_logits""", """output_end_logits"""]   ,							dynamic_axes={
                 """input_ids""": axes,
                 """attention_mask""": axes,
                 """token_type_ids""": axes,
                 """output_start_logits""": axes,
                 """output_end_logits""": axes,
             }   ,							verbose=a__   ,							)
         logger.info("""onnx export finished"""							)














"""simple docstring"""


import argparse
import json
import math
import os
import time
import traceback
import zipfile
from collections import Counter

import requests


def        UpperCAmelCase      (					a__     ,      a__=None ):





						'''simple docstring'''






						lowerCAmelCase   :str						=			None
						if token is not None:
												lowerCAmelCase   :List[Any]						=			{'Accept': 'application/vnd.github+json', 'Authorization': F"""Bearer {token}"""}

						lowerCAmelCase   :Dict						=			F"""https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id}/jobs?per_page=100"""
						lowerCAmelCase   :Optional[Any]						=			requests.get(a__     ,      headers=a__ ).json()
						lowerCAmelCase   :Tuple						=			{}

						try:
												job_links.update({job['name']: job['html_url'] for job in result['jobs']} )
												lowerCAmelCase   :List[Any]						=			math.ceil((result['total_count'] - 1_00) / 1_00 )

												for i in range(a__ ):
																		lowerCAmelCase   :List[Any]						=			requests.get(url + F"""&page={i + 2}"""     ,      headers=a__ ).json()
																		job_links.update({job['name']: job['html_url'] for job in result['jobs']} )

												return job_links
						except Exception:
												print(F"""Unknown error, could not fetch links:\n{traceback.format_exc()}""" )

						return {}


def        UpperCAmelCase      (					a__     ,      a__=None ):





						'''simple docstring'''






						lowerCAmelCase   :Optional[Any]						=			None
						if token is not None:
												lowerCAmelCase   :Any						=			{'Accept': 'application/vnd.github+json', 'Authorization': F"""Bearer {token}"""}

						lowerCAmelCase   :Optional[int]						=			F"""https://api.github.com/repos/huggingface/transformers/actions/runs/{worflow_run_id}/artifacts?per_page=100"""
						lowerCAmelCase   :Any						=			requests.get(a__     ,      headers=a__ ).json()
						lowerCAmelCase   :str						=			{}

						try:
												artifacts.update({artifact['name']: artifact['archive_download_url'] for artifact in result['artifacts']} )
												lowerCAmelCase   :List[Any]						=			math.ceil((result['total_count'] - 1_00) / 1_00 )

												for i in range(a__ ):
																		lowerCAmelCase   :List[Any]						=			requests.get(url + F"""&page={i + 2}"""     ,      headers=a__ ).json()
																		artifacts.update({artifact['name']: artifact['archive_download_url'] for artifact in result['artifacts']} )

												return artifacts
						except Exception:
												print(F"""Unknown error, could not fetch links:\n{traceback.format_exc()}""" )

						return {}


def        UpperCAmelCase      (					a__     ,      a__     ,      a__     ,      a__ ):





						'''simple docstring'''






						lowerCAmelCase   :Optional[Any]						=			None
						if token is not None:
												lowerCAmelCase   :Optional[Any]						=			{'Accept': 'application/vnd.github+json', 'Authorization': F"""Bearer {token}"""}

						lowerCAmelCase   :Tuple						=			requests.get(a__     ,      headers=a__     ,      allow_redirects=a__ )
						lowerCAmelCase   :Optional[int]						=			result.headers['Location']
						lowerCAmelCase   :int						=			requests.get(a__     ,      allow_redirects=a__ )
						lowerCAmelCase   :Union[str, Any]						=			os.path.join(a__     ,      F"""{artifact_name}.zip""" )
						with open(a__     ,      'wb' ) as fp:
												fp.write(response.content )


def        UpperCAmelCase      (					a__     ,      a__=None ):





						'''simple docstring'''






						lowerCAmelCase   :Optional[Any]						=			[]
						lowerCAmelCase   :Dict						=			[]
						lowerCAmelCase   :Optional[Any]						=			None

						with zipfile.ZipFile(a__ ) as z:
												for filename in z.namelist():
																		if not os.path.isdir(a__ ):
																								# read the file
																								if filename in ["failures_line.txt", "summary_short.txt", "job_name.txt"]:
																														with z.open(a__ ) as f:
																																				for line in f:
																																										lowerCAmelCase   :int						=			line.decode('UTF-8' ).strip()
																																										if filename == "failures_line.txt":
																																																try:
																																																						# `error_line` is the place where `error` occurs
																																																						lowerCAmelCase   :int						=			line[: line.index(': ' )]
																																																						lowerCAmelCase   :int						=			line[line.index(': ' ) + len(': ' ) :]
																																																						errors.append([error_line, error] )
																																																except Exception:
																																																						# skip un-related lines
																																																						pass
																																										elif filename == "summary_short.txt" and line.startswith('FAILED ' ):
																																																# `test` is the test method that failed
																																																lowerCAmelCase   :int						=			line[len('FAILED ' ) :]
																																																failed_tests.append(a__ )
																																										elif filename == "job_name.txt":
																																																lowerCAmelCase   :List[str]						=			line

						if len(a__ ) != len(a__ ):
												raise ValueError(
												    F"""`errors` and `failed_tests` should have the same number of elements. Got {len(a__ )} for `errors` """
												    F"""and {len(a__ )} for `failed_tests` instead. The test reports in {artifact_zip_path} have some"""
												    ' problem.' )

						lowerCAmelCase   :Optional[int]						=			None
						if job_name and job_links:
												lowerCAmelCase   :Dict						=			job_links.get(a__     ,      a__ )

						# A list with elements of the form (line of error, error, failed test)
						lowerCAmelCase   :Union[str, Any]						=			[x + [y] + [job_link] for x, y in zip(a__     ,      a__ )]

						return result


def        UpperCAmelCase      (					a__     ,      a__=None ):





						'''simple docstring'''






						lowerCAmelCase   :Any						=			[]

						lowerCAmelCase   :Optional[int]						=			[os.path.join(a__     ,      a__ ) for p in os.listdir(a__ ) if p.endswith('.zip' )]
						for p in paths:
												errors.extend(get_errors_from_single_artifact(a__     ,      job_links=a__ ) )

						return errors


def        UpperCAmelCase      (					a__     ,      a__=None ):





						'''simple docstring'''






						lowerCAmelCase   :int						=			Counter()
						counter.update([x[1] for x in logs] )
						lowerCAmelCase   :Tuple						=			counter.most_common()
						lowerCAmelCase   :Dict						=			{}
						for error, count in counts:
												if error_filter is None or error not in error_filter:
																		lowerCAmelCase   :Optional[int]						=			{'count': count, 'failed_tests': [(x[2], x[0]) for x in logs if x[1] == error]}

						lowerCAmelCase   :Optional[Any]						=			dict(sorted(r.items()     ,      key=lambda a__ : item[1]["count"]     ,      reverse=a__ ) )
						return r


def        UpperCAmelCase      (					a__ ):





						'''simple docstring'''






						lowerCAmelCase   :Tuple						=			test.split('::' )[0]
						if test.startswith('tests/models/' ):
												lowerCAmelCase   :Union[str, Any]						=			test.split('/' )[2]
						else:
												lowerCAmelCase   :Optional[int]						=			None

						return test


def        UpperCAmelCase      (					a__     ,      a__=None ):





						'''simple docstring'''






						lowerCAmelCase   :str						=			[(x[0], x[1], get_model(x[2] )) for x in logs]
						lowerCAmelCase   :Any						=			[x for x in logs if x[2] is not None]
						lowerCAmelCase   :Tuple						=			{x[2] for x in logs}

						lowerCAmelCase   :Optional[Any]						=			{}
						for test in tests:
												lowerCAmelCase   :Union[str, Any]						=			Counter()
												# count by errors in `test`
												counter.update([x[1] for x in logs if x[2] == test] )
												lowerCAmelCase   :List[str]						=			counter.most_common()
												lowerCAmelCase   :str						=			{error: count for error, count in counts if (error_filter is None or error not in error_filter)}
												lowerCAmelCase   :Optional[Any]						=			sum(error_counts.values() )
												if n_errors > 0:
																		lowerCAmelCase   :Dict						=			{'count': n_errors, 'errors': error_counts}

						lowerCAmelCase   :Dict						=			dict(sorted(r.items()     ,      key=lambda a__ : item[1]["count"]     ,      reverse=a__ ) )
						return r


def        UpperCAmelCase      (					a__ ):





						'''simple docstring'''






						lowerCAmelCase   :Any						=			'| no. | error | status |'
						lowerCAmelCase   :Optional[int]						=			'|-:|:-|:-|'
						lowerCAmelCase   :Tuple						=			[header, sep]
						for error in reduced_by_error:
												lowerCAmelCase   :Optional[Any]						=			reduced_by_error[error]['count']
												lowerCAmelCase   :List[str]						=			F"""| {count} | {error[:1_00]} |  |"""
												lines.append(a__ )

						return "\n".join(a__ )


def        UpperCAmelCase      (					a__ ):





						'''simple docstring'''






						lowerCAmelCase   :Any						=			'| model | no. of errors | major error | count |'
						lowerCAmelCase   :int						=			'|-:|-:|-:|-:|'
						lowerCAmelCase   :int						=			[header, sep]
						for model in reduced_by_model:
												lowerCAmelCase   :Dict						=			reduced_by_model[model]['count']
												lowerCAmelCase	, lowerCAmelCase   :Any						=			list(reduced_by_model[model]['errors'].items() )[0]
												lowerCAmelCase   :Any						=			F"""| {model} | {count} | {error[:60]} | {_count} |"""
												lines.append(a__ )

						return "\n".join(a__ )


if __name__ == "__main__":
		__SCREAMING_SNAKE_CASE   =   argparse.ArgumentParser()
		# Required parameters
		parser.add_argument('--workflow_run_id', type=str, required=True, help='A GitHub Actions workflow run id.')
		parser.add_argument(
		    '--output_dir',
		    type=str,
		    required=True,
		    help='Where to store the downloaded artifacts and other result files.',
		)
		parser.add_argument('--token', default=None, type=str, help='A token that has actions:read permission.')
		__SCREAMING_SNAKE_CASE   =   parser.parse_args()

		os.makedirs(args.output_dir, exist_ok=True)

		__SCREAMING_SNAKE_CASE   =   get_job_links(args.workflow_run_id, token=args.token)
		__SCREAMING_SNAKE_CASE   =   {}
		# To deal with `workflow_call` event, where a job name is the combination of the job names in the caller and callee.
		# For example, `PyTorch 1.11 / Model tests (models/albert, single-gpu)`.
		if _job_links:
				for k, v in _job_links.items():
						# This is how GitHub actions combine job names.
						if " / " in k:
								__SCREAMING_SNAKE_CASE   =   k.find(' / ')
								__SCREAMING_SNAKE_CASE   =   k[index + len(' / ') :]
						__SCREAMING_SNAKE_CASE   =   v
		with open(os.path.join(args.output_dir, 'job_links.json'), 'w', encoding='UTF-8') as fp:
				json.dump(job_links, fp, ensure_ascii=False, indent=4)

		__SCREAMING_SNAKE_CASE   =   get_artifacts_links(args.workflow_run_id, token=args.token)
		with open(os.path.join(args.output_dir, 'artifacts.json'), 'w', encoding='UTF-8') as fp:
				json.dump(artifacts, fp, ensure_ascii=False, indent=4)

		for idx, (name, url) in enumerate(artifacts.items()):
				download_artifact(name, url, args.output_dir, args.token)
				# Be gentle to GitHub
				time.sleep(1)

		__SCREAMING_SNAKE_CASE   =   get_all_errors(args.output_dir, job_links=job_links)

		# `e[1]` is the error
		__SCREAMING_SNAKE_CASE   =   Counter()
		counter.update([e[1] for e in errors])

		# print the top 30 most common test errors
		__SCREAMING_SNAKE_CASE   =   counter.most_common(30)
		for item in most_common:
				print(item)

		with open(os.path.join(args.output_dir, 'errors.json'), 'w', encoding='UTF-8') as fp:
				json.dump(errors, fp, ensure_ascii=False, indent=4)

		__SCREAMING_SNAKE_CASE   =   reduce_by_error(errors)
		__SCREAMING_SNAKE_CASE   =   reduce_by_model(errors)

		__SCREAMING_SNAKE_CASE   =   make_github_table(reduced_by_error)
		__SCREAMING_SNAKE_CASE   =   make_github_table_per_model(reduced_by_model)

		with open(os.path.join(args.output_dir, 'reduced_by_error.txt'), 'w', encoding='UTF-8') as fp:
				fp.write(sa)
		with open(os.path.join(args.output_dir, 'reduced_by_model.txt'), 'w', encoding='UTF-8') as fp:
				fp.write(sa)