


'''simple docstring'''







import argparse
import json
import math
import os
import time
import traceback
import zipfile
from collections import Counter

import requests






def       lowercase					(_A ,    _A=None       ):







       """simple docstring"""







       _lowerCAmelCase     :				Optional[int]       =							None
       if token is not None:
              _lowerCAmelCase     :				Optional[int]       =							{'Accept': 'application/vnd.github+json', 'Authorization': f'Bearer {token}'}

       _lowerCAmelCase     :				Tuple       =							f'https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id}/jobs?per_page=100'
       _lowerCAmelCase     :				int       =							requests.get(_snake_case ,    headers=_snake_case       ).json()
       _lowerCAmelCase     :				Optional[int]       =							{}

       try:
              job_links.update({job['name']: job['html_url'] for job in result['jobs']}       )
              _lowerCAmelCase     :				Optional[int]       =							math.ceil((result['total_count'] - 1_0_0) / 1_0_0       )

              for i in range(_snake_case       ):
                     _lowerCAmelCase     :				List[str]       =							requests.get(url + f'&page={i + 2}' ,    headers=_snake_case       ).json()
                     job_links.update({job['name']: job['html_url'] for job in result['jobs']}       )

              return job_links
       except Exception:
              print(f'Unknown error, could not fetch links:\n{traceback.format_exc()}'       )

       return {}






def       lowercase					(_A ,    _A=None       ):







       """simple docstring"""







       _lowerCAmelCase     :				Any       =							None
       if token is not None:
              _lowerCAmelCase     :				Optional[int]       =							{'Accept': 'application/vnd.github+json', 'Authorization': f'Bearer {token}'}

       _lowerCAmelCase     :				List[Any]       =							f'https://api.github.com/repos/huggingface/transformers/actions/runs/{worflow_run_id}/artifacts?per_page=100'
       _lowerCAmelCase     :				int       =							requests.get(_snake_case ,    headers=_snake_case       ).json()
       _lowerCAmelCase     :				Optional[Any]       =							{}

       try:
              artifacts.update({artifact['name']: artifact['archive_download_url'] for artifact in result['artifacts']}       )
              _lowerCAmelCase     :				str       =							math.ceil((result['total_count'] - 1_0_0) / 1_0_0       )

              for i in range(_snake_case       ):
                     _lowerCAmelCase     :				Union[str, Any]       =							requests.get(url + f'&page={i + 2}' ,    headers=_snake_case       ).json()
                     artifacts.update({artifact['name']: artifact['archive_download_url'] for artifact in result['artifacts']}       )

              return artifacts
       except Exception:
              print(f'Unknown error, could not fetch links:\n{traceback.format_exc()}'       )

       return {}






def       lowercase					(_A ,    _A ,    _A ,    _A       ):







       """simple docstring"""







       _lowerCAmelCase     :				List[Any]       =							None
       if token is not None:
              _lowerCAmelCase     :				Optional[int]       =							{'Accept': 'application/vnd.github+json', 'Authorization': f'Bearer {token}'}

       _lowerCAmelCase     :				Optional[Any]       =							requests.get(_snake_case ,    headers=_snake_case ,    allow_redirects=_snake_case       )
       _lowerCAmelCase     :				List[str]       =							result.headers['Location']
       _lowerCAmelCase     :				Union[str, Any]       =							requests.get(_snake_case ,    allow_redirects=_snake_case       )
       _lowerCAmelCase     :				int       =							os.path.join(_snake_case ,    f'{artifact_name}.zip'       )
       with open(_snake_case ,    'wb'       ) as fp:
              fp.write(response.content       )






def       lowercase					(_A ,    _A=None       ):







       """simple docstring"""







       _lowerCAmelCase     :				Union[str, Any]       =							[]
       _lowerCAmelCase     :				str       =							[]
       _lowerCAmelCase     :				List[str]       =							None

       with zipfile.ZipFile(_snake_case       ) as z:
              for filename in z.namelist():
                     if not os.path.isdir(_snake_case       ):
                            # read the file
                            if filename in ["failures_line.txt", "summary_short.txt", "job_name.txt"]:
                                   with z.open(_snake_case       ) as f:
                                          for line in f:
                                                 _lowerCAmelCase     :				Any       =							line.decode('UTF-8'       ).strip()
                                                 if filename == "failures_line.txt":
                                                        try:
                                                               # `error_line` is the place where `error` occurs
                                                               _lowerCAmelCase     :				int       =							line[: line.index(': '       )]
                                                               _lowerCAmelCase     :				Any       =							line[line.index(': '       ) + len(': '       ) :]
                                                               errors.append([error_line, error]       )
                                                        except Exception:
                                                               # skip un-related lines
                                                               pass
                                                 elif filename == "summary_short.txt" and line.startswith('FAILED '       ):
                                                        # `test` is the test method that failed
                                                        _lowerCAmelCase     :				List[str]       =							line[len('FAILED '       ) :]
                                                        failed_tests.append(_snake_case       )
                                                 elif filename == "job_name.txt":
                                                        _lowerCAmelCase     :				Tuple       =							line

       if len(_snake_case       ) != len(_snake_case       ):
              raise ValueError(
                  f'`errors` and `failed_tests` should have the same number of elements. Got {len(_snake_case       )} for `errors` '
                  f'and {len(_snake_case       )} for `failed_tests` instead. The test reports in {artifact_zip_path} have some'
                  ' problem.'       )

       _lowerCAmelCase     :				List[str]       =							None
       if job_name and job_links:
              _lowerCAmelCase     :				List[Any]       =							job_links.get(_snake_case ,    _snake_case       )

       # A list with elements of the form (line of error, error, failed test)
       _lowerCAmelCase     :				List[str]       =							[x + [y] + [job_link] for x, y in zip(_snake_case ,    _snake_case       )]

       return result






def       lowercase					(_A ,    _A=None       ):







       """simple docstring"""







       _lowerCAmelCase     :				str       =							[]

       _lowerCAmelCase     :				Optional[Any]       =							[os.path.join(_snake_case ,    _snake_case       ) for p in os.listdir(_snake_case       ) if p.endswith('.zip'       )]
       for p in paths:
              errors.extend(get_errors_from_single_artifact(_snake_case ,    job_links=_snake_case       )       )

       return errors






def       lowercase					(_A ,    _A=None       ):







       """simple docstring"""







       _lowerCAmelCase     :				List[Any]       =							Counter()
       counter.update([x[1] for x in logs]       )
       _lowerCAmelCase     :				Optional[int]       =							counter.most_common()
       _lowerCAmelCase     :				Any       =							{}
       for error, count in counts:
              if error_filter is None or error not in error_filter:
                     _lowerCAmelCase     :				Dict       =							{'count': count, 'failed_tests': [(x[2], x[0]) for x in logs if x[1] == error]}

       _lowerCAmelCase     :				Union[str, Any]       =							dict(sorted(r.items() ,    key=lambda _A       : item[1]["count"] ,    reverse=_snake_case       )       )
       return r






def       lowercase					(_A       ):







       """simple docstring"""







       _lowerCAmelCase     :				Optional[Any]       =							test.split('::'       )[0]
       if test.startswith('tests/models/'       ):
              _lowerCAmelCase     :				Dict       =							test.split('/'       )[2]
       else:
              _lowerCAmelCase     :				Union[str, Any]       =							None

       return test






def       lowercase					(_A ,    _A=None       ):







       """simple docstring"""







       _lowerCAmelCase     :				List[str]       =							[(x[0], x[1], get_model(x[2]       )) for x in logs]
       _lowerCAmelCase     :				Optional[int]       =							[x for x in logs if x[2] is not None]
       _lowerCAmelCase     :				int       =							{x[2] for x in logs}

       _lowerCAmelCase     :				List[Any]       =							{}
       for test in tests:
              _lowerCAmelCase     :				List[Any]       =							Counter()
              # count by errors in `test`
              counter.update([x[1] for x in logs if x[2] == test]       )
              _lowerCAmelCase     :				Optional[Any]       =							counter.most_common()
              _lowerCAmelCase     :				str       =							{error: count for error, count in counts if (error_filter is None or error not in error_filter)}
              _lowerCAmelCase     :				Union[str, Any]       =							sum(error_counts.values()       )
              if n_errors > 0:
                     _lowerCAmelCase     :				Optional[int]       =							{'count': n_errors, 'errors': error_counts}

       _lowerCAmelCase     :				Optional[Any]       =							dict(sorted(r.items() ,    key=lambda _A       : item[1]["count"] ,    reverse=_snake_case       )       )
       return r






def       lowercase					(_A       ):







       """simple docstring"""







       _lowerCAmelCase     :				List[str]       =							'| no. | error | status |'
       _lowerCAmelCase     :				Any       =							'|-:|:-|:-|'
       _lowerCAmelCase     :				Optional[Any]       =							[header, sep]
       for error in reduced_by_error:
              _lowerCAmelCase     :				List[Any]       =							reduced_by_error[error]['count']
              _lowerCAmelCase     :				int       =							f'| {count} | {error[:1_0_0]} |  |'
              lines.append(_snake_case       )

       return "\n".join(_snake_case       )






def       lowercase					(_A       ):







       """simple docstring"""







       _lowerCAmelCase     :				Any       =							'| model | no. of errors | major error | count |'
       _lowerCAmelCase     :				List[str]       =							'|-:|-:|-:|-:|'
       _lowerCAmelCase     :				str       =							[header, sep]
       for model in reduced_by_model:
              _lowerCAmelCase     :				int       =							reduced_by_model[model]['count']
              _lowerCAmelCase      ,  _lowerCAmelCase     :				Optional[Any]       =							list(reduced_by_model[model]['errors'].items()       )[0]
              _lowerCAmelCase     :				Tuple       =							f'| {model} | {count} | {error[:6_0]} | {_count} |'
              lines.append(_snake_case       )

       return "\n".join(_snake_case       )


if __name__ == "__main__":
       lowerCAmelCase     :  Any											=    argparse.ArgumentParser()
       # Required parameters
       parser.add_argument("""--workflow_run_id""", type=str, required=True, help="""A GitHub Actions workflow run id.""")
       parser.add_argument(
           """--output_dir""",
           type=str,
           required=True,
           help="""Where to store the downloaded artifacts and other result files.""",
       )
       parser.add_argument("""--token""", default=None, type=str, help="""A token that has actions:read permission.""")
       lowerCAmelCase     :  Optional[Any]											=    parser.parse_args()

       os.makedirs(args.output_dir, exist_ok=True)

       lowerCAmelCase     :  Union[str, Any]											=    get_job_links(args.workflow_run_id, token=args.token)
       lowerCAmelCase     :  Dict											=    {}
       # To deal with `workflow_call` event, where a job name is the combination of the job names in the caller and callee.
       # For example, `PyTorch 1.11 / Model tests (models/albert, single-gpu)`.
       if _job_links:
              for k, v in _job_links.items():
                     # This is how GitHub actions combine job names.
                     if " / " in k:
                            lowerCAmelCase     :  Tuple											=    k.find(""" / """)
                            lowerCAmelCase     :  Union[str, Any]											=    k[index + len(""" / """) :]
                     lowerCAmelCase     :  Union[str, Any]											=    v
       with open(os.path.join(args.output_dir, """job_links.json"""), """w""", encoding="""UTF-8""") as fp:
              json.dump(job_links, fp, ensure_ascii=False, indent=4)

       lowerCAmelCase     :  List[str]											=    get_artifacts_links(args.workflow_run_id, token=args.token)
       with open(os.path.join(args.output_dir, """artifacts.json"""), """w""", encoding="""UTF-8""") as fp:
              json.dump(artifacts, fp, ensure_ascii=False, indent=4)

       for idx, (name, url) in enumerate(artifacts.items()):
              download_artifact(name, url, args.output_dir, args.token)
              # Be gentle to GitHub
              time.sleep(1)

       lowerCAmelCase     :  Tuple											=    get_all_errors(args.output_dir, job_links=job_links)

       # `e[1]` is the error
       lowerCAmelCase     :  Optional[Any]											=    Counter()
       counter.update([e[1] for e in errors])

       # print the top 30 most common test errors
       lowerCAmelCase     :  Optional[int]											=    counter.most_common(30)
       for item in most_common:
              print(item)

       with open(os.path.join(args.output_dir, """errors.json"""), """w""", encoding="""UTF-8""") as fp:
              json.dump(errors, fp, ensure_ascii=False, indent=4)

       lowerCAmelCase     :  Optional[Any]											=    reduce_by_error(errors)
       lowerCAmelCase     :  str											=    reduce_by_model(errors)

       lowerCAmelCase     :  Optional[Any]											=    make_github_table(reduced_by_error)
       lowerCAmelCase     :  str											=    make_github_table_per_model(reduced_by_model)

       with open(os.path.join(args.output_dir, """reduced_by_error.txt"""), """w""", encoding="""UTF-8""") as fp:
              fp.write(sa)
       with open(os.path.join(args.output_dir, """reduced_by_model.txt"""), """w""", encoding="""UTF-8""") as fp:
              fp.write(sa)









import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


_SCREAMING_SNAKE_CASE   =  logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
    import torch_xla.core.xla_model as xm
    import torch_xla.debug.metrics as met






class 					SCREAMING_SNAKE_CASE_  (       _a ):
       """simple docstring"""


       def __init__(				self	:Dict,      *snake_case	:int,      snake_case	:Any=None,      snake_case	:str=None,      snake_case	:int=None,      **snake_case	:List[str]):



          """simple docstring"""

          super().__init__(*snake_case,      **snake_case)
          _lowercase              =eval_examples
          _lowercase              =post_process_function
          _lowercase              =quant_trainer_args
          _lowercase              =128  # default number of calibration samples





       def 				UpperCamelCase__						(				self	:Dict,      snake_case	:List[str]=None):



          """simple docstring"""

          if calib_dataset is None and self.calib_dataset is None:
             raise ValueError('Trainer: calibration requires an calib_dataset.')
          _lowercase              =calib_dataset if calib_dataset is not None else self.calib_dataset

          _lowercase              =self._remove_unused_columns(snake_case,      description='Calibration')

          return DataLoader(
              snake_case,      batch_size=self.args.eval_batch_size,      collate_fn=self.data_collator,      drop_last=self.args.dataloader_drop_last,      num_workers=self.args.dataloader_num_workers,      pin_memory=self.args.dataloader_pin_memory,      shuffle=snake_case,      )





       def 				UpperCamelCase__						(				self	:Any,      snake_case	:Union[str, Any]=None):



          """simple docstring"""

          _lowercase              =self.train_dataset if calib_dataset is None else calib_dataset
          _lowercase              =self.get_calib_dataloader(snake_case)

          _lowercase              =self.model
          quant_trainer.configure_model(snake_case,      self.quant_trainer_args,      calib=snake_case)
          model.eval()
          quant_trainer.enable_calibration(snake_case)

          logger.info('***** Running calibration *****')
          logger.info(f'''  Num examples = {self.calib_num}''')
          logger.info(f'''  Batch size = {calib_dataloader.batch_size}''')

          for step, inputs in enumerate(snake_case):
             # Prediction step
             _lowercase       ,       _lowercase       ,       _lowercase              =self.prediction_step(snake_case,      snake_case,      prediction_loss_only=snake_case)
             if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
                break

          quant_trainer.finish_calibration(snake_case,      self.quant_trainer_args)
          _lowercase              =model





       def 				UpperCamelCase__						(				self	:Optional[int],      snake_case	:Union[str, Any]=None,      snake_case	:Optional[int]=None,      snake_case	:Union[str, Any]=None,      snake_case	:str = "eval"):



          """simple docstring"""

          _lowercase              =self.eval_dataset if eval_dataset is None else eval_dataset
          _lowercase              =self.get_eval_dataloader(snake_case)
          _lowercase              =self.eval_examples if eval_examples is None else eval_examples

          # Temporarily disable metric computation, we will do it in the loop here.
          _lowercase              =self.compute_metrics
          _lowercase              =None
          _lowercase              =self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
          try:
             _lowercase              =eval_loop(
                 snake_case,      description='Evaluation',      prediction_loss_only=True if compute_metrics is None else None,      ignore_keys=snake_case,      )
          finally:
             _lowercase              =compute_metrics

          if self.post_process_function is not None and self.compute_metrics is not None:
             _lowercase              =self.post_process_function(snake_case,      snake_case,      output.predictions)
             _lowercase              =self.compute_metrics(snake_case)

             # Prefix all keys with metric_key_prefix + '_'
             for key in list(metrics.keys()):
                if not key.startswith(f'''{metric_key_prefix}_'''):
                   _lowercase              =metrics.pop(snake_case)

             self.log(snake_case)
          else:
             _lowercase              ={}

          if self.args.tpu_metrics_debug or self.args.debug:
             # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
             xm.master_print(met.metrics_report())

          _lowercase              =self.callback_handler.on_evaluate(self.args,      self.state,      self.control,      snake_case)
          return metrics





       def 				UpperCamelCase__						(				self	:Optional[int],      snake_case	:str,      snake_case	:Optional[Any],      snake_case	:Any=None,      snake_case	:str = "test"):



          """simple docstring"""

          _lowercase              =self.get_test_dataloader(snake_case)

          # Temporarily disable metric computation, we will do it in the loop here.
          _lowercase              =self.compute_metrics
          _lowercase              =None
          _lowercase              =self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
          try:
             _lowercase              =eval_loop(
                 snake_case,      description='Prediction',      prediction_loss_only=True if compute_metrics is None else None,      ignore_keys=snake_case,      )
          finally:
             _lowercase              =compute_metrics

          if self.post_process_function is None or self.compute_metrics is None:
             return output

          _lowercase              =self.post_process_function(snake_case,      snake_case,      output.predictions,      'predict')
          _lowercase              =self.compute_metrics(snake_case)

          # Prefix all keys with metric_key_prefix + '_'
          for key in list(metrics.keys()):
             if not key.startswith(f'''{metric_key_prefix}_'''):
                _lowercase              =metrics.pop(snake_case)

          return PredictionOutput(predictions=predictions.predictions,      label_ids=predictions.label_ids,      metrics=snake_case)





       def 				UpperCamelCase__						(				self	:List[str],      snake_case	:Dict="./"):



          """simple docstring"""

          _lowercase              =self.eval_dataset
          _lowercase              =self.get_eval_dataloader(snake_case)

          _lowercase              =next(iter(snake_case))

          # saving device - to make it consistent
          _lowercase              =torch.device('cuda' if torch.cuda.is_available() else 'cpu')

          # convert to tuple
          _lowercase              =tuple(v.to(snake_case) for k, v in batch.items())

          logger.info('Converting model to be onnx compatible')
          from pytorch_quantization.nn import TensorQuantizer

          _lowercase              =True

          _lowercase              =self.model.to(snake_case)

          model.eval()
          model.float()

          _lowercase              =model.module if hasattr(snake_case,      'module') else model
          quant_trainer.configure_model(snake_case,      self.quant_trainer_args)

          _lowercase              =os.path.join(snake_case,      'model.onnx')
          logger.info(f'''exporting model to {output_model_file}''')

          _lowercase              ={0: 'batch_size', 1: 'seq_len'}

          torch.onnx.export(
              snake_case,      snake_case,      snake_case,      export_params=snake_case,      opset_version=13,      do_constant_folding=snake_case,      input_names=['input_ids', 'attention_mask', 'token_type_ids'],      output_names=['output_start_logits', 'output_end_logits'],      dynamic_axes={
                  'input_ids': axes,
                  'attention_mask': axes,
                  'token_type_ids': axes,
                  'output_start_logits': axes,
                  'output_end_logits': axes,
              },      verbose=snake_case,      )
          logger.info('onnx export finished')


