





import collections
import json
import math
import os
import re
import time
from fnmatch import fnmatch
from typing import Dict

import requests
from slack_sdk import WebClient


lowercase_		:					Dict      						=    WebClient(token=os.environ['''CI_SLACK_BOT_TOKEN'''])

def    A__(		__lowerCAmelCase			):
				_snake_case			:      List[str]	    = test_results.split(' '			)

				_snake_case			:      str	    = 0
				_snake_case			:      str	    = 0

				# When the output is short enough, the output is surrounded by = signs: "== OUTPUT =="
				# When it is too long, those signs are not present.
				_snake_case			:      int	    = expressions[-2] if '=' in expressions[-1] else expressions[-1]

				for i, expression in enumerate(lowerCAmelCase_			):
								if "failed" in expression:
												failed += int(expressions[i - 1]			)
								if "passed" in expression:
												success += int(expressions[i - 1]			)

				return failed, success, time_spent

def    A__(		__lowerCAmelCase			):
				_snake_case			:      List[str]	    = {}
				_snake_case			:      int	    = None
				_snake_case			:      str	    = False
				for line in failures_short_lines.split('\n'			):
								if re.search(R'_ \[doctest\]'						,      lowerCAmelCase_			):
												_snake_case			:      List[Any]	    = True
												_snake_case			:      Dict	    = line.split(' '			)[2]
								elif in_error and not line.split(' '			)[0].isdigit():
												_snake_case			:      Optional[Any]	    = line
												_snake_case			:      List[str]	    = False

				return failures


class       lowercase					:



						"""simple docstring"""

						def __init__(     self      :		Dict			,					lowerCamelCase_      :		str			,					lowerCamelCase_      :		Dict       ):





										'''simple docstring'''







										_snake_case			:      Optional[Any]	    = title

										_snake_case			:      str	    = doc_test_results['time_spent'].split(','       )[0]
										_snake_case			:      Union[str, Any]	    = doc_test_results['success']
										_snake_case			:      int	    = doc_test_results['failures']
										_snake_case			:      Dict	    = self.n_success + self.n_failures

										# Failures and success of the modeling tests
										_snake_case			:      Tuple	    = doc_test_results







						@property
						def 		__UpperCAmelCase							(     self      :		Dict       ):





										'''simple docstring'''







										_snake_case			:      Union[str, Any]	    = [self._time_spent]
										_snake_case			:      Tuple	    = 0

										for time in time_spent:
														_snake_case			:      str	    = time.split(':'       )

														# Time can be formatted as xx:xx:xx, as .xx, or as x.xx if the time spent was less than a minute.
														if len(_lowerCAmelCase       ) == 1:
																		_snake_case			:      List[Any]	    = [0, 0, time_parts[0]]

														_snake_case    ,      _snake_case    ,      _snake_case			:      Optional[int]	    = int(time_parts[0]       ), int(time_parts[1]       ), float(time_parts[2]       )
														total_secs += hours * 36_00 + minutes * 60 + seconds

										_snake_case    ,      _snake_case    ,      _snake_case			:      List[Any]	    = total_secs // 36_00, (total_secs % 36_00) // 60, total_secs % 60
										return f'''{int(_lowerCAmelCase       )}h{int(_lowerCAmelCase       )}m{int(_lowerCAmelCase       )}s'''







						@property
						def 		__UpperCAmelCase							(     self      :		Union[str, Any]       ):





										'''simple docstring'''







										return {"type": "header", "text": {"type": "plain_text", "text": self.title}}







						@property
						def 		__UpperCAmelCase							(     self      :		str       ):





										'''simple docstring'''







										return {
										    "type": "section",
										    "text": {
										        "type": "plain_text",
										        "text": f'''🌞 There were no failures: all {self.n_tests} tests passed. The suite ran in {self.time}.''',
										        "emoji": True,
										    },
										    "accessory": {
										        "type": "button",
										        "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
										        "url": f'''https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}''',
										    },
										}







						@property
						def 		__UpperCAmelCase							(     self      :		Tuple       ):





										'''simple docstring'''







										return {
										    "type": "section",
										    "text": {
										        "type": "plain_text",
										        "text": (
										            f'''There were {self.n_failures} failures, out of {self.n_tests} tests.\nThe suite ran in'''
										            f''' {self.time}.'''
										        ),
										        "emoji": True,
										    },
										    "accessory": {
										        "type": "button",
										        "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
										        "url": f'''https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}''',
										    },
										}







						@property
						def 		__UpperCAmelCase							(     self      :		Union[str, Any]       ):





										'''simple docstring'''







										_snake_case			:      int	    = 40
										_snake_case			:      int	    = {k: v['failed'] for k, v in doc_test_results.items() if isinstance(_lowerCAmelCase			,					_lowerCAmelCase       )}

										_snake_case			:      int	    = ''
										for category, failures in category_failures.items():
														if len(_lowerCAmelCase       ) == 0:
																		continue

														if report != "":
																		report += "\n\n"

														report += f'''*{category} failures*:'''.ljust(line_length // 2       ).rjust(line_length // 2       ) + "\n"
														report += "`"
														report += "`\n`".join(_lowerCAmelCase       )
														report += "`"

										return {
										    "type": "section",
										    "text": {
										        "type": "mrkdwn",
										        "text": f'''The following examples had failures:\n\n\n{report}\n''',
										    },
										}







						@property
						def 		__UpperCAmelCase							(     self      :		Optional[Any]       ):





										'''simple docstring'''







										_snake_case			:      Any	    = [self.header]

										if self.n_failures > 0:
														blocks.append(self.failures       )

										if self.n_failures > 0:
														blocks.extend([self.category_failures]       )

										if self.n_failures == 0:
														blocks.append(self.no_failures       )

										return json.dumps(_lowerCAmelCase       )







						@staticmethod
						def 		__UpperCAmelCase							(     ):





										'''simple docstring'''







										_snake_case			:      List[str]	    = [
										    {
										        'type': 'section',
										        'text': {
										            'type': 'plain_text',
										            'text': 'There was an issue running the tests.',
										        },
										        'accessory': {
										            'type': 'button',
										            'text': {'type': 'plain_text', 'text': 'Check Action results', 'emoji': True},
										            'url': f'''https://github.com/huggingface/transformers/actions/runs/{os.environ['GITHUB_RUN_ID']}''',
										        },
										    }
										]

										print('Sending the following payload'       )
										print(json.dumps({'blocks': json.loads(_lowerCAmelCase       )}       )       )

										client.chat_postMessage(
										    channel=os.environ['CI_SLACK_CHANNEL_ID_DAILY']			,					text='There was an issue running the tests.'			,					blocks=_lowerCAmelCase			,					)







						def 		__UpperCAmelCase							(     self      :		int       ):





										'''simple docstring'''







										print('Sending the following payload'       )
										print(json.dumps({'blocks': json.loads(self.payload       )}       )       )

										_snake_case			:      List[Any]	    = f'''{self.n_failures} failures out of {self.n_tests} tests,''' if self.n_failures else 'All tests passed.'

										_snake_case			:      str	    = client.chat_postMessage(
										    channel=os.environ['CI_SLACK_CHANNEL_ID_DAILY']			,					blocks=self.payload			,					text=_lowerCAmelCase			,					)







						def 		__UpperCAmelCase							(     self      :		str			,					lowerCamelCase_      :		int			,					lowerCamelCase_      :		int			,					lowerCamelCase_      :		Optional[int]			,					lowerCamelCase_      :		Tuple       ):





										'''simple docstring'''







										_snake_case			:      int	    = ''
										for key, value in failures.items():
														_snake_case			:      Union[str, Any]	    = value[:2_00] + ' [Truncated]' if len(_lowerCAmelCase       ) > 2_50 else value
														failures_text += f'''*{key}*\n_{value}_\n\n'''

										_snake_case			:      List[str]	    = job_name
										_snake_case			:      Union[str, Any]	    = {'type': 'section', 'text': {'type': 'mrkdwn', 'text': text}}

										if job_link is not None:
														_snake_case			:      int	    = {
														    'type': 'button',
														    'text': {'type': 'plain_text', 'text': 'GitHub Action job', 'emoji': True},
														    'url': job_link,
														}

										return [
										    {"type": "header", "text": {"type": "plain_text", "text": title.upper(), "emoji": True}},
										    content,
										    {"type": "section", "text": {"type": "mrkdwn", "text": failures_text}},
										]







						def 		__UpperCAmelCase							(     self      :		Union[str, Any]       ):





										'''simple docstring'''







										if self.thread_ts is None:
														raise ValueError('Can only post reply if a post has been made.'       )

										_snake_case			:      Optional[int]	    = self.doc_test_results.pop('job_link'       )
										self.doc_test_results.pop('failures'       )
										self.doc_test_results.pop('success'       )
										self.doc_test_results.pop('time_spent'       )

										_snake_case			:      int	    = sorted(self.doc_test_results.items()			,					key=lambda lowerCamelCase_       : t[0]       )
										for job, job_result in sorted_dict:
														if len(job_result['failures']       ):
																		_snake_case			:      List[Any]	    = f'''*Num failures* :{len(job_result['failed']       )} \n'''
																		_snake_case			:      Dict	    = job_result['failures']
																		_snake_case			:      List[Any]	    = self.get_reply_blocks(_lowerCAmelCase			,					_lowerCAmelCase			,					_lowerCAmelCase			,					text=_lowerCAmelCase       )

																		print('Sending the following reply'       )
																		print(json.dumps({'blocks': blocks}       )       )

																		client.chat_postMessage(
																		    channel=os.environ['CI_SLACK_CHANNEL_ID_DAILY']			,					text=f'''Results for {job}'''			,					blocks=_lowerCAmelCase			,					thread_ts=self.thread_ts['ts']			,					)

																		time.sleep(1       )



def    A__(		):
				_snake_case			:      Optional[int]	    = os.environ['GITHUB_RUN_ID']
				_snake_case			:      int	    = F'''https://api.github.com/repos/huggingface/transformers/actions/runs/{run_id}/jobs?per_page=100'''
				_snake_case			:      List[str]	    = requests.get(lowerCAmelCase_			).json()
				_snake_case			:      Dict	    = {}

				try:
								jobs.update({job['name']: job['html_url'] for job in result['jobs']}			)
								_snake_case			:      List[Any]	    = math.ceil((result['total_count'] - 1_00) / 1_00			)

								for i in range(lowerCAmelCase_			):
												_snake_case			:      Any	    = requests.get(url + F'''&page={i + 2}'''			).json()
												jobs.update({job['name']: job['html_url'] for job in result['jobs']}			)

								return jobs
				except Exception as e:
								print('Unknown error, could not fetch links.'						,      lowerCAmelCase_			)

				return {}

def    A__(		__lowerCAmelCase			):
				_snake_case			:      List[str]	    = {}

				if os.path.exists(lowerCAmelCase_			):
								_snake_case			:      Union[str, Any]	    = os.listdir(lowerCAmelCase_			)
								for file in files:
												try:
																with open(os.path.join(lowerCAmelCase_						,      lowerCAmelCase_			)						,      encoding='utf-8'			) as f:
																				_snake_case			:      List[str]	    = f.read()
												except UnicodeDecodeError as e:
																raise ValueError(F'''Could not open {os.path.join(lowerCAmelCase_						,      lowerCAmelCase_			)}.'''			) from e

				return _artifact

def    A__(		):


				class       lowercase					:



										"""simple docstring"""

										def __init__(     self      :		List[Any]			,					lowerCamelCase_      :		str       ):





														'''simple docstring'''







														_snake_case			:      int	    = name
														_snake_case			:      int	    = []







										def __str__(     self      :		Optional[Any]       ):





														'''simple docstring'''







														return self.name







										def 		__UpperCAmelCase							(     self      :		int			,					lowerCamelCase_      :		str       ):





														'''simple docstring'''







														self.paths.append({'name': self.name, 'path': path}       )



				_snake_case			:      List[str]	    = {}

				_snake_case			:      int	    = filter(os.path.isdir						,      os.listdir()			)
				for directory in directories:
								_snake_case			:      Union[str, Any]	    = directory
								if artifact_name not in _available_artifacts:
												_snake_case			:      str	    = Artifact(lowerCAmelCase_			)

												_available_artifacts[artifact_name].add_path(lowerCAmelCase_			)

				return _available_artifacts


if __name__ == "__main__":
		lowercase_		:					List[Any]      						=    get_job_links()
		lowercase_		:					str      						=    retrieve_available_artifacts()

		lowercase_		:					List[Any]      						=    collections.OrderedDict(
		    [
		        ('''*.py''', '''API Examples'''),
		        ('''*.md''', '''MD Examples'''),
		    ]
		)

		# This dict will contain all the information relative to each doc test category:
		# - failed: list of failed tests
		# - failures: dict in the format 'test': 'error_message'
		lowercase_		:					Optional[Any]      						=    {
		    v: {
		        '''failed''': [],
		        '''failures''': {},
		    }
		    for v in docs.values()
		}

		# Link to the GitHub Action job
		lowercase_		:					Dict      						=    github_actions_job_links.get('''run_doctests''')

		lowercase_		:					Any      						=    available_artifacts['''doc_tests_gpu_test_reports'''].paths[0]
		lowercase_		:					str      						=    retrieve_artifact(artifact_path['''name'''])
		if "stats" in artifact:
				lowercase_		:					Optional[Any]      						=    handle_test_results(artifact['''stats'''])
				lowercase_		:					Optional[int]      						=    failed
				lowercase_		:					int      						=    success
				lowercase_		:					str      						=    time_spent[1:-1] + ''', '''

				lowercase_		:					int      						=    extract_first_line_failure(artifact['''failures_short'''])
				for line in artifact["summary_short"].split('''\n'''):
						if re.search('''FAILED''', line):
								lowercase_		:					str      						=    line.replace('''FAILED ''', '''''')
								lowercase_		:					Tuple      						=    line.split()[0].replace('''\n''', '''''')

								if "::" in line:
										lowercase_		:					int      						=    line.split('''::''')
								else:
										lowercase_		:					str      						=    line, line

								for file_regex in docs.keys():
										if fnmatch(file_path, file_regex):
												lowercase_		:					List[Any]      						=    docs[file_regex]
												doc_test_results[category]["failed"].append(test)

												lowercase_		:					int      						=    all_failures[test] if test in all_failures else '''N/A'''
												lowercase_		:					str      						=    failure
												break

		lowercase_		:					Optional[Any]      						=    Message('''🤗 Results of the doc tests.''', doc_test_results)
		message.post()
		message.post_reply()










import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
   import torch_xla.core.xla_model as xm
   import torch_xla.debug.metrics as met


class      __lowerCAmelCase  (			a       ):






  """simple docstring"""





  def __init__(  self							:							Union[str, Any]     ,					*_lowerCAmelCase							:							Dict     ,					_lowerCAmelCase							:							List[Any]=None     ,					_lowerCAmelCase							:							str=None     ,					**_lowerCAmelCase							:							str	) ->			Dict:




     """simple docstring"""
     super().__init__(*_lowerCAmelCase     ,					**_lowerCAmelCase	)
     snake_case_            =       eval_examples
     snake_case_            =       post_process_function



  def        lowerCAmelCase__	(  self							:							Union[str, Any]     ,					_lowerCAmelCase							:							Optional[Dataset] = None     ,					_lowerCAmelCase							:							Any=None     ,					_lowerCAmelCase							:							Optional[List[str]] = None     ,					_lowerCAmelCase							:							str = "eval"     ,					**_lowerCAmelCase							:							Tuple     ,					) ->			Dict[str, float]:




     """simple docstring"""
     snake_case_            =       gen_kwargs.copy()
     snake_case_            =       (
         gen_kwargs["max_length"] if gen_kwargs.get("max_length"	) is not None else self.args.generation_max_length
     )
     snake_case_            =       (
         gen_kwargs["num_beams"] if gen_kwargs.get("num_beams"	) is not None else self.args.generation_num_beams
     )
     snake_case_            =       gen_kwargs

     snake_case_            =       self.eval_dataset if eval_dataset is None else eval_dataset
     snake_case_            =       self.get_eval_dataloader(_lowerCAmelCase	)
     snake_case_            =       self.eval_examples if eval_examples is None else eval_examples

     # Temporarily disable metric computation, we will do it in the loop here.
     snake_case_            =       self.compute_metrics
     snake_case_            =       None
     snake_case_            =       time.time()
     snake_case_            =       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
     try:
        snake_case_            =       eval_loop(
            _lowerCAmelCase     ,					description="Evaluation"     ,					prediction_loss_only=True if compute_metrics is None else None     ,					ignore_keys=_lowerCAmelCase     ,					metric_key_prefix=_lowerCAmelCase     ,					)
     finally:
        snake_case_            =       compute_metrics
     snake_case_            =       self.args.eval_batch_size * self.args.world_size
     if F'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
        start_time += output.metrics[F'''{metric_key_prefix}_jit_compilation_time''']
     output.metrics.update(
         speed_metrics(
             _lowerCAmelCase     ,					_lowerCAmelCase     ,					num_samples=output.num_samples     ,					num_steps=math.ceil(output.num_samples / total_batch_size	)     ,					)	)

     if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
        # Only the main node write the results by default
        snake_case_            =       self.post_process_function(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase	)
        snake_case_            =       self.compute_metrics(_lowerCAmelCase	)

        # Prefix all keys with metric_key_prefix + '_'
        for key in list(metrics.keys()	):
           if not key.startswith(F'''{metric_key_prefix}_'''	):
              snake_case_            =       metrics.pop(_lowerCAmelCase	)

        metrics.update(output.metrics	)
     else:
        snake_case_            =       output.metrics

     if self.args.should_log:
        # Only the main node log the results by default
        self.log(_lowerCAmelCase	)

     if self.args.tpu_metrics_debug or self.args.debug:
        # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
        xm.master_print(met.metrics_report()	)

     snake_case_            =       self.callback_handler.on_evaluate(self.args     ,					self.state     ,					self.control     ,					_lowerCAmelCase	)
     return metrics



  def        lowerCAmelCase__	(  self							:							Optional[Any]     ,					_lowerCAmelCase							:							Optional[Any]     ,					_lowerCAmelCase							:							Union[str, Any]     ,					_lowerCAmelCase							:							List[str]=None     ,					_lowerCAmelCase							:							str = "test"     ,					**_lowerCAmelCase							:							Optional[Any]	) ->			Union[str, Any]:




     """simple docstring"""
     snake_case_            =       gen_kwargs.copy()

     snake_case_            =       self.get_test_dataloader(_lowerCAmelCase	)

     # Temporarily disable metric computation, we will do it in the loop here.
     snake_case_            =       self.compute_metrics
     snake_case_            =       None
     snake_case_            =       time.time()
     snake_case_            =       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
     try:
        snake_case_            =       eval_loop(
            _lowerCAmelCase     ,					description="Prediction"     ,					prediction_loss_only=True if compute_metrics is None else None     ,					ignore_keys=_lowerCAmelCase     ,					metric_key_prefix=_lowerCAmelCase     ,					)
     finally:
        snake_case_            =       compute_metrics

     snake_case_            =       self.args.eval_batch_size * self.args.world_size
     if F'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
        start_time += output.metrics[F'''{metric_key_prefix}_jit_compilation_time''']
     output.metrics.update(
         speed_metrics(
             _lowerCAmelCase     ,					_lowerCAmelCase     ,					num_samples=output.num_samples     ,					num_steps=math.ceil(output.num_samples / total_batch_size	)     ,					)	)
     if self.post_process_function is None or self.compute_metrics is None:
        return output

     snake_case_            =       self.post_process_function(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase     ,					"predict"	)
     snake_case_            =       self.compute_metrics(_lowerCAmelCase	)

     # Prefix all keys with metric_key_prefix + '_'
     for key in list(metrics.keys()	):
        if not key.startswith(F'''{metric_key_prefix}_'''	):
           snake_case_            =       metrics.pop(_lowerCAmelCase	)
     metrics.update(output.metrics	)
     return PredictionOutput(predictions=predictions.predictions     ,					label_ids=predictions.label_ids     ,					metrics=_lowerCAmelCase	)




