



'''simple docstring'''



import argparse
import json
import math
import os
import time
import traceback
import zipfile
from collections import Counter

import requests
def        UpperCAmelCase (    A	:					Dict       ,		A	:					List[Any]=None					):
							SCREAMING_SNAKE_CASE       :   Any         =      None
							if token is not None:
														SCREAMING_SNAKE_CASE       :   Optional[int]         =      {'''Accept''': '''application/vnd.github+json''', '''Authorization''': F"""Bearer {token}"""}

							SCREAMING_SNAKE_CASE       :   List[Any]         =      F"""https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id}/jobs?per_page=100"""
							SCREAMING_SNAKE_CASE       :   Optional[int]         =      requests.get(A       ,		headers=A					).json()
							SCREAMING_SNAKE_CASE       :   Tuple         =      {}

							try:
														job_links.update({job['''name''']: job['''html_url'''] for job in result['''jobs''']}					)
														SCREAMING_SNAKE_CASE       :   str         =      math.ceil((result['''total_count'''] - 100) / 100					)

														for i in range(A					):
																					SCREAMING_SNAKE_CASE       :   Optional[Any]         =      requests.get(url + F"""&page={i + 2}"""       ,		headers=A					).json()
																					job_links.update({job['''name''']: job['''html_url'''] for job in result['''jobs''']}					)

														return job_links
							except Exception:
														print(F"""Unknown error, could not fetch links:\n{traceback.format_exc()}"""					)

							return {}
def        UpperCAmelCase (    A	:					Union[str, Any]       ,		A	:					Any=None					):
							SCREAMING_SNAKE_CASE       :   Any         =      None
							if token is not None:
														SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      {'''Accept''': '''application/vnd.github+json''', '''Authorization''': F"""Bearer {token}"""}

							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      F"""https://api.github.com/repos/huggingface/transformers/actions/runs/{worflow_run_id}/artifacts?per_page=100"""
							SCREAMING_SNAKE_CASE       :   Dict         =      requests.get(A       ,		headers=A					).json()
							SCREAMING_SNAKE_CASE       :   Tuple         =      {}

							try:
														artifacts.update({artifact['''name''']: artifact['''archive_download_url'''] for artifact in result['''artifacts''']}					)
														SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      math.ceil((result['''total_count'''] - 100) / 100					)

														for i in range(A					):
																					SCREAMING_SNAKE_CASE       :   Dict         =      requests.get(url + F"""&page={i + 2}"""       ,		headers=A					).json()
																					artifacts.update({artifact['''name''']: artifact['''archive_download_url'''] for artifact in result['''artifacts''']}					)

														return artifacts
							except Exception:
														print(F"""Unknown error, could not fetch links:\n{traceback.format_exc()}"""					)

							return {}
def        UpperCAmelCase (    A	:					Optional[Any]       ,		A	:					List[Any]       ,		A	:					Tuple       ,		A	:					int					):
							SCREAMING_SNAKE_CASE       :   List[str]         =      None
							if token is not None:
														SCREAMING_SNAKE_CASE       :   List[Any]         =      {'''Accept''': '''application/vnd.github+json''', '''Authorization''': F"""Bearer {token}"""}

							SCREAMING_SNAKE_CASE       :   str         =      requests.get(A       ,		headers=A       ,		allow_redirects=A					)
							SCREAMING_SNAKE_CASE       :   str         =      result.headers['''Location''']
							SCREAMING_SNAKE_CASE       :   Tuple         =      requests.get(A       ,		allow_redirects=A					)
							SCREAMING_SNAKE_CASE       :   List[Any]         =      os.path.join(A       ,		F"""{artifact_name}.zip"""					)
							with open(A       ,		'''wb'''					) as fp:
														fp.write(response.content					)
def        UpperCAmelCase (    A	:					str       ,		A	:					str=None					):
							SCREAMING_SNAKE_CASE       :   str         =      []
							SCREAMING_SNAKE_CASE       :   int         =      []
							SCREAMING_SNAKE_CASE       :   Tuple         =      None

							with zipfile.ZipFile(A					) as z:
														for filename in z.namelist():
																					if not os.path.isdir(A					):
																												# read the file
																												if filename in ["failures_line.txt", "summary_short.txt", "job_name.txt"]:
																																			with z.open(A					) as f:
																																										for line in f:
																																																	SCREAMING_SNAKE_CASE       :   Optional[int]         =      line.decode('''UTF-8'''					).strip()
																																																	if filename == "failures_line.txt":
																																																								try:
																																																															# `error_line` is the place where `error` occurs
																																																															SCREAMING_SNAKE_CASE       :   List[Any]         =      line[: line.index(''': '''					)]
																																																															SCREAMING_SNAKE_CASE       :   Dict         =      line[line.index(''': '''					) + len(''': '''					) :]
																																																															errors.append([error_line, error]					)
																																																								except Exception:
																																																															# skip un-related lines
																																																															pass
																																																	elif filename == "summary_short.txt" and line.startswith('''FAILED '''					):
																																																								# `test` is the test method that failed
																																																								SCREAMING_SNAKE_CASE       :   List[str]         =      line[len('''FAILED '''					) :]
																																																								failed_tests.append(A					)
																																																	elif filename == "job_name.txt":
																																																								SCREAMING_SNAKE_CASE       :   List[Any]         =      line

							if len(A					) != len(A					):
														raise ValueError(
														    F"""`errors` and `failed_tests` should have the same number of elements. Got {len(A					)} for `errors` """
														    F"""and {len(A					)} for `failed_tests` instead. The test reports in {artifact_zip_path} have some"""
														    ''' problem.'''					)

							SCREAMING_SNAKE_CASE       :   List[str]         =      None
							if job_name and job_links:
														SCREAMING_SNAKE_CASE       :   Optional[int]         =      job_links.get(A       ,		A					)

							# A list with elements of the form (line of error, error, failed test)
							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      [x + [y] + [job_link] for x, y in zip(A       ,		A					)]

							return result
def        UpperCAmelCase (    A	:					List[Any]       ,		A	:					List[str]=None					):
							SCREAMING_SNAKE_CASE       :   List[str]         =      []

							SCREAMING_SNAKE_CASE       :   str         =      [os.path.join(A       ,		A					) for p in os.listdir(A					) if p.endswith('''.zip'''					)]
							for p in paths:
														errors.extend(get_errors_from_single_artifact(A       ,		job_links=A					)					)

							return errors
def        UpperCAmelCase (    A	:					int       ,		A	:					List[str]=None					):
							SCREAMING_SNAKE_CASE       :   Dict         =      Counter()
							counter.update([x[1] for x in logs]					)
							SCREAMING_SNAKE_CASE       :   List[Any]         =      counter.most_common()
							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      {}
							for error, count in counts:
														if error_filter is None or error not in error_filter:
																					SCREAMING_SNAKE_CASE       :   Dict         =      {'''count''': count, '''failed_tests''': [(x[2], x[0]) for x in logs if x[1] == error]}

							SCREAMING_SNAKE_CASE       :   Tuple         =      dict(sorted(r.items()       ,		key=lambda A					: item[1]["count"]       ,		reverse=A					)					)
							return r
def        UpperCAmelCase (    A	:					Tuple					):
							SCREAMING_SNAKE_CASE       :   List[str]         =      test.split('''::'''					)[0]
							if test.startswith('''tests/models/'''					):
														SCREAMING_SNAKE_CASE       :   int         =      test.split('''/'''					)[2]
							else:
														SCREAMING_SNAKE_CASE       :   str         =      None

							return test
def        UpperCAmelCase (    A	:					Any       ,		A	:					Dict=None					):
							SCREAMING_SNAKE_CASE       :   Dict         =      [(x[0], x[1], get_model(x[2]					)) for x in logs]
							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      [x for x in logs if x[2] is not None]
							SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      {x[2] for x in logs}

							SCREAMING_SNAKE_CASE       :   Optional[int]         =      {}
							for test in tests:
														SCREAMING_SNAKE_CASE       :   Optional[Any]         =      Counter()
														# count by errors in `test`
														counter.update([x[1] for x in logs if x[2] == test]					)
														SCREAMING_SNAKE_CASE       :   Tuple         =      counter.most_common()
														SCREAMING_SNAKE_CASE       :   str         =      {error: count for error, count in counts if (error_filter is None or error not in error_filter)}
														SCREAMING_SNAKE_CASE       :   List[Any]         =      sum(error_counts.values()					)
														if n_errors > 0:
																					SCREAMING_SNAKE_CASE       :   Tuple         =      {'''count''': n_errors, '''errors''': error_counts}

							SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      dict(sorted(r.items()       ,		key=lambda A					: item[1]["count"]       ,		reverse=A					)					)
							return r
def        UpperCAmelCase (    A	:					Dict					):
							SCREAMING_SNAKE_CASE       :   Dict         =      '''| no. | error | status |'''
							SCREAMING_SNAKE_CASE       :   Tuple         =      '''|-:|:-|:-|'''
							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      [header, sep]
							for error in reduced_by_error:
														SCREAMING_SNAKE_CASE       :   Any         =      reduced_by_error[error]['''count''']
														SCREAMING_SNAKE_CASE       :   Tuple         =      F"""| {count} | {error[:100]} |  |"""
														lines.append(A					)

							return "\n".join(A					)
def        UpperCAmelCase (    A	:					List[str]					):
							SCREAMING_SNAKE_CASE       :   List[str]         =      '''| model | no. of errors | major error | count |'''
							SCREAMING_SNAKE_CASE       :   Optional[int]         =      '''|-:|-:|-:|-:|'''
							SCREAMING_SNAKE_CASE       :   Optional[Any]         =      [header, sep]
							for model in reduced_by_model:
														SCREAMING_SNAKE_CASE       :   Tuple         =      reduced_by_model[model]['''count''']
														SCREAMING_SNAKE_CASE       ,SCREAMING_SNAKE_CASE       :   str         =      list(reduced_by_model[model]['''errors'''].items()					)[0]
														SCREAMING_SNAKE_CASE       :   int         =      F"""| {model} | {count} | {error[:60]} | {_count} |"""
														lines.append(A					)

							return "\n".join(A					)


if __name__ == "__main__":
							lowerCAmelCase_  :							Optional[Any]             =    argparse.ArgumentParser()
							# Required parameters
							parser.add_argument('--workflow_run_id', type=str, required=True, help='A GitHub Actions workflow run id.')
							parser.add_argument(
							    '--output_dir',
							    type=str,
							    required=True,
							    help='Where to store the downloaded artifacts and other result files.',
							)
							parser.add_argument('--token', default=None, type=str, help='A token that has actions:read permission.')
							lowerCAmelCase_  :							List[Any]             =    parser.parse_args()

							os.makedirs(args.output_dir, exist_ok=True)

							lowerCAmelCase_  :							List[Any]             =    get_job_links(args.workflow_run_id, token=args.token)
							lowerCAmelCase_  :							int             =    {}
							# To deal with `workflow_call` event, where a job name is the combination of the job names in the caller and callee.
							# For example, `PyTorch 1.11 / Model tests (models/albert, single-gpu)`.
							if _job_links:
														for k, v in _job_links.items():
																					# This is how GitHub actions combine job names.
																					if " / " in k:
																												lowerCAmelCase_  :							Tuple             =    k.find(' / ')
																												lowerCAmelCase_  :							Optional[Any]             =    k[index + len(' / ') :]
																					lowerCAmelCase_  :							Union[str, Any]             =    v
							with open(os.path.join(args.output_dir, 'job_links.json'), 'w', encoding='UTF-8') as fp:
														json.dump(job_links, fp, ensure_ascii=False, indent=4)

							lowerCAmelCase_  :							Dict             =    get_artifacts_links(args.workflow_run_id, token=args.token)
							with open(os.path.join(args.output_dir, 'artifacts.json'), 'w', encoding='UTF-8') as fp:
														json.dump(artifacts, fp, ensure_ascii=False, indent=4)

							for idx, (name, url) in enumerate(artifacts.items()):
														download_artifact(name, url, args.output_dir, args.token)
														# Be gentle to GitHub
														time.sleep(1)

							lowerCAmelCase_  :							Optional[int]             =    get_all_errors(args.output_dir, job_links=job_links)

							# `e[1]` is the error
							lowerCAmelCase_  :							List[str]             =    Counter()
							counter.update([e[1] for e in errors])

							# print the top 30 most common test errors
							lowerCAmelCase_  :							Tuple             =    counter.most_common(30)
							for item in most_common:
														print(item)

							with open(os.path.join(args.output_dir, 'errors.json'), 'w', encoding='UTF-8') as fp:
														json.dump(errors, fp, ensure_ascii=False, indent=4)

							lowerCAmelCase_  :							int             =    reduce_by_error(errors)
							lowerCAmelCase_  :							Dict             =    reduce_by_model(errors)

							lowerCAmelCase_  :							Any             =    make_github_table(reduced_by_error)
							lowerCAmelCase_  :							List[Any]             =    make_github_table_per_model(reduced_by_model)

							with open(os.path.join(args.output_dir, 'reduced_by_error.txt'), 'w', encoding='UTF-8') as fp:
														fp.write(sa)
							with open(os.path.join(args.output_dir, 'reduced_by_model.txt'), 'w', encoding='UTF-8') as fp:
														fp.write(sa)









'''simple docstring'''



import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFAutoModel, is_tensorflow_text_available, is_tf_available
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.testing_utils import require_tensorflow_text, require_tf, slow


if is_tf_available():
							import tensorflow as tf

if is_tensorflow_text_available():
							from transformers.models.bert import TFBertTokenizer


lowerCAmelCase_  :							List[str]             =    ['bert-base-uncased', 'bert-base-cased']
lowerCAmelCase_  :							int             =    'hf-internal-testing/tiny-bert-tf-only'

if is_tf_available():
							class 						lowerCamelCase_				(       tf.keras.Model      ):







													def __init__(	self :   Optional[int]					,     lowerCAmelCase__ :   Optional[Any]					):

																				"""simple docstring"""
																				super().__init__()
																				SCREAMING_SNAKE_CASE       :   List[str]         =      tokenizer
																				SCREAMING_SNAKE_CASE       :   str         =      AutoConfig.from_pretrained(lowerCAmelCase__					)
																				SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      TFAutoModel.from_config(lowerCAmelCase__					)







													def        __lowercase     (	self :   Tuple					,     lowerCAmelCase__ :   Tuple					):

																				"""simple docstring"""
																				SCREAMING_SNAKE_CASE       :   Optional[Any]         =      self.tokenizer(lowerCAmelCase__					)
																				SCREAMING_SNAKE_CASE       :   str         =      self.bert(**lowerCAmelCase__					)
																				return out["pooler_output"]
@require_tf
@require_tensorflow_text
class 						lowerCamelCase_				(       unittest.TestCase      ):







						def        __lowercase     (	self :   str					):

													"""simple docstring"""
													super().setUp()

													SCREAMING_SNAKE_CASE       :   int         =      [
													    BertTokenizer.from_pretrained(lowerCAmelCase__					) for checkpoint in (TOKENIZER_CHECKPOINTS * 2)
													]  # repeat for when fast_bert_tokenizer=false
													SCREAMING_SNAKE_CASE       :   Dict         =      [TFBertTokenizer.from_pretrained(lowerCAmelCase__					) for checkpoint in TOKENIZER_CHECKPOINTS] + [
													    TFBertTokenizer.from_pretrained(lowerCAmelCase__					,     use_fast_bert_tokenizer=lowerCAmelCase__					)
													    for checkpoint in TOKENIZER_CHECKPOINTS
													]
													assert len(self.tokenizers					) == len(self.tf_tokenizers					)

													SCREAMING_SNAKE_CASE       :   Any         =      [
													    '''This is a straightforward English test sentence.''',
													    '''This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.''',
													    '''Now we\'re going to add some Chinese: 一 二 三 一二三''',
													    '''And some much more rare Chinese: 齉 堃 齉堃''',
													    '''Je vais aussi écrire en français pour tester les accents''',
													    '''Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ''',
													]
													SCREAMING_SNAKE_CASE       :   int         =      list(zip(self.test_sentences					,     self.test_sentences[::-1]					)					)







						def        __lowercase     (	self :   Any					):

													"""simple docstring"""
													for tokenizer, tf_tokenizer in zip(self.tokenizers					,     self.tf_tokenizers					):
																				for test_inputs in (self.test_sentences, self.paired_sentences):
																											SCREAMING_SNAKE_CASE       :   Tuple         =      tokenizer(lowerCAmelCase__					,     return_tensors='''tf'''					,     padding='''longest'''					)
																											SCREAMING_SNAKE_CASE       :   List[Any]         =      tf_tokenizer(lowerCAmelCase__					)

																											for key in python_outputs.keys():
																																		self.assertTrue(tf.reduce_all(python_outputs[key].shape == tf_outputs[key].shape					)					)
																																		self.assertTrue(tf.reduce_all(tf.cast(python_outputs[key]					,     tf.intaa					) == tf_outputs[key]					)					)







						@slow
						def        __lowercase     (	self :   Dict					):

													"""simple docstring"""
													for tf_tokenizer in self.tf_tokenizers:
																				SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      tf_tokenizer(self.paired_sentences					)
																				SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      tf_tokenizer(
																				    text=[sentence[0] for sentence in self.paired_sentences]					,     text_pair=[sentence[1] for sentence in self.paired_sentences]					,     )
																				for key in merged_outputs.keys():
																											self.assertTrue(tf.reduce_all(tf.cast(merged_outputs[key]					,     tf.intaa					) == separated_outputs[key]					)					)







						@slow
						def        __lowercase     (	self :   Union[str, Any]					):

													"""simple docstring"""
													for tf_tokenizer in self.tf_tokenizers:
																				SCREAMING_SNAKE_CASE       :   Any         =      tf.function(lowerCAmelCase__					)
																				for test_inputs in (self.test_sentences, self.paired_sentences):
																											SCREAMING_SNAKE_CASE       :   int         =      tf.constant(lowerCAmelCase__					)
																											SCREAMING_SNAKE_CASE       :   Any         =      compiled_tokenizer(lowerCAmelCase__					)
																											SCREAMING_SNAKE_CASE       :   List[str]         =      tf_tokenizer(lowerCAmelCase__					)

																											for key in eager_outputs.keys():
																																		self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]					)					)







						@slow
						def        __lowercase     (	self :   int					):

													"""simple docstring"""
													for tf_tokenizer in self.tf_tokenizers:
																				SCREAMING_SNAKE_CASE       :   int         =      ModelToSave(tokenizer=lowerCAmelCase__					)
																				SCREAMING_SNAKE_CASE       :   Tuple         =      tf.convert_to_tensor(self.test_sentences					)
																				SCREAMING_SNAKE_CASE       :   Optional[Any]         =      model(lowerCAmelCase__					)  # Build model with some sample inputs
																				with TemporaryDirectory() as tempdir:
																											SCREAMING_SNAKE_CASE       :   Optional[int]         =      Path(lowerCAmelCase__					) / '''saved.model'''
																											model.save(lowerCAmelCase__					)
																											SCREAMING_SNAKE_CASE       :   Union[str, Any]         =      tf.keras.models.load_model(lowerCAmelCase__					)
																				SCREAMING_SNAKE_CASE       :   Tuple         =      loaded_model(lowerCAmelCase__					)
																				# We may see small differences because the loaded model is compiled, so we need an epsilon for the test
																				self.assertLessEqual(tf.reduce_max(tf.abs(out - loaded_output					)					)					,     1e-5					)




