






"""simple docstring"""


import argparse
import json
from pathlib import Path

import torch
import torchaudio
from datasets import load_dataset
from huggingface_hub import hf_hub_download

from transformers import ASTConfig, ASTFeatureExtractor, ASTForAudioClassification
from transformers.utils import logging


logging.set_verbosity_info()
__SCREAMING_SNAKE_CASE   =   logging.get_logger(__name__)


def        UpperCAmelCase      (					a__ ):





						'''simple docstring'''






						lowerCAmelCase   :List[str]						=			ASTConfig()

						if "10-10" in model_name:
												pass
						elif "speech-commands" in model_name:
												lowerCAmelCase   :Dict						=			1_28
						elif "12-12" in model_name:
												lowerCAmelCase   :Union[str, Any]						=			12
												lowerCAmelCase   :Optional[Any]						=			12
						elif "14-14" in model_name:
												lowerCAmelCase   :str						=			14
												lowerCAmelCase   :Optional[int]						=			14
						elif "16-16" in model_name:
												lowerCAmelCase   :Any						=			16
												lowerCAmelCase   :Optional[int]						=			16
						else:
												raise ValueError('Model not supported' )

						lowerCAmelCase   :Union[str, Any]						=			'huggingface/label-files'
						if "speech-commands" in model_name:
												lowerCAmelCase   :Any						=			35
												lowerCAmelCase   :Optional[int]						=			'speech-commands-v2-id2label.json'
						else:
												lowerCAmelCase   :Any						=			5_27
												lowerCAmelCase   :Dict						=			'audioset-id2label.json'

						lowerCAmelCase   :str						=			json.load(open(hf_hub_download(a__     ,      a__     ,      repo_type='dataset' )     ,      'r' ) )
						lowerCAmelCase   :List[Any]						=			{int(a__ ): v for k, v in idalabel.items()}
						lowerCAmelCase   :Tuple						=			idalabel
						lowerCAmelCase   :Union[str, Any]						=			{v: k for k, v in idalabel.items()}

						return config


def        UpperCAmelCase      (					a__ ):





						'''simple docstring'''






						if "module.v" in name:
												lowerCAmelCase   :List[Any]						=			name.replace('module.v'     ,      'audio_spectrogram_transformer' )
						if "cls_token" in name:
												lowerCAmelCase   :str						=			name.replace('cls_token'     ,      'embeddings.cls_token' )
						if "dist_token" in name:
												lowerCAmelCase   :Any						=			name.replace('dist_token'     ,      'embeddings.distillation_token' )
						if "pos_embed" in name:
												lowerCAmelCase   :Any						=			name.replace('pos_embed'     ,      'embeddings.position_embeddings' )
						if "patch_embed.proj" in name:
												lowerCAmelCase   :Optional[Any]						=			name.replace('patch_embed.proj'     ,      'embeddings.patch_embeddings.projection' )
						# transformer blocks
						if "blocks" in name:
												lowerCAmelCase   :Any						=			name.replace('blocks'     ,      'encoder.layer' )
						if "attn.proj" in name:
												lowerCAmelCase   :int						=			name.replace('attn.proj'     ,      'attention.output.dense' )
						if "attn" in name:
												lowerCAmelCase   :Optional[int]						=			name.replace('attn'     ,      'attention.self' )
						if "norm1" in name:
												lowerCAmelCase   :Dict						=			name.replace('norm1'     ,      'layernorm_before' )
						if "norm2" in name:
												lowerCAmelCase   :Dict						=			name.replace('norm2'     ,      'layernorm_after' )
						if "mlp.fc1" in name:
												lowerCAmelCase   :List[Any]						=			name.replace('mlp.fc1'     ,      'intermediate.dense' )
						if "mlp.fc2" in name:
												lowerCAmelCase   :Optional[int]						=			name.replace('mlp.fc2'     ,      'output.dense' )
						# final layernorm
						if "audio_spectrogram_transformer.norm" in name:
												lowerCAmelCase   :Optional[Any]						=			name.replace('audio_spectrogram_transformer.norm'     ,      'audio_spectrogram_transformer.layernorm' )
						# classifier head
						if "module.mlp_head.0" in name:
												lowerCAmelCase   :str						=			name.replace('module.mlp_head.0'     ,      'classifier.layernorm' )
						if "module.mlp_head.1" in name:
												lowerCAmelCase   :Dict						=			name.replace('module.mlp_head.1'     ,      'classifier.dense' )

						return name


def        UpperCAmelCase      (					a__     ,      a__ ):





						'''simple docstring'''






						for key in orig_state_dict.copy().keys():
												lowerCAmelCase   :Dict						=			orig_state_dict.pop(a__ )

												if "qkv" in key:
																		lowerCAmelCase   :Optional[Any]						=			key.split('.' )
																		lowerCAmelCase   :Any						=			int(key_split[3] )
																		lowerCAmelCase   :Dict						=			config.hidden_size
																		if "weight" in key:
																								lowerCAmelCase   :str						=			val[:dim, :]
																								lowerCAmelCase   :Tuple						=			val[dim : dim * 2, :]
																								lowerCAmelCase   :Dict						=			val[-dim:, :]
																		else:
																								lowerCAmelCase   :Optional[Any]						=			val[:dim]
																								lowerCAmelCase   :int						=			val[dim : dim * 2]
																								lowerCAmelCase   :Union[str, Any]						=			val[-dim:]
												else:
																		lowerCAmelCase   :int						=			val

						return orig_state_dict


def        UpperCAmelCase      (					a__ ):





						'''simple docstring'''






						lowerCAmelCase   :List[Any]						=			[
						    'module.v.head.weight',
						    'module.v.head.bias',
						    'module.v.head_dist.weight',
						    'module.v.head_dist.bias',
						]
						for k in ignore_keys:
												state_dict.pop(a__     ,      a__ )


@torch.no_grad()
def        UpperCAmelCase      (					a__     ,      a__     ,      a__=False ):





						'''simple docstring'''






						lowerCAmelCase   :str						=			get_audio_spectrogram_transformer_config(a__ )

						lowerCAmelCase   :Tuple						=			{
						    'ast-finetuned-audioset-10-10-0.4593': (
						        'https://www.dropbox.com/s/ca0b1v2nlxzyeb4/audioset_10_10_0.4593.pth?dl=1'
						    ),
						    'ast-finetuned-audioset-10-10-0.450': (
						        'https://www.dropbox.com/s/1tv0hovue1bxupk/audioset_10_10_0.4495.pth?dl=1'
						    ),
						    'ast-finetuned-audioset-10-10-0.448': (
						        'https://www.dropbox.com/s/6u5sikl4b9wo4u5/audioset_10_10_0.4483.pth?dl=1'
						    ),
						    'ast-finetuned-audioset-10-10-0.448-v2': (
						        'https://www.dropbox.com/s/kt6i0v9fvfm1mbq/audioset_10_10_0.4475.pth?dl=1'
						    ),
						    'ast-finetuned-audioset-12-12-0.447': (
						        'https://www.dropbox.com/s/snfhx3tizr4nuc8/audioset_12_12_0.4467.pth?dl=1'
						    ),
						    'ast-finetuned-audioset-14-14-0.443': (
						        'https://www.dropbox.com/s/z18s6pemtnxm4k7/audioset_14_14_0.4431.pth?dl=1'
						    ),
						    'ast-finetuned-audioset-16-16-0.442': (
						        'https://www.dropbox.com/s/mdsa4t1xmcimia6/audioset_16_16_0.4422.pth?dl=1'
						    ),
						    'ast-finetuned-speech-commands-v2': (
						        'https://www.dropbox.com/s/q0tbqpwv44pquwy/speechcommands_10_10_0.9812.pth?dl=1'
						    ),
						}

						# load original state_dict
						lowerCAmelCase   :int						=			model_name_to_url[model_name]
						lowerCAmelCase   :Optional[int]						=			torch.hub.load_state_dict_from_url(a__     ,      map_location='cpu' )
						# remove some keys
						remove_keys(a__ )
						# rename some keys
						lowerCAmelCase   :str						=			convert_state_dict(a__     ,      a__ )

						# load 🤗 model
						lowerCAmelCase   :int						=			ASTForAudioClassification(a__ )
						model.eval()

						model.load_state_dict(a__ )

						# verify outputs on dummy input
						# source: https://github.com/YuanGongND/ast/blob/79e873b8a54d0a3b330dd522584ff2b9926cd581/src/run.py#L62
						lowerCAmelCase   :Optional[Any]						=			-4.2_677_393 if 'speech-commands' not in model_name else -6.845_978
						lowerCAmelCase   :Dict						=			4.5_689_974 if 'speech-commands' not in model_name else 5.5_654_526
						lowerCAmelCase   :Dict						=			10_24 if 'speech-commands' not in model_name else 1_28
						lowerCAmelCase   :List[str]						=			ASTFeatureExtractor(mean=a__     ,      std=a__     ,      max_length=a__ )

						if "speech-commands" in model_name:
												lowerCAmelCase   :Tuple						=			load_dataset('speech_commands'     ,      'v0.02'     ,      split='validation' )
												lowerCAmelCase   :List[str]						=			dataset[0]['audio']['array']
						else:
												lowerCAmelCase   :Optional[int]						=			hf_hub_download(
												    repo_id='nielsr/audio-spectogram-transformer-checkpoint'     ,      filename='sample_audio.flac'     ,      repo_type='dataset'     ,      )

												lowerCAmelCase	, lowerCAmelCase   :List[str]						=			torchaudio.load(a__ )
												lowerCAmelCase   :Any						=			waveform.squeeze().numpy()

						lowerCAmelCase   :Optional[Any]						=			feature_extractor(a__     ,      sampling_rate=1_60_00     ,      return_tensors='pt' )

						# forward pass
						lowerCAmelCase   :List[Any]						=			model(**a__ )
						lowerCAmelCase   :str						=			outputs.logits

						if model_name == "ast-finetuned-audioset-10-10-0.4593":
												lowerCAmelCase   :Optional[Any]						=			torch.tensor([-0.8_760, -7.0_042, -8.6_602] )
						elif model_name == "ast-finetuned-audioset-10-10-0.450":
												lowerCAmelCase   :Union[str, Any]						=			torch.tensor([-1.1_986, -7.0_903, -8.2_718] )
						elif model_name == "ast-finetuned-audioset-10-10-0.448":
												lowerCAmelCase   :Optional[int]						=			torch.tensor([-2.6_128, -8.0_080, -9.4_344] )
						elif model_name == "ast-finetuned-audioset-10-10-0.448-v2":
												lowerCAmelCase   :Union[str, Any]						=			torch.tensor([-1.5_080, -7.4_534, -8.8_917] )
						elif model_name == "ast-finetuned-audioset-12-12-0.447":
												lowerCAmelCase   :int						=			torch.tensor([-0.5_050, -6.5_833, -8.0_843] )
						elif model_name == "ast-finetuned-audioset-14-14-0.443":
												lowerCAmelCase   :Dict						=			torch.tensor([-0.3_826, -7.0_336, -8.2_413] )
						elif model_name == "ast-finetuned-audioset-16-16-0.442":
												lowerCAmelCase   :Dict						=			torch.tensor([-1.2_113, -6.9_101, -8.3_470] )
						elif model_name == "ast-finetuned-speech-commands-v2":
												lowerCAmelCase   :List[Any]						=			torch.tensor([6.1_589, -8.0_566, -8.7_984] )
						else:
												raise ValueError('Unknown model name' )
						if not torch.allclose(logits[0, :3]     ,      a__     ,      atol=1e-4 ):
												raise ValueError('Logits don\'t match' )
						print('Looks ok!' )

						if pytorch_dump_folder_path is not None:
												Path(a__ ).mkdir(exist_ok=a__ )
												print(F"""Saving model {model_name} to {pytorch_dump_folder_path}""" )
												model.save_pretrained(a__ )
												print(F"""Saving feature extractor to {pytorch_dump_folder_path}""" )
												feature_extractor.save_pretrained(a__ )

						if push_to_hub:
												print('Pushing model and feature extractor to the hub...' )
												model.push_to_hub(F"""MIT/{model_name}""" )
												feature_extractor.push_to_hub(F"""MIT/{model_name}""" )


if __name__ == "__main__":
		__SCREAMING_SNAKE_CASE   =   argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '--model_name',
		    default='ast-finetuned-audioset-10-10-0.4593',
		    type=str,
		    help='Name of the Audio Spectrogram Transformer model you\'d like to convert.',
		)
		parser.add_argument(
		    '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
		)
		parser.add_argument(
		    '--push_to_hub', action='store_true', help='Whether or not to push the converted model to the 🤗 hub.'
		)

		__SCREAMING_SNAKE_CASE   =   parser.parse_args()
		convert_audio_spectrogram_transformer_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)







"""simple docstring"""


import argparse
import json
import math
import os
import time
import traceback
import zipfile
from collections import Counter

import requests


def        UpperCAmelCase      (					a__     ,      a__=None ):





						'''simple docstring'''






						lowerCAmelCase   :str						=			None
						if token is not None:
												lowerCAmelCase   :List[Any]						=			{'Accept': 'application/vnd.github+json', 'Authorization': F"""Bearer {token}"""}

						lowerCAmelCase   :Dict						=			F"""https://api.github.com/repos/huggingface/transformers/actions/runs/{workflow_run_id}/jobs?per_page=100"""
						lowerCAmelCase   :Optional[Any]						=			requests.get(a__     ,      headers=a__ ).json()
						lowerCAmelCase   :Tuple						=			{}

						try:
												job_links.update({job['name']: job['html_url'] for job in result['jobs']} )
												lowerCAmelCase   :List[Any]						=			math.ceil((result['total_count'] - 1_00) / 1_00 )

												for i in range(a__ ):
																		lowerCAmelCase   :List[Any]						=			requests.get(url + F"""&page={i + 2}"""     ,      headers=a__ ).json()
																		job_links.update({job['name']: job['html_url'] for job in result['jobs']} )

												return job_links
						except Exception:
												print(F"""Unknown error, could not fetch links:\n{traceback.format_exc()}""" )

						return {}


def        UpperCAmelCase      (					a__     ,      a__=None ):





						'''simple docstring'''






						lowerCAmelCase   :Optional[Any]						=			None
						if token is not None:
												lowerCAmelCase   :Any						=			{'Accept': 'application/vnd.github+json', 'Authorization': F"""Bearer {token}"""}

						lowerCAmelCase   :Optional[int]						=			F"""https://api.github.com/repos/huggingface/transformers/actions/runs/{worflow_run_id}/artifacts?per_page=100"""
						lowerCAmelCase   :Any						=			requests.get(a__     ,      headers=a__ ).json()
						lowerCAmelCase   :str						=			{}

						try:
												artifacts.update({artifact['name']: artifact['archive_download_url'] for artifact in result['artifacts']} )
												lowerCAmelCase   :List[Any]						=			math.ceil((result['total_count'] - 1_00) / 1_00 )

												for i in range(a__ ):
																		lowerCAmelCase   :List[Any]						=			requests.get(url + F"""&page={i + 2}"""     ,      headers=a__ ).json()
																		artifacts.update({artifact['name']: artifact['archive_download_url'] for artifact in result['artifacts']} )

												return artifacts
						except Exception:
												print(F"""Unknown error, could not fetch links:\n{traceback.format_exc()}""" )

						return {}


def        UpperCAmelCase      (					a__     ,      a__     ,      a__     ,      a__ ):





						'''simple docstring'''






						lowerCAmelCase   :Optional[Any]						=			None
						if token is not None:
												lowerCAmelCase   :Optional[Any]						=			{'Accept': 'application/vnd.github+json', 'Authorization': F"""Bearer {token}"""}

						lowerCAmelCase   :Tuple						=			requests.get(a__     ,      headers=a__     ,      allow_redirects=a__ )
						lowerCAmelCase   :Optional[int]						=			result.headers['Location']
						lowerCAmelCase   :int						=			requests.get(a__     ,      allow_redirects=a__ )
						lowerCAmelCase   :Union[str, Any]						=			os.path.join(a__     ,      F"""{artifact_name}.zip""" )
						with open(a__     ,      'wb' ) as fp:
												fp.write(response.content )


def        UpperCAmelCase      (					a__     ,      a__=None ):





						'''simple docstring'''






						lowerCAmelCase   :Optional[Any]						=			[]
						lowerCAmelCase   :Dict						=			[]
						lowerCAmelCase   :Optional[Any]						=			None

						with zipfile.ZipFile(a__ ) as z:
												for filename in z.namelist():
																		if not os.path.isdir(a__ ):
																								# read the file
																								if filename in ["failures_line.txt", "summary_short.txt", "job_name.txt"]:
																														with z.open(a__ ) as f:
																																				for line in f:
																																										lowerCAmelCase   :int						=			line.decode('UTF-8' ).strip()
																																										if filename == "failures_line.txt":
																																																try:
																																																						# `error_line` is the place where `error` occurs
																																																						lowerCAmelCase   :int						=			line[: line.index(': ' )]
																																																						lowerCAmelCase   :int						=			line[line.index(': ' ) + len(': ' ) :]
																																																						errors.append([error_line, error] )
																																																except Exception:
																																																						# skip un-related lines
																																																						pass
																																										elif filename == "summary_short.txt" and line.startswith('FAILED ' ):
																																																# `test` is the test method that failed
																																																lowerCAmelCase   :int						=			line[len('FAILED ' ) :]
																																																failed_tests.append(a__ )
																																										elif filename == "job_name.txt":
																																																lowerCAmelCase   :List[str]						=			line

						if len(a__ ) != len(a__ ):
												raise ValueError(
												    F"""`errors` and `failed_tests` should have the same number of elements. Got {len(a__ )} for `errors` """
												    F"""and {len(a__ )} for `failed_tests` instead. The test reports in {artifact_zip_path} have some"""
												    ' problem.' )

						lowerCAmelCase   :Optional[int]						=			None
						if job_name and job_links:
												lowerCAmelCase   :Dict						=			job_links.get(a__     ,      a__ )

						# A list with elements of the form (line of error, error, failed test)
						lowerCAmelCase   :Union[str, Any]						=			[x + [y] + [job_link] for x, y in zip(a__     ,      a__ )]

						return result


def        UpperCAmelCase      (					a__     ,      a__=None ):





						'''simple docstring'''






						lowerCAmelCase   :Any						=			[]

						lowerCAmelCase   :Optional[int]						=			[os.path.join(a__     ,      a__ ) for p in os.listdir(a__ ) if p.endswith('.zip' )]
						for p in paths:
												errors.extend(get_errors_from_single_artifact(a__     ,      job_links=a__ ) )

						return errors


def        UpperCAmelCase      (					a__     ,      a__=None ):





						'''simple docstring'''






						lowerCAmelCase   :int						=			Counter()
						counter.update([x[1] for x in logs] )
						lowerCAmelCase   :Tuple						=			counter.most_common()
						lowerCAmelCase   :Dict						=			{}
						for error, count in counts:
												if error_filter is None or error not in error_filter:
																		lowerCAmelCase   :Optional[int]						=			{'count': count, 'failed_tests': [(x[2], x[0]) for x in logs if x[1] == error]}

						lowerCAmelCase   :Optional[Any]						=			dict(sorted(r.items()     ,      key=lambda a__ : item[1]["count"]     ,      reverse=a__ ) )
						return r


def        UpperCAmelCase      (					a__ ):





						'''simple docstring'''






						lowerCAmelCase   :Tuple						=			test.split('::' )[0]
						if test.startswith('tests/models/' ):
												lowerCAmelCase   :Union[str, Any]						=			test.split('/' )[2]
						else:
												lowerCAmelCase   :Optional[int]						=			None

						return test


def        UpperCAmelCase      (					a__     ,      a__=None ):





						'''simple docstring'''






						lowerCAmelCase   :str						=			[(x[0], x[1], get_model(x[2] )) for x in logs]
						lowerCAmelCase   :Any						=			[x for x in logs if x[2] is not None]
						lowerCAmelCase   :Tuple						=			{x[2] for x in logs}

						lowerCAmelCase   :Optional[Any]						=			{}
						for test in tests:
												lowerCAmelCase   :Union[str, Any]						=			Counter()
												# count by errors in `test`
												counter.update([x[1] for x in logs if x[2] == test] )
												lowerCAmelCase   :List[str]						=			counter.most_common()
												lowerCAmelCase   :str						=			{error: count for error, count in counts if (error_filter is None or error not in error_filter)}
												lowerCAmelCase   :Optional[Any]						=			sum(error_counts.values() )
												if n_errors > 0:
																		lowerCAmelCase   :Dict						=			{'count': n_errors, 'errors': error_counts}

						lowerCAmelCase   :Dict						=			dict(sorted(r.items()     ,      key=lambda a__ : item[1]["count"]     ,      reverse=a__ ) )
						return r


def        UpperCAmelCase      (					a__ ):





						'''simple docstring'''






						lowerCAmelCase   :Any						=			'| no. | error | status |'
						lowerCAmelCase   :Optional[int]						=			'|-:|:-|:-|'
						lowerCAmelCase   :Tuple						=			[header, sep]
						for error in reduced_by_error:
												lowerCAmelCase   :Optional[Any]						=			reduced_by_error[error]['count']
												lowerCAmelCase   :List[str]						=			F"""| {count} | {error[:1_00]} |  |"""
												lines.append(a__ )

						return "\n".join(a__ )


def        UpperCAmelCase      (					a__ ):





						'''simple docstring'''






						lowerCAmelCase   :Any						=			'| model | no. of errors | major error | count |'
						lowerCAmelCase   :int						=			'|-:|-:|-:|-:|'
						lowerCAmelCase   :int						=			[header, sep]
						for model in reduced_by_model:
												lowerCAmelCase   :Dict						=			reduced_by_model[model]['count']
												lowerCAmelCase	, lowerCAmelCase   :Any						=			list(reduced_by_model[model]['errors'].items() )[0]
												lowerCAmelCase   :Any						=			F"""| {model} | {count} | {error[:60]} | {_count} |"""
												lines.append(a__ )

						return "\n".join(a__ )


if __name__ == "__main__":
		__SCREAMING_SNAKE_CASE   =   argparse.ArgumentParser()
		# Required parameters
		parser.add_argument('--workflow_run_id', type=str, required=True, help='A GitHub Actions workflow run id.')
		parser.add_argument(
		    '--output_dir',
		    type=str,
		    required=True,
		    help='Where to store the downloaded artifacts and other result files.',
		)
		parser.add_argument('--token', default=None, type=str, help='A token that has actions:read permission.')
		__SCREAMING_SNAKE_CASE   =   parser.parse_args()

		os.makedirs(args.output_dir, exist_ok=True)

		__SCREAMING_SNAKE_CASE   =   get_job_links(args.workflow_run_id, token=args.token)
		__SCREAMING_SNAKE_CASE   =   {}
		# To deal with `workflow_call` event, where a job name is the combination of the job names in the caller and callee.
		# For example, `PyTorch 1.11 / Model tests (models/albert, single-gpu)`.
		if _job_links:
				for k, v in _job_links.items():
						# This is how GitHub actions combine job names.
						if " / " in k:
								__SCREAMING_SNAKE_CASE   =   k.find(' / ')
								__SCREAMING_SNAKE_CASE   =   k[index + len(' / ') :]
						__SCREAMING_SNAKE_CASE   =   v
		with open(os.path.join(args.output_dir, 'job_links.json'), 'w', encoding='UTF-8') as fp:
				json.dump(job_links, fp, ensure_ascii=False, indent=4)

		__SCREAMING_SNAKE_CASE   =   get_artifacts_links(args.workflow_run_id, token=args.token)
		with open(os.path.join(args.output_dir, 'artifacts.json'), 'w', encoding='UTF-8') as fp:
				json.dump(artifacts, fp, ensure_ascii=False, indent=4)

		for idx, (name, url) in enumerate(artifacts.items()):
				download_artifact(name, url, args.output_dir, args.token)
				# Be gentle to GitHub
				time.sleep(1)

		__SCREAMING_SNAKE_CASE   =   get_all_errors(args.output_dir, job_links=job_links)

		# `e[1]` is the error
		__SCREAMING_SNAKE_CASE   =   Counter()
		counter.update([e[1] for e in errors])

		# print the top 30 most common test errors
		__SCREAMING_SNAKE_CASE   =   counter.most_common(30)
		for item in most_common:
				print(item)

		with open(os.path.join(args.output_dir, 'errors.json'), 'w', encoding='UTF-8') as fp:
				json.dump(errors, fp, ensure_ascii=False, indent=4)

		__SCREAMING_SNAKE_CASE   =   reduce_by_error(errors)
		__SCREAMING_SNAKE_CASE   =   reduce_by_model(errors)

		__SCREAMING_SNAKE_CASE   =   make_github_table(reduced_by_error)
		__SCREAMING_SNAKE_CASE   =   make_github_table_per_model(reduced_by_model)

		with open(os.path.join(args.output_dir, 'reduced_by_error.txt'), 'w', encoding='UTF-8') as fp:
				fp.write(sa)
		with open(os.path.join(args.output_dir, 'reduced_by_model.txt'), 'w', encoding='UTF-8') as fp:
				fp.write(sa)