



"""simple docstring"""

import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


A_						:    Optional[int]				=      re.compile(R'\b(a|an|the)\b', re.UNICODE)

A_						:    str				=      None
def  __snake_case							(	)     ->       List[Any]:






				'''simple docstring'''
				SCREAMING_SNAKE_CASE						:							int							=     argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'     )
				parser.add_argument('data_file'							,		metavar='data.json'							,		help='Input data JSON file.'     )
				parser.add_argument('pred_file'							,		metavar='pred.json'							,		help='Model predictions.'     )
				parser.add_argument(
				    '--out-file'							,		'-o'							,		metavar='eval.json'							,		help='Write accuracy metrics to file (default is stdout).'     )
				parser.add_argument(
				    '--na-prob-file'							,		'-n'							,		metavar='na_prob.json'							,		help='Model estimates of probability of no answer.'     )
				parser.add_argument(
				    '--na-prob-thresh'							,		'-t'							,		type=__A							,		default=1.0							,		help='Predict "" if no-answer probability exceeds this (default = 1.0).'							,		)
				parser.add_argument(
				    '--out-image-dir'							,		'-p'							,		metavar='out_images'							,		default=__A							,		help='Save precision-recall curves to directory.'     )
				parser.add_argument('--verbose'							,		'-v'							,		action='store_true'     )
				if len(sys.argv     ) == 1:
								parser.print_help()
								sys.exit(1     )
				return parser.parse_args()
def  __snake_case							(	__A      :						List[str]     )     ->       Optional[int]:






				'''simple docstring'''
				SCREAMING_SNAKE_CASE						:							Dict							=     {}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																SCREAMING_SNAKE_CASE						:							Optional[Any]							=     bool(qa['answers']['text']     )
				return qid_to_has_ans
def  __snake_case							(	__A      :						int     )     ->       List[str]:






				'''simple docstring'''

				def remove_articles(__A      :						Tuple     ):
								return ARTICLES_REGEX.sub(' '							,		__A     )

				def white_space_fix(__A      :						Dict     ):
								return " ".join(text.split()     )

				def remove_punc(__A      :						Tuple     ):
								SCREAMING_SNAKE_CASE						:							List[Any]							=     set(string.punctuation     )
								return "".join(ch for ch in text if ch not in exclude     )

				def lower(__A      :						List[Any]     ):
								return text.lower()

				return white_space_fix(remove_articles(remove_punc(lower(__A     )     )     )     )
def  __snake_case							(	__A      :						str     )     ->       str:






				'''simple docstring'''
				if not s:
								return []
				return normalize_answer(__A     ).split()
def  __snake_case							(	__A      :						List[str]							,		__A      :						str     )     ->       int:






				'''simple docstring'''
				return int(normalize_answer(__A     ) == normalize_answer(__A     )     )
def  __snake_case							(	__A      :						Dict							,		__A      :						Dict     )     ->       List[Any]:






				'''simple docstring'''
				SCREAMING_SNAKE_CASE						:							str							=     get_tokens(__A     )
				SCREAMING_SNAKE_CASE						:							Optional[Any]							=     get_tokens(__A     )
				SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     collections.Counter(__A     ) & collections.Counter(__A     )
				SCREAMING_SNAKE_CASE						:							Optional[Any]							=     sum(common.values()     )
				if len(__A     ) == 0 or len(__A     ) == 0:
								# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
								return int(gold_toks == pred_toks     )
				if num_same == 0:
								return 0
				SCREAMING_SNAKE_CASE						:							int							=     1.0 * num_same / len(__A     )
				SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     1.0 * num_same / len(__A     )
				SCREAMING_SNAKE_CASE						:							Optional[int]							=     (2 * precision * recall) / (precision + recall)
				return fa
def  __snake_case							(	__A      :						Union[str, Any]							,		__A      :						List[str]     )     ->       str:






				'''simple docstring'''
				SCREAMING_SNAKE_CASE						:							str							=     {}
				SCREAMING_SNAKE_CASE						:							List[Any]							=     {}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																SCREAMING_SNAKE_CASE						:							Tuple							=     qa['id']
																SCREAMING_SNAKE_CASE						:							str							=     [t for t in qa['answers']['text'] if normalize_answer(__A     )]
																if not gold_answers:
																				# For unanswerable questions, only correct answer is empty string
																				SCREAMING_SNAKE_CASE						:							Optional[int]							=     ['']
																if qid not in preds:
																				print(F"""Missing prediction for {qid}"""     )
																				continue
																SCREAMING_SNAKE_CASE						:							List[str]							=     preds[qid]
																# Take max over all gold answers
																SCREAMING_SNAKE_CASE						:							str							=     max(compute_exact(__A							,		__A     ) for a in gold_answers     )
																SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     max(compute_fa(__A							,		__A     ) for a in gold_answers     )
				return exact_scores, fa_scores
def  __snake_case							(	__A      :						Optional[int]							,		__A      :						List[Any]							,		__A      :						List[Any]							,		__A      :						Union[str, Any]     )     ->       int:






				'''simple docstring'''
				SCREAMING_SNAKE_CASE						:							Tuple							=     {}
				for qid, s in scores.items():
								SCREAMING_SNAKE_CASE						:							str							=     na_probs[qid] > na_prob_thresh
								if pred_na:
												SCREAMING_SNAKE_CASE						:							List[Any]							=     float(not qid_to_has_ans[qid]     )
								else:
												SCREAMING_SNAKE_CASE						:							Any							=     s
				return new_scores
def  __snake_case							(	__A      :						Union[str, Any]							,		__A      :						Optional[Any]							,		__A      :						Any=None     )     ->       Optional[Any]:






				'''simple docstring'''
				if not qid_list:
								SCREAMING_SNAKE_CASE						:							Optional[int]							=     len(__A     )
								return collections.OrderedDict(
								    [
								        ('exact', 1_0_0.0 * sum(exact_scores.values()     ) / total),
								        ('f1', 1_0_0.0 * sum(fa_scores.values()     ) / total),
								        ('total', total),
								    ]     )
				else:
								SCREAMING_SNAKE_CASE						:							Any							=     len(__A     )
								return collections.OrderedDict(
								    [
								        ('exact', 1_0_0.0 * sum(exact_scores[k] for k in qid_list     ) / total),
								        ('f1', 1_0_0.0 * sum(fa_scores[k] for k in qid_list     ) / total),
								        ('total', total),
								    ]     )
def  __snake_case							(	__A      :						List[Any]							,		__A      :						Union[str, Any]							,		__A      :						Optional[int]     )     ->       List[Any]:






				'''simple docstring'''
				for k in new_eval:
								SCREAMING_SNAKE_CASE						:							int							=     new_eval[k]
def  __snake_case							(	__A      :						Optional[Any]							,		__A      :						Optional[int]							,		__A      :						str							,		__A      :						Dict     )     ->       Optional[Any]:






				'''simple docstring'''
				plt.step(__A							,		__A							,		color='b'							,		alpha=0.2							,		where='post'     )
				plt.fill_between(__A							,		__A							,		step='post'							,		alpha=0.2							,		color='b'     )
				plt.xlabel('Recall'     )
				plt.ylabel('Precision'     )
				plt.xlim([0.0, 1.0_5]     )
				plt.ylim([0.0, 1.0_5]     )
				plt.title(__A     )
				plt.savefig(__A     )
				plt.clf()
def  __snake_case							(	__A      :						Dict							,		__A      :						Any							,		__A      :						List[str]							,		__A      :						List[Any]							,		__A      :						Dict=None							,		__A      :						Tuple=None     )     ->       Any:






				'''simple docstring'''
				SCREAMING_SNAKE_CASE						:							int							=     sorted(__A							,		key=lambda __A     : na_probs[k]     )
				SCREAMING_SNAKE_CASE						:							Optional[int]							=     0.0
				SCREAMING_SNAKE_CASE						:							Any							=     1.0
				SCREAMING_SNAKE_CASE						:							List[str]							=     0.0
				SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     [1.0]
				SCREAMING_SNAKE_CASE						:							Tuple							=     [0.0]
				SCREAMING_SNAKE_CASE						:							str							=     0.0
				for i, qid in enumerate(__A     ):
								if qid_to_has_ans[qid]:
												true_pos += scores[qid]
								SCREAMING_SNAKE_CASE						:							int							=     true_pos / float(i + 1     )
								SCREAMING_SNAKE_CASE						:							List[str]							=     true_pos / float(__A     )
								if i == len(__A     ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
												# i.e., if we can put a threshold after this point
												avg_prec += cur_p * (cur_r - recalls[-1])
												precisions.append(__A     )
												recalls.append(__A     )
				if out_image:
								plot_pr_curve(__A							,		__A							,		__A							,		__A     )
				return {"ap": 1_0_0.0 * avg_prec}
def  __snake_case							(	__A      :						Tuple							,		__A      :						List[str]							,		__A      :						Union[str, Any]							,		__A      :						Optional[Any]							,		__A      :						Tuple							,		__A      :						str     )     ->       Union[str, Any]:






				'''simple docstring'''
				if out_image_dir and not os.path.exists(__A     ):
								os.makedirs(__A     )
				SCREAMING_SNAKE_CASE						:							Dict							=     sum(1 for v in qid_to_has_ans.values() if v     )
				if num_true_pos == 0:
								return
				SCREAMING_SNAKE_CASE						:							List[Any]							=     make_precision_recall_eval(
				    __A							,		__A							,		__A							,		__A							,		out_image=os.path.join(__A							,		'pr_exact.png'     )							,		title='Precision-Recall curve for Exact Match score'							,		)
				SCREAMING_SNAKE_CASE						:							Optional[Any]							=     make_precision_recall_eval(
				    __A							,		__A							,		__A							,		__A							,		out_image=os.path.join(__A							,		'pr_f1.png'     )							,		title='Precision-Recall curve for F1 score'							,		)
				SCREAMING_SNAKE_CASE						:							str							=     {k: float(__A     ) for k, v in qid_to_has_ans.items()}
				SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     make_precision_recall_eval(
				    __A							,		__A							,		__A							,		__A							,		out_image=os.path.join(__A							,		'pr_oracle.png'     )							,		title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'							,		)
				merge_eval(__A							,		__A							,		'pr_exact'     )
				merge_eval(__A							,		__A							,		'pr_f1'     )
				merge_eval(__A							,		__A							,		'pr_oracle'     )
def  __snake_case							(	__A      :						List[Any]							,		__A      :						str							,		__A      :						Tuple							,		__A      :						int     )     ->       str:






				'''simple docstring'''
				if not qid_list:
								return
				SCREAMING_SNAKE_CASE						:							Any							=     [na_probs[k] for k in qid_list]
				SCREAMING_SNAKE_CASE						:							int							=     np.ones_like(__A     ) / float(len(__A     )     )
				plt.hist(__A							,		weights=__A							,		bins=20							,		range=(0.0, 1.0)     )
				plt.xlabel('Model probability of no-answer'     )
				plt.ylabel('Proportion of dataset'     )
				plt.title(F"""Histogram of no-answer probability: {name}"""     )
				plt.savefig(os.path.join(__A							,		F"""na_prob_hist_{name}.png"""     )     )
				plt.clf()
def  __snake_case							(	__A      :						List[str]							,		__A      :						Optional[Any]							,		__A      :						Union[str, Any]							,		__A      :						Optional[int]     )     ->       Dict:






				'''simple docstring'''
				SCREAMING_SNAKE_CASE						:							str							=     sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]     )
				SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     num_no_ans
				SCREAMING_SNAKE_CASE						:							Tuple							=     cur_score
				SCREAMING_SNAKE_CASE						:							Dict							=     0.0
				SCREAMING_SNAKE_CASE						:							Tuple							=     sorted(__A							,		key=lambda __A     : na_probs[k]     )
				for i, qid in enumerate(__A     ):
								if qid not in scores:
												continue
								if qid_to_has_ans[qid]:
												SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     scores[qid]
								else:
												if preds[qid]:
																SCREAMING_SNAKE_CASE						:							Optional[Any]							=     -1
												else:
																SCREAMING_SNAKE_CASE						:							Optional[int]							=     0
								cur_score += diff
								if cur_score > best_score:
												SCREAMING_SNAKE_CASE						:							str							=     cur_score
												SCREAMING_SNAKE_CASE						:							int							=     na_probs[qid]
				return 1_0_0.0 * best_score / len(__A     ), best_thresh
def  __snake_case							(	__A      :						Any							,		__A      :						Optional[int]							,		__A      :						List[Any]							,		__A      :						Any							,		__A      :						Tuple							,		__A      :						Union[str, Any]     )     ->       Dict:






				'''simple docstring'''
				SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE						:							Optional[int]							=     find_best_thresh(__A							,		__A							,		__A							,		__A     )
				SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE						:							int							=     find_best_thresh(__A							,		__A							,		__A							,		__A     )
				SCREAMING_SNAKE_CASE						:							str							=     best_exact
				SCREAMING_SNAKE_CASE						:							Optional[int]							=     exact_thresh
				SCREAMING_SNAKE_CASE						:							Tuple							=     best_fa
				SCREAMING_SNAKE_CASE						:							List[str]							=     fa_thresh
def  __snake_case							(	)     ->       Optional[Any]:






				'''simple docstring'''
				with open(OPTS.data_file     ) as f:
								SCREAMING_SNAKE_CASE						:							int							=     json.load(__A     )
								SCREAMING_SNAKE_CASE						:							int							=     dataset_json['data']
				with open(OPTS.pred_file     ) as f:
								SCREAMING_SNAKE_CASE						:							Any							=     json.load(__A     )
				if OPTS.na_prob_file:
								with open(OPTS.na_prob_file     ) as f:
												SCREAMING_SNAKE_CASE						:							Optional[Any]							=     json.load(__A     )
				else:
								SCREAMING_SNAKE_CASE						:							List[Any]							=     {k: 0.0 for k in preds}
				SCREAMING_SNAKE_CASE						:							Union[str, Any]							=     make_qid_to_has_ans(__A     )  # maps qid to True/False
				SCREAMING_SNAKE_CASE						:							List[str]							=     [k for k, v in qid_to_has_ans.items() if v]
				SCREAMING_SNAKE_CASE						:							Tuple							=     [k for k, v in qid_to_has_ans.items() if not v]
				SCREAMING_SNAKE_CASE				,						SCREAMING_SNAKE_CASE						:							Optional[int]							=     get_raw_scores(__A							,		__A     )
				SCREAMING_SNAKE_CASE						:							Optional[int]							=     apply_no_ans_threshold(__A							,		__A							,		__A							,		OPTS.na_prob_thresh     )
				SCREAMING_SNAKE_CASE						:							List[Any]							=     apply_no_ans_threshold(__A							,		__A							,		__A							,		OPTS.na_prob_thresh     )
				SCREAMING_SNAKE_CASE						:							Any							=     make_eval_dict(__A							,		__A     )
				if has_ans_qids:
								SCREAMING_SNAKE_CASE						:							Optional[int]							=     make_eval_dict(__A							,		__A							,		qid_list=__A     )
								merge_eval(__A							,		__A							,		'HasAns'     )
				if no_ans_qids:
								SCREAMING_SNAKE_CASE						:							List[str]							=     make_eval_dict(__A							,		__A							,		qid_list=__A     )
								merge_eval(__A							,		__A							,		'NoAns'     )
				if OPTS.na_prob_file:
								find_all_best_thresh(__A							,		__A							,		__A							,		__A							,		__A							,		__A     )
				if OPTS.na_prob_file and OPTS.out_image_dir:
								run_precision_recall_analysis(__A							,		__A							,		__A							,		__A							,		__A							,		OPTS.out_image_dir     )
								histogram_na_prob(__A							,		__A							,		OPTS.out_image_dir							,		'hasAns'     )
								histogram_na_prob(__A							,		__A							,		OPTS.out_image_dir							,		'noAns'     )
				if OPTS.out_file:
								with open(OPTS.out_file							,		'w'     ) as f:
												json.dump(__A							,		__A     )
				else:
								print(json.dumps(__A							,		indent=2     )     )


if __name__ == "__main__":
							A_						:    List[str]				=      parse_args()
							if OPTS.out_image_dir:
														import matplotlib

														matplotlib.use('Agg')
														import matplotlib.pyplot as plt
							main()










"""simple docstring"""

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


A_						:    str				=      {'configuration_unispeech': ['UNISPEECH_PRETRAINED_CONFIG_ARCHIVE_MAP', 'UniSpeechConfig']}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							A_						:    Optional[int]				=      [
							    'UNISPEECH_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'UniSpeechForCTC',
							    'UniSpeechForPreTraining',
							    'UniSpeechForSequenceClassification',
							    'UniSpeechModel',
							    'UniSpeechPreTrainedModel',
							]

if TYPE_CHECKING:
							from .configuration_unispeech import UNISPEECH_PRETRAINED_CONFIG_ARCHIVE_MAP, UniSpeechConfig

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_unispeech import (
														    UNISPEECH_PRETRAINED_MODEL_ARCHIVE_LIST,
														    UniSpeechForCTC,
														    UniSpeechForPreTraining,
														    UniSpeechForSequenceClassification,
														    UniSpeechModel,
														    UniSpeechPreTrainedModel,
														)

else:
							import sys

							A_						:    int				=      _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





