
'''simple docstring'''







import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowercase					:							List[str]   =      re.compile(r"\b(a|an|the)\b", re.UNICODE)

lowercase					:							Union[str, Any]   =      None







def        SCREAMING_SNAKE_CASE__			(     ) ->			str:
	_snake_case     		=	argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'			)
	parser.add_argument('data_file'    ,       metavar='data.json'    ,       help='Input data JSON file.'			)
	parser.add_argument('pred_file'    ,       metavar='pred.json'    ,       help='Model predictions.'			)
	parser.add_argument(
	    '--out-file'    ,       '-o'    ,       metavar='eval.json'    ,       help='Write accuracy metrics to file (default is stdout).'			)
	parser.add_argument(
	    '--na-prob-file'    ,       '-n'    ,       metavar='na_prob.json'    ,       help='Model estimates of probability of no answer.'			)
	parser.add_argument(
	    '--na-prob-thresh'    ,       '-t'    ,       type=__A    ,       default=1.0    ,       help='Predict "" if no-answer probability exceeds this (default = 1.0).'    ,       )
	parser.add_argument(
	    '--out-image-dir'    ,       '-p'    ,       metavar='out_images'    ,       default=__A    ,       help='Save precision-recall curves to directory.'			)
	parser.add_argument('--verbose'    ,       '-v'    ,       action='store_true'			)
	if len(sys.argv			) == 1:
		parser.print_help()
		sys.exit(1			)
	return parser.parse_args()







def        SCREAMING_SNAKE_CASE__			(     __A			) ->			Union[str, Any]:
	_snake_case     		=	{}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				_snake_case     		=	bool(qa['answers']['text']			)
	return qid_to_has_ans







def        SCREAMING_SNAKE_CASE__			(     __A			) ->			Optional[Any]:

	def remove_articles(__A			):
		return ARTICLES_REGEX.sub(' '    ,       __A			)

	def white_space_fix(__A			):
		return " ".join(text.split()			)

	def remove_punc(__A			):
		_snake_case     		=	set(string.punctuation			)
		return "".join(ch for ch in text if ch not in exclude			)

	def lower(__A			):
		return text.lower()

	return white_space_fix(remove_articles(remove_punc(lower(__A			)			)			)			)







def        SCREAMING_SNAKE_CASE__			(     __A			) ->			Optional[int]:
	if not s:
		return []
	return normalize_answer(__A			).split()







def        SCREAMING_SNAKE_CASE__			(     __A    ,       __A			) ->			List[Any]:
	return int(normalize_answer(__A			) == normalize_answer(__A			)			)







def        SCREAMING_SNAKE_CASE__			(     __A    ,       __A			) ->			List[str]:
	_snake_case     		=	get_tokens(__A			)
	_snake_case     		=	get_tokens(__A			)
	_snake_case     		=	collections.Counter(__A			) & collections.Counter(__A			)
	_snake_case     		=	sum(common.values()			)
	if len(__A			) == 0 or len(__A			) == 0:
		# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
		return int(gold_toks == pred_toks			)
	if num_same == 0:
		return 0
	_snake_case     		=	1.0 * num_same / len(__A			)
	_snake_case     		=	1.0 * num_same / len(__A			)
	_snake_case     		=	(2 * precision * recall) / (precision + recall)
	return fa







def        SCREAMING_SNAKE_CASE__			(     __A    ,       __A			) ->			Any:
	_snake_case     		=	{}
	_snake_case     		=	{}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				_snake_case     		=	qa['id']
				_snake_case     		=	[t for t in qa['answers']['text'] if normalize_answer(__A			)]
				if not gold_answers:
					# For unanswerable questions, only correct answer is empty string
					_snake_case     		=	['']
				if qid not in preds:
					print(F'Missing prediction for {qid}'			)
					continue
				_snake_case     		=	preds[qid]
				# Take max over all gold answers
				_snake_case     		=	max(compute_exact(__A    ,       __A			) for a in gold_answers			)
				_snake_case     		=	max(compute_fa(__A    ,       __A			) for a in gold_answers			)
	return exact_scores, fa_scores







def        SCREAMING_SNAKE_CASE__			(     __A    ,       __A    ,       __A    ,       __A			) ->			int:
	_snake_case     		=	{}
	for qid, s in scores.items():
		_snake_case     		=	na_probs[qid] > na_prob_thresh
		if pred_na:
			_snake_case     		=	float(not qid_to_has_ans[qid]			)
		else:
			_snake_case     		=	s
	return new_scores







def        SCREAMING_SNAKE_CASE__			(     __A    ,       __A    ,       __A=None			) ->			Dict:
	if not qid_list:
		_snake_case     		=	len(__A			)
		return collections.OrderedDict(
		    [
		        ('exact', 1_0_0.0 * sum(exact_scores.values()			) / total),
		        ('f1', 1_0_0.0 * sum(fa_scores.values()			) / total),
		        ('total', total),
		    ]			)
	else:
		_snake_case     		=	len(__A			)
		return collections.OrderedDict(
		    [
		        ('exact', 1_0_0.0 * sum(exact_scores[k] for k in qid_list			) / total),
		        ('f1', 1_0_0.0 * sum(fa_scores[k] for k in qid_list			) / total),
		        ('total', total),
		    ]			)







def        SCREAMING_SNAKE_CASE__			(     __A    ,       __A    ,       __A			) ->			int:
	for k in new_eval:
		_snake_case     		=	new_eval[k]







def        SCREAMING_SNAKE_CASE__			(     __A    ,       __A    ,       __A    ,       __A			) ->			Tuple:
	plt.step(__A    ,       __A    ,       color='b'    ,       alpha=0.2    ,       where='post'			)
	plt.fill_between(__A    ,       __A    ,       step='post'    ,       alpha=0.2    ,       color='b'			)
	plt.xlabel('Recall'			)
	plt.ylabel('Precision'			)
	plt.xlim([0.0, 1.0_5]			)
	plt.ylim([0.0, 1.0_5]			)
	plt.title(__A			)
	plt.savefig(__A			)
	plt.clf()







def        SCREAMING_SNAKE_CASE__			(     __A    ,       __A    ,       __A    ,       __A    ,       __A=None    ,       __A=None			) ->			str:
	_snake_case     		=	sorted(__A    ,       key=lambda __A			: na_probs[k]			)
	_snake_case     		=	0.0
	_snake_case     		=	1.0
	_snake_case     		=	0.0
	_snake_case     		=	[1.0]
	_snake_case     		=	[0.0]
	_snake_case     		=	0.0
	for i, qid in enumerate(__A			):
		if qid_to_has_ans[qid]:
			true_pos += scores[qid]
		_snake_case     		=	true_pos / float(i + 1			)
		_snake_case     		=	true_pos / float(__A			)
		if i == len(__A			) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
			# i.e., if we can put a threshold after this point
			avg_prec += cur_p * (cur_r - recalls[-1])
			precisions.append(__A			)
			recalls.append(__A			)
	if out_image:
		plot_pr_curve(__A    ,       __A    ,       __A    ,       __A			)
	return {"ap": 1_0_0.0 * avg_prec}







def        SCREAMING_SNAKE_CASE__			(     __A    ,       __A    ,       __A    ,       __A    ,       __A    ,       __A			) ->			int:
	if out_image_dir and not os.path.exists(__A			):
		os.makedirs(__A			)
	_snake_case     		=	sum(1 for v in qid_to_has_ans.values() if v			)
	if num_true_pos == 0:
		return
	_snake_case     		=	make_precision_recall_eval(
	    __A    ,       __A    ,       __A    ,       __A    ,       out_image=os.path.join(__A    ,       'pr_exact.png'			)    ,       title='Precision-Recall curve for Exact Match score'    ,       )
	_snake_case     		=	make_precision_recall_eval(
	    __A    ,       __A    ,       __A    ,       __A    ,       out_image=os.path.join(__A    ,       'pr_f1.png'			)    ,       title='Precision-Recall curve for F1 score'    ,       )
	_snake_case     		=	{k: float(__A			) for k, v in qid_to_has_ans.items()}
	_snake_case     		=	make_precision_recall_eval(
	    __A    ,       __A    ,       __A    ,       __A    ,       out_image=os.path.join(__A    ,       'pr_oracle.png'			)    ,       title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'    ,       )
	merge_eval(__A    ,       __A    ,       'pr_exact'			)
	merge_eval(__A    ,       __A    ,       'pr_f1'			)
	merge_eval(__A    ,       __A    ,       'pr_oracle'			)







def        SCREAMING_SNAKE_CASE__			(     __A    ,       __A    ,       __A    ,       __A			) ->			Tuple:
	if not qid_list:
		return
	_snake_case     		=	[na_probs[k] for k in qid_list]
	_snake_case     		=	np.ones_like(__A			) / float(len(__A			)			)
	plt.hist(__A    ,       weights=__A    ,       bins=20    ,       range=(0.0, 1.0)			)
	plt.xlabel('Model probability of no-answer'			)
	plt.ylabel('Proportion of dataset'			)
	plt.title(F'Histogram of no-answer probability: {name}'			)
	plt.savefig(os.path.join(__A    ,       F'na_prob_hist_{name}.png'			)			)
	plt.clf()







def        SCREAMING_SNAKE_CASE__			(     __A    ,       __A    ,       __A    ,       __A			) ->			Any:
	_snake_case     		=	sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]			)
	_snake_case     		=	num_no_ans
	_snake_case     		=	cur_score
	_snake_case     		=	0.0
	_snake_case     		=	sorted(__A    ,       key=lambda __A			: na_probs[k]			)
	for i, qid in enumerate(__A			):
		if qid not in scores:
			continue
		if qid_to_has_ans[qid]:
			_snake_case     		=	scores[qid]
		else:
			if preds[qid]:
				_snake_case     		=	-1
			else:
				_snake_case     		=	0
		cur_score += diff
		if cur_score > best_score:
			_snake_case     		=	cur_score
			_snake_case     		=	na_probs[qid]
	return 1_0_0.0 * best_score / len(__A			), best_thresh







def        SCREAMING_SNAKE_CASE__			(     __A    ,       __A    ,       __A    ,       __A    ,       __A    ,       __A			) ->			List[Any]:
	_snake_case		,				_snake_case     		=	find_best_thresh(__A    ,       __A    ,       __A    ,       __A			)
	_snake_case		,				_snake_case     		=	find_best_thresh(__A    ,       __A    ,       __A    ,       __A			)
	_snake_case     		=	best_exact
	_snake_case     		=	exact_thresh
	_snake_case     		=	best_fa
	_snake_case     		=	fa_thresh







def        SCREAMING_SNAKE_CASE__			(     ) ->			Optional[Any]:
	with open(OPTS.data_file			) as f:
		_snake_case     		=	json.load(__A			)
		_snake_case     		=	dataset_json['data']
	with open(OPTS.pred_file			) as f:
		_snake_case     		=	json.load(__A			)
	if OPTS.na_prob_file:
		with open(OPTS.na_prob_file			) as f:
			_snake_case     		=	json.load(__A			)
	else:
		_snake_case     		=	{k: 0.0 for k in preds}
	_snake_case     		=	make_qid_to_has_ans(__A			)  # maps qid to True/False
	_snake_case     		=	[k for k, v in qid_to_has_ans.items() if v]
	_snake_case     		=	[k for k, v in qid_to_has_ans.items() if not v]
	_snake_case		,				_snake_case     		=	get_raw_scores(__A    ,       __A			)
	_snake_case     		=	apply_no_ans_threshold(__A    ,       __A    ,       __A    ,       OPTS.na_prob_thresh			)
	_snake_case     		=	apply_no_ans_threshold(__A    ,       __A    ,       __A    ,       OPTS.na_prob_thresh			)
	_snake_case     		=	make_eval_dict(__A    ,       __A			)
	if has_ans_qids:
		_snake_case     		=	make_eval_dict(__A    ,       __A    ,       qid_list=__A			)
		merge_eval(__A    ,       __A    ,       'HasAns'			)
	if no_ans_qids:
		_snake_case     		=	make_eval_dict(__A    ,       __A    ,       qid_list=__A			)
		merge_eval(__A    ,       __A    ,       'NoAns'			)
	if OPTS.na_prob_file:
		find_all_best_thresh(__A    ,       __A    ,       __A    ,       __A    ,       __A    ,       __A			)
	if OPTS.na_prob_file and OPTS.out_image_dir:
		run_precision_recall_analysis(__A    ,       __A    ,       __A    ,       __A    ,       __A    ,       OPTS.out_image_dir			)
		histogram_na_prob(__A    ,       __A    ,       OPTS.out_image_dir    ,       'hasAns'			)
		histogram_na_prob(__A    ,       __A    ,       OPTS.out_image_dir    ,       'noAns'			)
	if OPTS.out_file:
		with open(OPTS.out_file    ,       'w'			) as f:
			json.dump(__A    ,       __A			)
	else:
		print(json.dumps(__A    ,       indent=2			)			)


if __name__ == "__main__":
					lowercase					:							Optional[Any]   =      parse_args()
					if OPTS.out_image_dir:
										import matplotlib

										matplotlib.use("Agg")
										import matplotlib.pyplot as plt
					main()



'''simple docstring'''







lowercase					:							Tuple   =      {
    "a": "AAAAA",
    "b": "AAAAB",
    "c": "AAABA",
    "d": "AAABB",
    "e": "AABAA",
    "f": "AABAB",
    "g": "AABBA",
    "h": "AABBB",
    "i": "ABAAA",
    "j": "BBBAA",
    "k": "ABAAB",
    "l": "ABABA",
    "m": "ABABB",
    "n": "ABBAA",
    "o": "ABBAB",
    "p": "ABBBA",
    "q": "ABBBB",
    "r": "BAAAA",
    "s": "BAAAB",
    "t": "BAABA",
    "u": "BAABB",
    "v": "BBBAB",
    "w": "BABAA",
    "x": "BABAB",
    "y": "BABBA",
    "z": "BABBB",
    " ": " ",
}


lowercase					:							Union[str, Any]   =      {value: key for key, value in encode_dict.items()}







def        SCREAMING_SNAKE_CASE__			(     __A			) ->			str:
	_snake_case     		=	''
	for letter in word.lower():
		if letter.isalpha() or letter == " ":
			encoded += encode_dict[letter]
		else:
			raise Exception('encode() accepts only letters of the alphabet and spaces'			)
	return encoded







def        SCREAMING_SNAKE_CASE__			(     __A			) ->			str:
	if set(__A			) - {"A", "B", " "} != set():
		raise Exception('decode() accepts only \'A\', \'B\' and spaces'			)
	_snake_case     		=	''
	for word in coded.split():
		while len(__A			) != 0:
			decoded += decode_dict[word[:5]]
			_snake_case     		=	word[5:]
		decoded += " "
	return decoded.strip()


if __name__ == "__main__":
					from doctest import testmod

					testmod()

