





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCAmelCase_        =							re.compile(r'''\b(a|an|the)\b''', re.UNICODE)

lowerCAmelCase_        =							None






def 	lowerCamelCase_		(							)						->		int:






				"""simple docstring"""


				snake_case_		:       str           =    argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''					)
				parser.add_argument('''data_file'''					,							metavar='''data.json'''					,							help='''Input data JSON file.'''					)
				parser.add_argument('''pred_file'''					,							metavar='''pred.json'''					,							help='''Model predictions.'''					)
				parser.add_argument(
				    '''--out-file'''					,							'''-o'''					,							metavar='''eval.json'''					,							help='''Write accuracy metrics to file (default is stdout).'''					)
				parser.add_argument(
				    '''--na-prob-file'''					,							'''-n'''					,							metavar='''na_prob.json'''					,							help='''Model estimates of probability of no answer.'''					)
				parser.add_argument(
				    '''--na-prob-thresh'''					,							'''-t'''					,							type=snake_case__					,							default=1.0					,							help='''Predict \"\" if no-answer probability exceeds this (default = 1.0).'''					,							)
				parser.add_argument(
				    '''--out-image-dir'''					,							'''-p'''					,							metavar='''out_images'''					,							default=snake_case__					,							help='''Save precision-recall curves to directory.'''					)
				parser.add_argument('''--verbose'''					,							'''-v'''					,							action='''store_true'''					)
				if len(sys.argv					) == 1:
								parser.print_help()
								sys.exit(1					)
				return parser.parse_args()






def 	lowerCamelCase_		(							_UpperCamelCase					)						->		str:






				"""simple docstring"""


				snake_case_		:       Union[str, Any]           =    {}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																snake_case_		:       Union[str, Any]           =    bool(qa['''answers''']['''text''']					)
				return qid_to_has_ans






def 	lowerCamelCase_		(							_UpperCamelCase					)						->		Optional[int]:






				"""simple docstring"""


				def remove_articles(_UpperCamelCase					):
								return ARTICLES_REGEX.sub(''' '''					,							snake_case__					)

				def white_space_fix(_UpperCamelCase					):
								return " ".join(text.split()					)

				def remove_punc(_UpperCamelCase					):
								snake_case_		:       Optional[Any]           =    set(string.punctuation					)
								return "".join(ch for ch in text if ch not in exclude					)

				def lower(_UpperCamelCase					):
								return text.lower()

				return white_space_fix(remove_articles(remove_punc(lower(snake_case__					)					)					)					)






def 	lowerCamelCase_		(							_UpperCamelCase					)						->		int:






				"""simple docstring"""


				if not s:
								return []
				return normalize_answer(snake_case__					).split()






def 	lowerCamelCase_		(							_UpperCamelCase					,							_UpperCamelCase					)						->		Any:






				"""simple docstring"""


				return int(normalize_answer(snake_case__					) == normalize_answer(snake_case__					)					)






def 	lowerCamelCase_		(							_UpperCamelCase					,							_UpperCamelCase					)						->		List[str]:






				"""simple docstring"""


				snake_case_		:       List[str]           =    get_tokens(snake_case__					)
				snake_case_		:       int           =    get_tokens(snake_case__					)
				snake_case_		:       Optional[int]           =    collections.Counter(snake_case__					) & collections.Counter(snake_case__					)
				snake_case_		:       Dict           =    sum(common.values()					)
				if len(snake_case__					) == 0 or len(snake_case__					) == 0:
								# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
								return int(gold_toks == pred_toks					)
				if num_same == 0:
								return 0
				snake_case_		:       List[str]           =    1.0 * num_same / len(snake_case__					)
				snake_case_		:       int           =    1.0 * num_same / len(snake_case__					)
				snake_case_		:       Optional[int]           =    (2 * precision * recall) / (precision + recall)
				return fa






def 	lowerCamelCase_		(							_UpperCamelCase					,							_UpperCamelCase					)						->		List[str]:






				"""simple docstring"""


				snake_case_		:       Any           =    {}
				snake_case_		:       Union[str, Any]           =    {}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																snake_case_		:       Optional[Any]           =    qa["""id"""]
																snake_case_		:       Any           =    [t for t in qa["""answers"""]["""text"""] if normalize_answer(snake_case__					)]
																if not gold_answers:
																				# For unanswerable questions, only correct answer is empty string
																				snake_case_		:       Tuple           =    [""""""]
																if qid not in preds:
																				print(f'''Missing prediction for {qid}'''					)
																				continue
																snake_case_		:       Tuple           =    preds[qid]
																# Take max over all gold answers
																snake_case_		:       Tuple           =    max(compute_exact(snake_case__					,							snake_case__					) for a in gold_answers					)
																snake_case_		:       int           =    max(compute_fa(snake_case__					,							snake_case__					) for a in gold_answers					)
				return exact_scores, fa_scores






def 	lowerCamelCase_		(							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					)						->		List[Any]:






				"""simple docstring"""


				snake_case_		:       List[Any]           =    {}
				for qid, s in scores.items():
								snake_case_		:       Union[str, Any]           =    na_probs[qid] > na_prob_thresh
								if pred_na:
												snake_case_		:       Tuple           =    float(not qid_to_has_ans[qid]					)
								else:
												snake_case_		:       Any           =    s
				return new_scores






def 	lowerCamelCase_		(							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase=None					)						->		int:






				"""simple docstring"""


				if not qid_list:
								snake_case_		:       Optional[int]           =    len(snake_case__					)
								return collections.OrderedDict(
								    [
								        ('''exact''', 100.0 * sum(exact_scores.values()					) / total),
								        ('''f1''', 100.0 * sum(fa_scores.values()					) / total),
								        ('''total''', total),
								    ]					)
				else:
								snake_case_		:       Tuple           =    len(snake_case__					)
								return collections.OrderedDict(
								    [
								        ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list					) / total),
								        ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list					) / total),
								        ('''total''', total),
								    ]					)






def 	lowerCamelCase_		(							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					)						->		Any:






				"""simple docstring"""


				for k in new_eval:
								snake_case_		:       List[str]           =    new_eval[k]






def 	lowerCamelCase_		(							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					)						->		Tuple:






				"""simple docstring"""


				plt.step(snake_case__					,							snake_case__					,							color='''b'''					,							alpha=0.2					,							where='''post'''					)
				plt.fill_between(snake_case__					,							snake_case__					,							step='''post'''					,							alpha=0.2					,							color='''b'''					)
				plt.xlabel('''Recall'''					)
				plt.ylabel('''Precision'''					)
				plt.xlim([0.0, 1.05]					)
				plt.ylim([0.0, 1.05]					)
				plt.title(snake_case__					)
				plt.savefig(snake_case__					)
				plt.clf()






def 	lowerCamelCase_		(							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase=None					,							_UpperCamelCase=None					)						->		Tuple:






				"""simple docstring"""


				snake_case_		:       str           =    sorted(snake_case__					,							key=lambda _UpperCamelCase					: na_probs[k]					)
				snake_case_		:       Tuple           =    0.0
				snake_case_		:       List[Any]           =    1.0
				snake_case_		:       Optional[Any]           =    0.0
				snake_case_		:       Any           =    [1.0]
				snake_case_		:       Union[str, Any]           =    [0.0]
				snake_case_		:       List[Any]           =    0.0
				for i, qid in enumerate(snake_case__					):
								if qid_to_has_ans[qid]:
												true_pos += scores[qid]
								snake_case_		:       Any           =    true_pos / float(i + 1					)
								snake_case_		:       Dict           =    true_pos / float(snake_case__					)
								if i == len(snake_case__					) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
												# i.e., if we can put a threshold after this point
												avg_prec += cur_p * (cur_r - recalls[-1])
												precisions.append(snake_case__					)
												recalls.append(snake_case__					)
				if out_image:
								plot_pr_curve(snake_case__					,							snake_case__					,							snake_case__					,							snake_case__					)
				return {"ap": 100.0 * avg_prec}






def 	lowerCamelCase_		(							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					)						->		Dict:






				"""simple docstring"""


				if out_image_dir and not os.path.exists(snake_case__					):
								os.makedirs(snake_case__					)
				snake_case_		:       Union[str, Any]           =    sum(1 for v in qid_to_has_ans.values() if v					)
				if num_true_pos == 0:
								return
				snake_case_		:       Tuple           =    make_precision_recall_eval(
				    snake_case__					,							snake_case__					,							snake_case__					,							snake_case__					,							out_image=os.path.join(snake_case__					,							'''pr_exact.png'''					)					,							title='''Precision-Recall curve for Exact Match score'''					,							)
				snake_case_		:       Any           =    make_precision_recall_eval(
				    snake_case__					,							snake_case__					,							snake_case__					,							snake_case__					,							out_image=os.path.join(snake_case__					,							'''pr_f1.png'''					)					,							title='''Precision-Recall curve for F1 score'''					,							)
				snake_case_		:       Tuple           =    {k: float(snake_case__					) for k, v in qid_to_has_ans.items()}
				snake_case_		:       int           =    make_precision_recall_eval(
				    snake_case__					,							snake_case__					,							snake_case__					,							snake_case__					,							out_image=os.path.join(snake_case__					,							'''pr_oracle.png'''					)					,							title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''					,							)
				merge_eval(snake_case__					,							snake_case__					,							'''pr_exact'''					)
				merge_eval(snake_case__					,							snake_case__					,							'''pr_f1'''					)
				merge_eval(snake_case__					,							snake_case__					,							'''pr_oracle'''					)






def 	lowerCamelCase_		(							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					)						->		Tuple:






				"""simple docstring"""


				if not qid_list:
								return
				snake_case_		:       int           =    [na_probs[k] for k in qid_list]
				snake_case_		:       List[Any]           =    np.ones_like(snake_case__					) / float(len(snake_case__					)					)
				plt.hist(snake_case__					,							weights=snake_case__					,							bins=20					,							range=(0.0, 1.0)					)
				plt.xlabel('''Model probability of no-answer'''					)
				plt.ylabel('''Proportion of dataset'''					)
				plt.title(f'''Histogram of no-answer probability: {name}'''					)
				plt.savefig(os.path.join(snake_case__					,							f'''na_prob_hist_{name}.png'''					)					)
				plt.clf()






def 	lowerCamelCase_		(							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					)						->		Optional[int]:






				"""simple docstring"""


				snake_case_		:       Dict           =    sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]					)
				snake_case_		:       List[str]           =    num_no_ans
				snake_case_		:       Union[str, Any]           =    cur_score
				snake_case_		:       Any           =    0.0
				snake_case_		:       Dict           =    sorted(snake_case__					,							key=lambda _UpperCamelCase					: na_probs[k]					)
				for i, qid in enumerate(snake_case__					):
								if qid not in scores:
												continue
								if qid_to_has_ans[qid]:
												snake_case_		:       List[Any]           =    scores[qid]
								else:
												if preds[qid]:
																snake_case_		:       Union[str, Any]           =    -1
												else:
																snake_case_		:       List[Any]           =    0
								cur_score += diff
								if cur_score > best_score:
												snake_case_		:       Union[str, Any]           =    cur_score
												snake_case_		:       Dict           =    na_probs[qid]
				return 100.0 * best_score / len(snake_case__					), best_thresh






def 	lowerCamelCase_		(							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					,							_UpperCamelCase					)						->		Optional[Any]:






				"""simple docstring"""


				snake_case_		:       Dict           =    find_best_thresh(snake_case__					,							snake_case__					,							snake_case__					,							snake_case__					)
				snake_case_		:       Optional[int]           =    find_best_thresh(snake_case__					,							snake_case__					,							snake_case__					,							snake_case__					)
				snake_case_		:       Dict           =    best_exact
				snake_case_		:       List[str]           =    exact_thresh
				snake_case_		:       str           =    best_fa
				snake_case_		:       int           =    fa_thresh






def 	lowerCamelCase_		(							)						->		Optional[int]:






				"""simple docstring"""


				with open(OPTS.data_file					) as f:
								snake_case_		:       Any           =    json.load(snake_case__					)
								snake_case_		:       List[Any]           =    dataset_json["""data"""]
				with open(OPTS.pred_file					) as f:
								snake_case_		:       List[Any]           =    json.load(snake_case__					)
				if OPTS.na_prob_file:
								with open(OPTS.na_prob_file					) as f:
												snake_case_		:       Dict           =    json.load(snake_case__					)
				else:
								snake_case_		:       Union[str, Any]           =    {k: 0.0 for k in preds}
				snake_case_		:       Union[str, Any]           =    make_qid_to_has_ans(snake_case__					)  # maps qid to True/False
				snake_case_		:       int           =    [k for k, v in qid_to_has_ans.items() if v]
				snake_case_		:       Tuple           =    [k for k, v in qid_to_has_ans.items() if not v]
				snake_case_		:       List[Any]           =    get_raw_scores(snake_case__					,							snake_case__					)
				snake_case_		:       Optional[int]           =    apply_no_ans_threshold(snake_case__					,							snake_case__					,							snake_case__					,							OPTS.na_prob_thresh					)
				snake_case_		:       int           =    apply_no_ans_threshold(snake_case__					,							snake_case__					,							snake_case__					,							OPTS.na_prob_thresh					)
				snake_case_		:       Optional[int]           =    make_eval_dict(snake_case__					,							snake_case__					)
				if has_ans_qids:
								snake_case_		:       Optional[Any]           =    make_eval_dict(snake_case__					,							snake_case__					,							qid_list=snake_case__					)
								merge_eval(snake_case__					,							snake_case__					,							'''HasAns'''					)
				if no_ans_qids:
								snake_case_		:       Optional[Any]           =    make_eval_dict(snake_case__					,							snake_case__					,							qid_list=snake_case__					)
								merge_eval(snake_case__					,							snake_case__					,							'''NoAns'''					)
				if OPTS.na_prob_file:
								find_all_best_thresh(snake_case__					,							snake_case__					,							snake_case__					,							snake_case__					,							snake_case__					,							snake_case__					)
				if OPTS.na_prob_file and OPTS.out_image_dir:
								run_precision_recall_analysis(snake_case__					,							snake_case__					,							snake_case__					,							snake_case__					,							snake_case__					,							OPTS.out_image_dir					)
								histogram_na_prob(snake_case__					,							snake_case__					,							OPTS.out_image_dir					,							'''hasAns'''					)
								histogram_na_prob(snake_case__					,							snake_case__					,							OPTS.out_image_dir					,							'''noAns'''					)
				if OPTS.out_file:
								with open(OPTS.out_file					,							'''w'''					) as f:
												json.dump(snake_case__					,							snake_case__					)
				else:
								print(json.dumps(snake_case__					,							indent=2					)					)


if __name__ == "__main__":
	lowerCAmelCase_        =							parse_args()
	if OPTS.out_image_dir:
		import matplotlib

		matplotlib.use('''Agg''')
		import matplotlib.pyplot as plt
	main()








def 	UpperCamelCase	(							snake_case__  : list	):




     '''simple docstring'''






     if not grid or not grid[0]:
          raise TypeError("""The grid does not contain the appropriate information"""	)

     for cell_n in range(1						,len(grid[0]	)	):
          grid[0][cell_n] += grid[0][cell_n - 1]
     __snake_case      :str 							=    grid[0]

     for row_n in range(1						,len(snake_case__	)	):
          __snake_case      :Optional[int] 							=    grid[row_n]
          __snake_case      :Optional[Any] 							=    fill_row(snake_case__						,snake_case__	)
          __snake_case      :Dict 							=    grid[row_n]

     return grid[-1][-1]






def 	UpperCamelCase	(							snake_case__  : list						,snake_case__  : list	):




     '''simple docstring'''






     current_row[0] += row_above[0]
     for cell_n in range(1						,len(snake_case__	)	):
          current_row[cell_n] += min(current_row[cell_n - 1]						,row_above[cell_n]	)

     return current_row


if __name__ == "__main__":
     import doctest

     doctest.testmod()


