

'''simple docstring'''





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


UpperCamelCase_							=					re.compile(r"""\b(a|an|the)\b""", re.UNICODE)

UpperCamelCase_							=					None






def 							_lowerCAmelCase (      )							->     List[str]:
						lowercase   :       Any								=argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''	)
						parser.add_argument('''data_file'''      ,   metavar='''data.json'''      ,   help='''Input data JSON file.'''	)
						parser.add_argument('''pred_file'''      ,   metavar='''pred.json'''      ,   help='''Model predictions.'''	)
						parser.add_argument(
						    '''--out-file'''      ,   '''-o'''      ,   metavar='''eval.json'''      ,   help='''Write accuracy metrics to file (default is stdout).'''	)
						parser.add_argument(
						    '''--na-prob-file'''      ,   '''-n'''      ,   metavar='''na_prob.json'''      ,   help='''Model estimates of probability of no answer.'''	)
						parser.add_argument(
						    '''--na-prob-thresh'''      ,   '''-t'''      ,   type=__magic_name__      ,   default=1.0      ,   help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''      ,   )
						parser.add_argument(
						    '''--out-image-dir'''      ,   '''-p'''      ,   metavar='''out_images'''      ,   default=__magic_name__      ,   help='''Save precision-recall curves to directory.'''	)
						parser.add_argument('''--verbose'''      ,   '''-v'''      ,   action='''store_true'''	)
						if len(sys.argv	) == 1:
												parser.print_help()
												sys.exit(1	)
						return parser.parse_args()






def 							_lowerCAmelCase (      __magic_name__	:    Any	)							->     int:
						lowercase   :       List[Any]								={}
						for article in dataset:
												for p in article["paragraphs"]:
																		for qa in p["qas"]:
																								lowercase   :       Union[str, Any]								=bool(qa['''answers''']['''text''']	)
						return qid_to_has_ans






def 							_lowerCAmelCase (      __magic_name__	:    Tuple	)							->     List[str]:

						def remove_articles(__magic_name__	:    Optional[int]	):
												return ARTICLES_REGEX.sub(''' '''      ,   __magic_name__	)

						def white_space_fix(__magic_name__	:    List[str]	):
												return " ".join(text.split()	)

						def remove_punc(__magic_name__	:    List[Any]	):
												lowercase   :       Any								=set(string.punctuation	)
												return "".join(ch for ch in text if ch not in exclude	)

						def lower(__magic_name__	:    List[Any]	):
												return text.lower()

						return white_space_fix(remove_articles(remove_punc(lower(__magic_name__	)	)	)	)






def 							_lowerCAmelCase (      __magic_name__	:    Any	)							->     str:
						if not s:
												return []
						return normalize_answer(__magic_name__	).split()






def 							_lowerCAmelCase (      __magic_name__	:    List[str]      ,   __magic_name__	:    Optional[int]	)							->     Tuple:
						return int(normalize_answer(__magic_name__	) == normalize_answer(__magic_name__	)	)






def 							_lowerCAmelCase (      __magic_name__	:    List[Any]      ,   __magic_name__	:    Optional[Any]	)							->     Dict:
						lowercase   :       Union[str, Any]								=get_tokens(__magic_name__	)
						lowercase   :       Optional[Any]								=get_tokens(__magic_name__	)
						lowercase   :       str								=collections.Counter(__magic_name__	) & collections.Counter(__magic_name__	)
						lowercase   :       Optional[int]								=sum(common.values()	)
						if len(__magic_name__	) == 0 or len(__magic_name__	) == 0:
												# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
												return int(gold_toks == pred_toks	)
						if num_same == 0:
												return 0
						lowercase   :       Union[str, Any]								=1.0 * num_same / len(__magic_name__	)
						lowercase   :       List[Any]								=1.0 * num_same / len(__magic_name__	)
						lowercase   :       List[Any]								=(2 * precision * recall) / (precision + recall)
						return fa






def 							_lowerCAmelCase (      __magic_name__	:    Any      ,   __magic_name__	:    int	)							->     Optional[Any]:
						lowercase   :       Any								={}
						lowercase   :       Dict								={}
						for article in dataset:
												for p in article["paragraphs"]:
																		for qa in p["qas"]:
																								lowercase   :       Union[str, Any]								=qa['''id''']
																								lowercase   :       int								=[t for t in qa['''answers''']['''text'''] if normalize_answer(__magic_name__	)]
																								if not gold_answers:
																														# For unanswerable questions, only correct answer is empty string
																														lowercase   :       Union[str, Any]								=['''''']
																								if qid not in preds:
																														print(f'''Missing prediction for {qid}'''	)
																														continue
																								lowercase   :       Optional[int]								=preds[qid]
																								# Take max over all gold answers
																								lowercase   :       List[str]								=max(compute_exact(__magic_name__      ,   __magic_name__	) for a in gold_answers	)
																								lowercase   :       Optional[int]								=max(compute_fa(__magic_name__      ,   __magic_name__	) for a in gold_answers	)
						return exact_scores, fa_scores






def 							_lowerCAmelCase (      __magic_name__	:    Union[str, Any]      ,   __magic_name__	:    List[Any]      ,   __magic_name__	:    Tuple      ,   __magic_name__	:    Any	)							->     List[Any]:
						lowercase   :       List[str]								={}
						for qid, s in scores.items():
												lowercase   :       Optional[Any]								=na_probs[qid] > na_prob_thresh
												if pred_na:
																		lowercase   :       Union[str, Any]								=float(not qid_to_has_ans[qid]	)
												else:
																		lowercase   :       Union[str, Any]								=s
						return new_scores






def 							_lowerCAmelCase (      __magic_name__	:    str      ,   __magic_name__	:    Optional[Any]      ,   __magic_name__	:    Tuple=None	)							->     Union[str, Any]:
						if not qid_list:
												lowercase   :       Union[str, Any]								=len(__magic_name__	)
												return collections.OrderedDict(
												    [
												        ('''exact''', 1_0_0.0 * sum(exact_scores.values()	) / total),
												        ('''f1''', 1_0_0.0 * sum(fa_scores.values()	) / total),
												        ('''total''', total),
												    ]	)
						else:
												lowercase   :       Union[str, Any]								=len(__magic_name__	)
												return collections.OrderedDict(
												    [
												        ('''exact''', 1_0_0.0 * sum(exact_scores[k] for k in qid_list	) / total),
												        ('''f1''', 1_0_0.0 * sum(fa_scores[k] for k in qid_list	) / total),
												        ('''total''', total),
												    ]	)






def 							_lowerCAmelCase (      __magic_name__	:    List[str]      ,   __magic_name__	:    Optional[int]      ,   __magic_name__	:    int	)							->     Optional[Any]:
						for k in new_eval:
												lowercase   :       Optional[int]								=new_eval[k]






def 							_lowerCAmelCase (      __magic_name__	:    List[Any]      ,   __magic_name__	:    List[str]      ,   __magic_name__	:    Optional[int]      ,   __magic_name__	:    Dict	)							->     Union[str, Any]:
						plt.step(__magic_name__      ,   __magic_name__      ,   color='''b'''      ,   alpha=0.2      ,   where='''post'''	)
						plt.fill_between(__magic_name__      ,   __magic_name__      ,   step='''post'''      ,   alpha=0.2      ,   color='''b'''	)
						plt.xlabel('''Recall'''	)
						plt.ylabel('''Precision'''	)
						plt.xlim([0.0, 1.0_5]	)
						plt.ylim([0.0, 1.0_5]	)
						plt.title(__magic_name__	)
						plt.savefig(__magic_name__	)
						plt.clf()






def 							_lowerCAmelCase (      __magic_name__	:    List[str]      ,   __magic_name__	:    Tuple      ,   __magic_name__	:    Optional[int]      ,   __magic_name__	:    int      ,   __magic_name__	:    Optional[int]=None      ,   __magic_name__	:    Optional[Any]=None	)							->     Optional[Any]:
						lowercase   :       int								=sorted(__magic_name__      ,   key=lambda __magic_name__	: na_probs[k]	)
						lowercase   :       int								=0.0
						lowercase   :       int								=1.0
						lowercase   :       Optional[int]								=0.0
						lowercase   :       Tuple								=[1.0]
						lowercase   :       Union[str, Any]								=[0.0]
						lowercase   :       Any								=0.0
						for i, qid in enumerate(__magic_name__	):
												if qid_to_has_ans[qid]:
																		true_pos += scores[qid]
												lowercase   :       List[str]								=true_pos / float(i + 1	)
												lowercase   :       List[str]								=true_pos / float(__magic_name__	)
												if i == len(__magic_name__	) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
																		# i.e., if we can put a threshold after this point
																		avg_prec += cur_p * (cur_r - recalls[-1])
																		precisions.append(__magic_name__	)
																		recalls.append(__magic_name__	)
						if out_image:
												plot_pr_curve(__magic_name__      ,   __magic_name__      ,   __magic_name__      ,   __magic_name__	)
						return {"ap": 1_0_0.0 * avg_prec}






def 							_lowerCAmelCase (      __magic_name__	:    Dict      ,   __magic_name__	:    List[str]      ,   __magic_name__	:    Optional[int]      ,   __magic_name__	:    Dict      ,   __magic_name__	:    List[str]      ,   __magic_name__	:    List[Any]	)							->     List[str]:
						if out_image_dir and not os.path.exists(__magic_name__	):
												os.makedirs(__magic_name__	)
						lowercase   :       Optional[Any]								=sum(1 for v in qid_to_has_ans.values() if v	)
						if num_true_pos == 0:
												return
						lowercase   :       List[str]								=make_precision_recall_eval(
						    __magic_name__      ,   __magic_name__      ,   __magic_name__      ,   __magic_name__      ,   out_image=os.path.join(__magic_name__      ,   '''pr_exact.png'''	)      ,   title='''Precision-Recall curve for Exact Match score'''      ,   )
						lowercase   :       List[Any]								=make_precision_recall_eval(
						    __magic_name__      ,   __magic_name__      ,   __magic_name__      ,   __magic_name__      ,   out_image=os.path.join(__magic_name__      ,   '''pr_f1.png'''	)      ,   title='''Precision-Recall curve for F1 score'''      ,   )
						lowercase   :       Optional[Any]								={k: float(__magic_name__	) for k, v in qid_to_has_ans.items()}
						lowercase   :       List[str]								=make_precision_recall_eval(
						    __magic_name__      ,   __magic_name__      ,   __magic_name__      ,   __magic_name__      ,   out_image=os.path.join(__magic_name__      ,   '''pr_oracle.png'''	)      ,   title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''      ,   )
						merge_eval(__magic_name__      ,   __magic_name__      ,   '''pr_exact'''	)
						merge_eval(__magic_name__      ,   __magic_name__      ,   '''pr_f1'''	)
						merge_eval(__magic_name__      ,   __magic_name__      ,   '''pr_oracle'''	)






def 							_lowerCAmelCase (      __magic_name__	:    List[str]      ,   __magic_name__	:    List[Any]      ,   __magic_name__	:    List[Any]      ,   __magic_name__	:    Optional[Any]	)							->     Optional[int]:
						if not qid_list:
												return
						lowercase   :       str								=[na_probs[k] for k in qid_list]
						lowercase   :       int								=np.ones_like(__magic_name__	) / float(len(__magic_name__	)	)
						plt.hist(__magic_name__      ,   weights=__magic_name__      ,   bins=20      ,   range=(0.0, 1.0)	)
						plt.xlabel('''Model probability of no-answer'''	)
						plt.ylabel('''Proportion of dataset'''	)
						plt.title(f'''Histogram of no-answer probability: {name}'''	)
						plt.savefig(os.path.join(__magic_name__      ,   f'''na_prob_hist_{name}.png'''	)	)
						plt.clf()






def 							_lowerCAmelCase (      __magic_name__	:    str      ,   __magic_name__	:    int      ,   __magic_name__	:    Tuple      ,   __magic_name__	:    List[Any]	)							->     Any:
						lowercase   :       List[str]								=sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]	)
						lowercase   :       Any								=num_no_ans
						lowercase   :       Union[str, Any]								=cur_score
						lowercase   :       Tuple								=0.0
						lowercase   :       int								=sorted(__magic_name__      ,   key=lambda __magic_name__	: na_probs[k]	)
						for i, qid in enumerate(__magic_name__	):
												if qid not in scores:
																		continue
												if qid_to_has_ans[qid]:
																		lowercase   :       Tuple								=scores[qid]
												else:
																		if preds[qid]:
																								lowercase   :       Any								=-1
																		else:
																								lowercase   :       Tuple								=0
												cur_score += diff
												if cur_score > best_score:
																		lowercase   :       Tuple								=cur_score
																		lowercase   :       Any								=na_probs[qid]
						return 1_0_0.0 * best_score / len(__magic_name__	), best_thresh






def 							_lowerCAmelCase (      __magic_name__	:    List[Any]      ,   __magic_name__	:    int      ,   __magic_name__	:    List[Any]      ,   __magic_name__	:    int      ,   __magic_name__	:    int      ,   __magic_name__	:    Any	)							->     Dict:
						lowercase		,  lowercase   :       Union[str, Any]								=find_best_thresh(__magic_name__      ,   __magic_name__      ,   __magic_name__      ,   __magic_name__	)
						lowercase		,  lowercase   :       Union[str, Any]								=find_best_thresh(__magic_name__      ,   __magic_name__      ,   __magic_name__      ,   __magic_name__	)
						lowercase   :       Optional[Any]								=best_exact
						lowercase   :       Optional[Any]								=exact_thresh
						lowercase   :       str								=best_fa
						lowercase   :       Tuple								=fa_thresh






def 							_lowerCAmelCase (      )							->     List[Any]:
						with open(OPTS.data_file	) as f:
												lowercase   :       Tuple								=json.load(__magic_name__	)
												lowercase   :       List[Any]								=dataset_json['''data''']
						with open(OPTS.pred_file	) as f:
												lowercase   :       Optional[int]								=json.load(__magic_name__	)
						if OPTS.na_prob_file:
												with open(OPTS.na_prob_file	) as f:
																		lowercase   :       Union[str, Any]								=json.load(__magic_name__	)
						else:
												lowercase   :       Optional[int]								={k: 0.0 for k in preds}
						lowercase   :       List[str]								=make_qid_to_has_ans(__magic_name__	)  # maps qid to True/False
						lowercase   :       Optional[Any]								=[k for k, v in qid_to_has_ans.items() if v]
						lowercase   :       List[Any]								=[k for k, v in qid_to_has_ans.items() if not v]
						lowercase		,  lowercase   :       int								=get_raw_scores(__magic_name__      ,   __magic_name__	)
						lowercase   :       Optional[Any]								=apply_no_ans_threshold(__magic_name__      ,   __magic_name__      ,   __magic_name__      ,   OPTS.na_prob_thresh	)
						lowercase   :       Dict								=apply_no_ans_threshold(__magic_name__      ,   __magic_name__      ,   __magic_name__      ,   OPTS.na_prob_thresh	)
						lowercase   :       List[Any]								=make_eval_dict(__magic_name__      ,   __magic_name__	)
						if has_ans_qids:
												lowercase   :       Optional[Any]								=make_eval_dict(__magic_name__      ,   __magic_name__      ,   qid_list=__magic_name__	)
												merge_eval(__magic_name__      ,   __magic_name__      ,   '''HasAns'''	)
						if no_ans_qids:
												lowercase   :       int								=make_eval_dict(__magic_name__      ,   __magic_name__      ,   qid_list=__magic_name__	)
												merge_eval(__magic_name__      ,   __magic_name__      ,   '''NoAns'''	)
						if OPTS.na_prob_file:
												find_all_best_thresh(__magic_name__      ,   __magic_name__      ,   __magic_name__      ,   __magic_name__      ,   __magic_name__      ,   __magic_name__	)
						if OPTS.na_prob_file and OPTS.out_image_dir:
												run_precision_recall_analysis(__magic_name__      ,   __magic_name__      ,   __magic_name__      ,   __magic_name__      ,   __magic_name__      ,   OPTS.out_image_dir	)
												histogram_na_prob(__magic_name__      ,   __magic_name__      ,   OPTS.out_image_dir      ,   '''hasAns'''	)
												histogram_na_prob(__magic_name__      ,   __magic_name__      ,   OPTS.out_image_dir      ,   '''noAns'''	)
						if OPTS.out_file:
												with open(OPTS.out_file      ,   '''w'''	) as f:
																		json.dump(__magic_name__      ,   __magic_name__	)
						else:
												print(json.dumps(__magic_name__      ,   indent=2	)	)


if __name__ == "__main__":
							UpperCamelCase_							=					parse_args()
							if OPTS.out_image_dir:
														import matplotlib

														matplotlib.use("""Agg""")
														import matplotlib.pyplot as plt
							main()








'''simple docstring'''





def 							_lowerCAmelCase (      __magic_name__	:    int      ,   __magic_name__	:    Optional[Any]	)							->     str:
						lowercase   :       Optional[Any]								=[0 for i in range(r + 1	)]
						# nc0 = 1
						lowercase   :       Optional[Any]								=1
						for i in range(1      ,   n + 1	):
												# to compute current row from previous row.
												lowercase   :       str								=min(__magic_name__      ,   __magic_name__	)
												while j > 0:
																		c[j] += c[j - 1]
																		j -= 1
						return c[r]







print(binomial_coefficient(n=10, r=5))





