





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


a__: str							   =						re.compile(r'\b(a|an|the)\b', re.UNICODE)

a__: Optional[int]							   =						None


def       UpperCamelCase__(			)->Any:
       A__		   =		argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''    )
       parser.add_argument('''data_file'''					,	metavar='''data.json'''					,	help='''Input data JSON file.'''    )
       parser.add_argument('''pred_file'''					,	metavar='''pred.json'''					,	help='''Model predictions.'''    )
       parser.add_argument(
           '''--out-file'''					,	'''-o'''					,	metavar='''eval.json'''					,	help='''Write accuracy metrics to file (default is stdout).'''    )
       parser.add_argument(
           '''--na-prob-file'''					,	'''-n'''					,	metavar='''na_prob.json'''					,	help='''Model estimates of probability of no answer.'''    )
       parser.add_argument(
           '''--na-prob-thresh'''					,	'''-t'''					,	type=UpperCamelCase__					,	default=1.0					,	help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''					,	)
       parser.add_argument(
           '''--out-image-dir'''					,	'''-p'''					,	metavar='''out_images'''					,	default=UpperCamelCase__					,	help='''Save precision-recall curves to directory.'''    )
       parser.add_argument('''--verbose'''					,	'''-v'''					,	action='''store_true'''    )
       if len(sys.argv    ) == 1:
              parser.print_help()
              sys.exit(1    )
       return parser.parse_args()


def       UpperCamelCase__(			UpperCamelCase__     :			Optional[Any]    )->str:
       A__		   =		{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            A__		   =		bool(qa['''answers''']['''text''']    )
       return qid_to_has_ans


def       UpperCamelCase__(			UpperCamelCase__     :			Any    )->Dict:

       def remove_articles(UpperCamelCase__     :			List[Any]    ):
              return ARTICLES_REGEX.sub(''' '''					,	UpperCamelCase__    )

       def white_space_fix(UpperCamelCase__     :			Union[str, Any]    ):
              return " ".join(text.split()    )

       def remove_punc(UpperCamelCase__     :			Any    ):
              A__		   =		set(string.punctuation    )
              return "".join(ch for ch in text if ch not in exclude    )

       def lower(UpperCamelCase__     :			str    ):
              return text.lower()

       return white_space_fix(remove_articles(remove_punc(lower(UpperCamelCase__    )    )    )    )


def       UpperCamelCase__(			UpperCamelCase__     :			int    )->int:
       if not s:
              return []
       return normalize_answer(UpperCamelCase__    ).split()


def       UpperCamelCase__(			UpperCamelCase__     :			Optional[int]					,	UpperCamelCase__     :			Any    )->Dict:
       return int(normalize_answer(UpperCamelCase__    ) == normalize_answer(UpperCamelCase__    )    )


def       UpperCamelCase__(			UpperCamelCase__     :			Tuple					,	UpperCamelCase__     :			Optional[int]    )->Tuple:
       A__		   =		get_tokens(UpperCamelCase__    )
       A__		   =		get_tokens(UpperCamelCase__    )
       A__		   =		collections.Counter(UpperCamelCase__    ) & collections.Counter(UpperCamelCase__    )
       A__		   =		sum(common.values()    )
       if len(UpperCamelCase__    ) == 0 or len(UpperCamelCase__    ) == 0:
              # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
              return int(gold_toks == pred_toks    )
       if num_same == 0:
              return 0
       A__		   =		1.0 * num_same / len(UpperCamelCase__    )
       A__		   =		1.0 * num_same / len(UpperCamelCase__    )
       A__		   =		(2 * precision * recall) / (precision + recall)
       return fa


def       UpperCamelCase__(			UpperCamelCase__     :			Any					,	UpperCamelCase__     :			List[str]    )->List[Any]:
       A__		   =		{}
       A__		   =		{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            A__		   =		qa['''id''']
                            A__		   =		[t for t in qa['''answers''']['''text'''] if normalize_answer(UpperCamelCase__    )]
                            if not gold_answers:
                                   # For unanswerable questions, only correct answer is empty string
                                   A__		   =		['''''']
                            if qid not in preds:
                                   print(f"Missing prediction for {qid}"    )
                                   continue
                            A__		   =		preds[qid]
                            # Take max over all gold answers
                            A__		   =		max(compute_exact(UpperCamelCase__					,	UpperCamelCase__    ) for a in gold_answers    )
                            A__		   =		max(compute_fa(UpperCamelCase__					,	UpperCamelCase__    ) for a in gold_answers    )
       return exact_scores, fa_scores


def       UpperCamelCase__(			UpperCamelCase__     :			List[str]					,	UpperCamelCase__     :			Optional[Any]					,	UpperCamelCase__     :			Tuple					,	UpperCamelCase__     :			List[Any]    )->Any:
       A__		   =		{}
       for qid, s in scores.items():
              A__		   =		na_probs[qid] > na_prob_thresh
              if pred_na:
                     A__		   =		float(not qid_to_has_ans[qid]    )
              else:
                     A__		   =		s
       return new_scores


def       UpperCamelCase__(			UpperCamelCase__     :			Optional[int]					,	UpperCamelCase__     :			Tuple					,	UpperCamelCase__     :			Tuple=None    )->Union[str, Any]:
       if not qid_list:
              A__		   =		len(UpperCamelCase__    )
              return collections.OrderedDict(
                  [
                      ('''exact''', 100.0 * sum(exact_scores.values()    ) / total),
                      ('''f1''', 100.0 * sum(fa_scores.values()    ) / total),
                      ('''total''', total),
                  ]    )
       else:
              A__		   =		len(UpperCamelCase__    )
              return collections.OrderedDict(
                  [
                      ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list    ) / total),
                      ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list    ) / total),
                      ('''total''', total),
                  ]    )


def       UpperCamelCase__(			UpperCamelCase__     :			Union[str, Any]					,	UpperCamelCase__     :			Optional[int]					,	UpperCamelCase__     :			Optional[int]    )->Tuple:
       for k in new_eval:
              A__		   =		new_eval[k]


def       UpperCamelCase__(			UpperCamelCase__     :			str					,	UpperCamelCase__     :			Any					,	UpperCamelCase__     :			int					,	UpperCamelCase__     :			Tuple    )->Union[str, Any]:
       plt.step(UpperCamelCase__					,	UpperCamelCase__					,	color='''b'''					,	alpha=0.2					,	where='''post'''    )
       plt.fill_between(UpperCamelCase__					,	UpperCamelCase__					,	step='''post'''					,	alpha=0.2					,	color='''b'''    )
       plt.xlabel('''Recall'''    )
       plt.ylabel('''Precision'''    )
       plt.xlim([0.0, 1.05]    )
       plt.ylim([0.0, 1.05]    )
       plt.title(UpperCamelCase__    )
       plt.savefig(UpperCamelCase__    )
       plt.clf()


def       UpperCamelCase__(			UpperCamelCase__     :			List[str]					,	UpperCamelCase__     :			str					,	UpperCamelCase__     :			Tuple					,	UpperCamelCase__     :			Optional[Any]					,	UpperCamelCase__     :			Union[str, Any]=None					,	UpperCamelCase__     :			Optional[int]=None    )->Any:
       A__		   =		sorted(UpperCamelCase__					,	key=lambda UpperCamelCase__    : na_probs[k]    )
       A__		   =		0.0
       A__		   =		1.0
       A__		   =		0.0
       A__		   =		[1.0]
       A__		   =		[0.0]
       A__		   =		0.0
       for i, qid in enumerate(UpperCamelCase__    ):
              if qid_to_has_ans[qid]:
                     true_pos += scores[qid]
              A__		   =		true_pos / float(i + 1    )
              A__		   =		true_pos / float(UpperCamelCase__    )
              if i == len(UpperCamelCase__    ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                     # i.e., if we can put a threshold after this point
                     avg_prec += cur_p * (cur_r - recalls[-1])
                     precisions.append(UpperCamelCase__    )
                     recalls.append(UpperCamelCase__    )
       if out_image:
              plot_pr_curve(UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__    )
       return {"ap": 100.0 * avg_prec}


def       UpperCamelCase__(			UpperCamelCase__     :			Tuple					,	UpperCamelCase__     :			Union[str, Any]					,	UpperCamelCase__     :			Any					,	UpperCamelCase__     :			List[str]					,	UpperCamelCase__     :			List[str]					,	UpperCamelCase__     :			Any    )->Union[str, Any]:
       if out_image_dir and not os.path.exists(UpperCamelCase__    ):
              os.makedirs(UpperCamelCase__    )
       A__		   =		sum(1 for v in qid_to_has_ans.values() if v    )
       if num_true_pos == 0:
              return
       A__		   =		make_precision_recall_eval(
           UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	out_image=os.path.join(UpperCamelCase__					,	'''pr_exact.png'''    )					,	title='''Precision-Recall curve for Exact Match score'''					,	)
       A__		   =		make_precision_recall_eval(
           UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	out_image=os.path.join(UpperCamelCase__					,	'''pr_f1.png'''    )					,	title='''Precision-Recall curve for F1 score'''					,	)
       A__		   =		{k: float(UpperCamelCase__    ) for k, v in qid_to_has_ans.items()}
       A__		   =		make_precision_recall_eval(
           UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	out_image=os.path.join(UpperCamelCase__					,	'''pr_oracle.png'''    )					,	title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''					,	)
       merge_eval(UpperCamelCase__					,	UpperCamelCase__					,	'''pr_exact'''    )
       merge_eval(UpperCamelCase__					,	UpperCamelCase__					,	'''pr_f1'''    )
       merge_eval(UpperCamelCase__					,	UpperCamelCase__					,	'''pr_oracle'''    )


def       UpperCamelCase__(			UpperCamelCase__     :			List[Any]					,	UpperCamelCase__     :			List[str]					,	UpperCamelCase__     :			int					,	UpperCamelCase__     :			str    )->Dict:
       if not qid_list:
              return
       A__		   =		[na_probs[k] for k in qid_list]
       A__		   =		np.ones_like(UpperCamelCase__    ) / float(len(UpperCamelCase__    )    )
       plt.hist(UpperCamelCase__					,	weights=UpperCamelCase__					,	bins=20					,	range=(0.0, 1.0)    )
       plt.xlabel('''Model probability of no-answer'''    )
       plt.ylabel('''Proportion of dataset'''    )
       plt.title(f"Histogram of no-answer probability: {name}"    )
       plt.savefig(os.path.join(UpperCamelCase__					,	f"na_prob_hist_{name}.png"    )    )
       plt.clf()


def       UpperCamelCase__(			UpperCamelCase__     :			List[Any]					,	UpperCamelCase__     :			Union[str, Any]					,	UpperCamelCase__     :			Optional[Any]					,	UpperCamelCase__     :			Dict    )->List[str]:
       A__		   =		sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]    )
       A__		   =		num_no_ans
       A__		   =		cur_score
       A__		   =		0.0
       A__		   =		sorted(UpperCamelCase__					,	key=lambda UpperCamelCase__    : na_probs[k]    )
       for i, qid in enumerate(UpperCamelCase__    ):
              if qid not in scores:
                     continue
              if qid_to_has_ans[qid]:
                     A__		   =		scores[qid]
              else:
                     if preds[qid]:
                            A__		   =		-1
                     else:
                            A__		   =		0
              cur_score += diff
              if cur_score > best_score:
                     A__		   =		cur_score
                     A__		   =		na_probs[qid]
       return 100.0 * best_score / len(UpperCamelCase__    ), best_thresh


def       UpperCamelCase__(			UpperCamelCase__     :			Tuple					,	UpperCamelCase__     :			int					,	UpperCamelCase__     :			Optional[int]					,	UpperCamelCase__     :			str					,	UpperCamelCase__     :			List[str]					,	UpperCamelCase__     :			Any    )->List[Any]:
       A__   ,      A__		   =		find_best_thresh(UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__    )
       A__   ,      A__		   =		find_best_thresh(UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__    )
       A__		   =		best_exact
       A__		   =		exact_thresh
       A__		   =		best_fa
       A__		   =		fa_thresh


def       UpperCamelCase__(			)->Union[str, Any]:
       with open(OPTS.data_file    ) as f:
              A__		   =		json.load(UpperCamelCase__    )
              A__		   =		dataset_json['''data''']
       with open(OPTS.pred_file    ) as f:
              A__		   =		json.load(UpperCamelCase__    )
       if OPTS.na_prob_file:
              with open(OPTS.na_prob_file    ) as f:
                     A__		   =		json.load(UpperCamelCase__    )
       else:
              A__		   =		{k: 0.0 for k in preds}
       A__		   =		make_qid_to_has_ans(UpperCamelCase__    )  # maps qid to True/False
       A__		   =		[k for k, v in qid_to_has_ans.items() if v]
       A__		   =		[k for k, v in qid_to_has_ans.items() if not v]
       A__   ,      A__		   =		get_raw_scores(UpperCamelCase__					,	UpperCamelCase__    )
       A__		   =		apply_no_ans_threshold(UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	OPTS.na_prob_thresh    )
       A__		   =		apply_no_ans_threshold(UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	OPTS.na_prob_thresh    )
       A__		   =		make_eval_dict(UpperCamelCase__					,	UpperCamelCase__    )
       if has_ans_qids:
              A__		   =		make_eval_dict(UpperCamelCase__					,	UpperCamelCase__					,	qid_list=UpperCamelCase__    )
              merge_eval(UpperCamelCase__					,	UpperCamelCase__					,	'''HasAns'''    )
       if no_ans_qids:
              A__		   =		make_eval_dict(UpperCamelCase__					,	UpperCamelCase__					,	qid_list=UpperCamelCase__    )
              merge_eval(UpperCamelCase__					,	UpperCamelCase__					,	'''NoAns'''    )
       if OPTS.na_prob_file:
              find_all_best_thresh(UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__    )
       if OPTS.na_prob_file and OPTS.out_image_dir:
              run_precision_recall_analysis(UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	OPTS.out_image_dir    )
              histogram_na_prob(UpperCamelCase__					,	UpperCamelCase__					,	OPTS.out_image_dir					,	'''hasAns'''    )
              histogram_na_prob(UpperCamelCase__					,	UpperCamelCase__					,	OPTS.out_image_dir					,	'''noAns'''    )
       if OPTS.out_file:
              with open(OPTS.out_file					,	'''w'''    ) as f:
                     json.dump(UpperCamelCase__					,	UpperCamelCase__    )
       else:
              print(json.dumps(UpperCamelCase__					,	indent=2    )    )


if __name__ == "__main__":
       a__: Dict							   =						parse_args()
       if OPTS.out_image_dir:
              import matplotlib

              matplotlib.use('Agg')
              import matplotlib.pyplot as plt
       main()












class        SCREAMING_SNAKE_CASE__					:



 def __init__(				self,__lowerCamelCase					):
        A__		   =		size
        A__		   =		[0] * size
        A__		   =		[0] * size



 @staticmethod
 def 							UpperCamelCase     (				__lowerCamelCase					):
        return index | (index + 1)



 @staticmethod
 def 							UpperCamelCase     (				__lowerCamelCase					):
        return (index & (index + 1)) - 1



 def 							UpperCamelCase     (				self,__lowerCamelCase,__lowerCamelCase					):
        A__		   =		value
        while index < self.size:
               A__		   =		self.get_prev(__lowerCamelCase					) + 1
               if current_left_border == index:
                      A__		   =		value
               else:
                      A__		   =		max(__lowerCamelCase,__lowerCamelCase,__lowerCamelCase					)
               A__		   =		self.get_next(__lowerCamelCase					)




 def 							UpperCamelCase     (				self,__lowerCamelCase,__lowerCamelCase					):
        right -= 1  # Because of right is exclusive
        A__		   =		0
        while left <= right:
               A__		   =		self.get_prev(__lowerCamelCase					)
               if left <= current_left:
                      A__		   =		max(__lowerCamelCase,self.tree[right]					)
                      A__		   =		current_left
               else:
                      A__		   =		max(__lowerCamelCase,self.arr[right]					)
                      right -= 1
        return result


if __name__ == "__main__":
       import doctest

       doctest.testmod()





