


import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_snake_case	:						List[str]         =							re.compile(R"\b(a|an|the)\b", re.UNICODE)

_snake_case	:						Union[str, Any]         =							None







def 							lowerCAmelCase_	(   ):
							__snake_case						:     int									=  argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."      )
							parser.add_argument("data_file"				,      metavar="data.json"				,      help="Input data JSON file."      )
							parser.add_argument("pred_file"				,      metavar="pred.json"				,      help="Model predictions."      )
							parser.add_argument(
							    "--out-file"				,      "-o"				,      metavar="eval.json"				,      help="Write accuracy metrics to file (default is stdout)."      )
							parser.add_argument(
							    "--na-prob-file"				,      "-n"				,      metavar="na_prob.json"				,      help="Model estimates of probability of no answer."      )
							parser.add_argument(
							    "--na-prob-thresh"				,      "-t"				,      type=lowerCamelCase_				,      default=1.0				,      help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."				,      )
							parser.add_argument(
							    "--out-image-dir"				,      "-p"				,      metavar="out_images"				,      default=lowerCamelCase_				,      help="Save precision-recall curves to directory."      )
							parser.add_argument("--verbose"				,      "-v"				,      action="store_true"      )
							if len(sys.argv      ) == 1:
														parser.print_help()
														sys.exit(1      )
							return parser.parse_args()







def 							lowerCAmelCase_	(   __lowerCamelCase      ):
							__snake_case						:     Optional[int]									=  {}
							for article in dataset:
														for p in article["paragraphs"]:
																					for qa in p["qas"]:
																												__snake_case						:     Tuple									=  bool(qa["answers"]["text"]      )
							return qid_to_has_ans







def 							lowerCAmelCase_	(   __lowerCamelCase      ):

							def remove_articles(__lowerCamelCase      ):
														return ARTICLES_REGEX.sub(" "				,      lowerCamelCase_      )

							def white_space_fix(__lowerCamelCase      ):
														return " ".join(text.split()      )

							def remove_punc(__lowerCamelCase      ):
														__snake_case						:     Union[str, Any]									=  set(string.punctuation      )
														return "".join(ch for ch in text if ch not in exclude      )

							def lower(__lowerCamelCase      ):
														return text.lower()

							return white_space_fix(remove_articles(remove_punc(lower(lowerCamelCase_      )      )      )      )







def 							lowerCAmelCase_	(   __lowerCamelCase      ):
							if not s:
														return []
							return normalize_answer(lowerCamelCase_      ).split()







def 							lowerCAmelCase_	(   __lowerCamelCase				,      __lowerCamelCase      ):
							return int(normalize_answer(lowerCamelCase_      ) == normalize_answer(lowerCamelCase_      )      )







def 							lowerCAmelCase_	(   __lowerCamelCase				,      __lowerCamelCase      ):
							__snake_case						:     Any									=  get_tokens(lowerCamelCase_      )
							__snake_case						:     List[Any]									=  get_tokens(lowerCamelCase_      )
							__snake_case						:     int									=  collections.Counter(lowerCamelCase_      ) & collections.Counter(lowerCamelCase_      )
							__snake_case						:     Dict									=  sum(common.values()      )
							if len(lowerCamelCase_      ) == 0 or len(lowerCamelCase_      ) == 0:
														# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
														return int(gold_toks == pred_toks      )
							if num_same == 0:
														return 0
							__snake_case						:     Any									=  1.0 * num_same / len(lowerCamelCase_      )
							__snake_case						:     List[Any]									=  1.0 * num_same / len(lowerCamelCase_      )
							__snake_case						:     Any									=  (2 * precision * recall) / (precision + recall)
							return fa







def 							lowerCAmelCase_	(   __lowerCamelCase				,      __lowerCamelCase      ):
							__snake_case						:     Union[str, Any]									=  {}
							__snake_case						:     int									=  {}
							for article in dataset:
														for p in article["paragraphs"]:
																					for qa in p["qas"]:
																												__snake_case						:     int									=  qa['''id''']
																												__snake_case						:     Union[str, Any]									=  [t for t in qa['''answers''']['''text'''] if normalize_answer(lowerCamelCase_      )]
																												if not gold_answers:
																																			# For unanswerable questions, only correct answer is empty string
																																			__snake_case						:     Tuple									=  ['''''']
																												if qid not in preds:
																																			print(F'Missing prediction for {qid}'      )
																																			continue
																												__snake_case						:     Dict									=  preds[qid]
																												# Take max over all gold answers
																												__snake_case						:     List[str]									=  max(compute_exact(lowerCamelCase_				,      lowerCamelCase_      ) for a in gold_answers      )
																												__snake_case						:     Any									=  max(compute_fa(lowerCamelCase_				,      lowerCamelCase_      ) for a in gold_answers      )
							return exact_scores, fa_scores







def 							lowerCAmelCase_	(   __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase      ):
							__snake_case						:     Optional[Any]									=  {}
							for qid, s in scores.items():
														__snake_case						:     Any									=  na_probs[qid] > na_prob_thresh
														if pred_na:
																					__snake_case						:     Any									=  float(not qid_to_has_ans[qid]      )
														else:
																					__snake_case						:     Tuple									=  s
							return new_scores







def 							lowerCAmelCase_	(   __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase=None      ):
							if not qid_list:
														__snake_case						:     Union[str, Any]									=  len(lowerCamelCase_      )
														return collections.OrderedDict(
														    [
														        ("exact", 1_0_0.0 * sum(exact_scores.values()      ) / total),
														        ("f1", 1_0_0.0 * sum(fa_scores.values()      ) / total),
														        ("total", total),
														    ]      )
							else:
														__snake_case						:     List[Any]									=  len(lowerCamelCase_      )
														return collections.OrderedDict(
														    [
														        ("exact", 1_0_0.0 * sum(exact_scores[k] for k in qid_list      ) / total),
														        ("f1", 1_0_0.0 * sum(fa_scores[k] for k in qid_list      ) / total),
														        ("total", total),
														    ]      )







def 							lowerCAmelCase_	(   __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase      ):
							for k in new_eval:
														__snake_case						:     List[str]									=  new_eval[k]







def 							lowerCAmelCase_	(   __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase      ):
							plt.step(lowerCamelCase_				,      lowerCamelCase_				,      color="b"				,      alpha=0.2				,      where="post"      )
							plt.fill_between(lowerCamelCase_				,      lowerCamelCase_				,      step="post"				,      alpha=0.2				,      color="b"      )
							plt.xlabel("Recall"      )
							plt.ylabel("Precision"      )
							plt.xlim([0.0, 1.0_5]      )
							plt.ylim([0.0, 1.0_5]      )
							plt.title(lowerCamelCase_      )
							plt.savefig(lowerCamelCase_      )
							plt.clf()







def 							lowerCAmelCase_	(   __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase=None				,      __lowerCamelCase=None      ):
							__snake_case						:     Optional[int]									=  sorted(lowerCamelCase_				,      key=lambda __lowerCamelCase      : na_probs[k]      )
							__snake_case						:     List[str]									=  0.0
							__snake_case						:     Union[str, Any]									=  1.0
							__snake_case						:     List[Any]									=  0.0
							__snake_case						:     List[Any]									=  [1.0]
							__snake_case						:     List[Any]									=  [0.0]
							__snake_case						:     Optional[Any]									=  0.0
							for i, qid in enumerate(lowerCamelCase_      ):
														if qid_to_has_ans[qid]:
																					true_pos += scores[qid]
														__snake_case						:     Dict									=  true_pos / float(i + 1      )
														__snake_case						:     List[Any]									=  true_pos / float(lowerCamelCase_      )
														if i == len(lowerCamelCase_      ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
																					# i.e., if we can put a threshold after this point
																					avg_prec += cur_p * (cur_r - recalls[-1])
																					precisions.append(lowerCamelCase_      )
																					recalls.append(lowerCamelCase_      )
							if out_image:
														plot_pr_curve(lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_      )
							return {"ap": 1_0_0.0 * avg_prec}







def 							lowerCAmelCase_	(   __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase      ):
							if out_image_dir and not os.path.exists(lowerCamelCase_      ):
														os.makedirs(lowerCamelCase_      )
							__snake_case						:     Union[str, Any]									=  sum(1 for v in qid_to_has_ans.values() if v      )
							if num_true_pos == 0:
														return
							__snake_case						:     Any									=  make_precision_recall_eval(
							    lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      out_image=os.path.join(lowerCamelCase_				,      "pr_exact.png"      )				,      title="Precision-Recall curve for Exact Match score"				,      )
							__snake_case						:     Tuple									=  make_precision_recall_eval(
							    lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      out_image=os.path.join(lowerCamelCase_				,      "pr_f1.png"      )				,      title="Precision-Recall curve for F1 score"				,      )
							__snake_case						:     Dict									=  {k: float(lowerCamelCase_      ) for k, v in qid_to_has_ans.items()}
							__snake_case						:     Union[str, Any]									=  make_precision_recall_eval(
							    lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      out_image=os.path.join(lowerCamelCase_				,      "pr_oracle.png"      )				,      title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"				,      )
							merge_eval(lowerCamelCase_				,      lowerCamelCase_				,      "pr_exact"      )
							merge_eval(lowerCamelCase_				,      lowerCamelCase_				,      "pr_f1"      )
							merge_eval(lowerCamelCase_				,      lowerCamelCase_				,      "pr_oracle"      )







def 							lowerCAmelCase_	(   __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase      ):
							if not qid_list:
														return
							__snake_case						:     Dict									=  [na_probs[k] for k in qid_list]
							__snake_case						:     Tuple									=  np.ones_like(lowerCamelCase_      ) / float(len(lowerCamelCase_      )      )
							plt.hist(lowerCamelCase_				,      weights=lowerCamelCase_				,      bins=2_0				,      range=(0.0, 1.0)      )
							plt.xlabel("Model probability of no-answer"      )
							plt.ylabel("Proportion of dataset"      )
							plt.title(F'Histogram of no-answer probability: {name}'      )
							plt.savefig(os.path.join(lowerCamelCase_				,      F'na_prob_hist_{name}.png'      )      )
							plt.clf()







def 							lowerCAmelCase_	(   __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase      ):
							__snake_case						:     Dict									=  sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]      )
							__snake_case						:     str									=  num_no_ans
							__snake_case						:     int									=  cur_score
							__snake_case						:     Optional[int]									=  0.0
							__snake_case						:     Optional[Any]									=  sorted(lowerCamelCase_				,      key=lambda __lowerCamelCase      : na_probs[k]      )
							for i, qid in enumerate(lowerCamelCase_      ):
														if qid not in scores:
																					continue
														if qid_to_has_ans[qid]:
																					__snake_case						:     Tuple									=  scores[qid]
														else:
																					if preds[qid]:
																												__snake_case						:     List[Any]									=  -1
																					else:
																												__snake_case						:     Tuple									=  0
														cur_score += diff
														if cur_score > best_score:
																					__snake_case						:     List[Any]									=  cur_score
																					__snake_case						:     str									=  na_probs[qid]
							return 1_0_0.0 * best_score / len(lowerCamelCase_      ), best_thresh







def 							lowerCAmelCase_	(   __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase				,      __lowerCamelCase      ):
							__snake_case						:     Tuple									=  find_best_thresh(lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_      )
							__snake_case						:     int									=  find_best_thresh(lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_      )
							__snake_case						:     str									=  best_exact
							__snake_case						:     Optional[int]									=  exact_thresh
							__snake_case						:     Tuple									=  best_fa
							__snake_case						:     Optional[Any]									=  fa_thresh







def 							lowerCAmelCase_	(   ):
							with open(OPTS.data_file      ) as f:
														__snake_case						:     List[Any]									=  json.load(lowerCamelCase_      )
														__snake_case						:     Union[str, Any]									=  dataset_json['''data''']
							with open(OPTS.pred_file      ) as f:
														__snake_case						:     Dict									=  json.load(lowerCamelCase_      )
							if OPTS.na_prob_file:
														with open(OPTS.na_prob_file      ) as f:
																					__snake_case						:     int									=  json.load(lowerCamelCase_      )
							else:
														__snake_case						:     List[Any]									=  {k: 0.0 for k in preds}
							__snake_case						:     Dict									=  make_qid_to_has_ans(lowerCamelCase_      )  # maps qid to True/False
							__snake_case						:     Optional[int]									=  [k for k, v in qid_to_has_ans.items() if v]
							__snake_case						:     Tuple									=  [k for k, v in qid_to_has_ans.items() if not v]
							__snake_case						:     str									=  get_raw_scores(lowerCamelCase_				,      lowerCamelCase_      )
							__snake_case						:     Union[str, Any]									=  apply_no_ans_threshold(lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      OPTS.na_prob_thresh      )
							__snake_case						:     Optional[int]									=  apply_no_ans_threshold(lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      OPTS.na_prob_thresh      )
							__snake_case						:     int									=  make_eval_dict(lowerCamelCase_				,      lowerCamelCase_      )
							if has_ans_qids:
														__snake_case						:     str									=  make_eval_dict(lowerCamelCase_				,      lowerCamelCase_				,      qid_list=lowerCamelCase_      )
														merge_eval(lowerCamelCase_				,      lowerCamelCase_				,      "HasAns"      )
							if no_ans_qids:
														__snake_case						:     Union[str, Any]									=  make_eval_dict(lowerCamelCase_				,      lowerCamelCase_				,      qid_list=lowerCamelCase_      )
														merge_eval(lowerCamelCase_				,      lowerCamelCase_				,      "NoAns"      )
							if OPTS.na_prob_file:
														find_all_best_thresh(lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_      )
							if OPTS.na_prob_file and OPTS.out_image_dir:
														run_precision_recall_analysis(lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      lowerCamelCase_				,      OPTS.out_image_dir      )
														histogram_na_prob(lowerCamelCase_				,      lowerCamelCase_				,      OPTS.out_image_dir				,      "hasAns"      )
														histogram_na_prob(lowerCamelCase_				,      lowerCamelCase_				,      OPTS.out_image_dir				,      "noAns"      )
							if OPTS.out_file:
														with open(OPTS.out_file				,      "w"      ) as f:
																					json.dump(lowerCamelCase_				,      lowerCamelCase_      )
							else:
														print(json.dumps(lowerCamelCase_				,      indent=2      )      )


if __name__ == "__main__":
			_snake_case	:						Union[str, Any]         =							parse_args()
			if OPTS.out_image_dir:
						import matplotlib

						matplotlib.use("Agg")
						import matplotlib.pyplot as plt
			main()




lowercase			:				Dict   =     [sum(int(c, 10) ** 2 for c in i.__str__()) for i in range(10_00_00)]






def        snake_case__		(    lowerCamelCase_		):
							A      :					List[str]      		=					0
							while number:
														# Increased Speed Slightly by checking every 5 digits together.
														sum_of_digits_squared += DIGITS_SQUARED[number % 100000]
														number //= 100000

							return sum_of_digits_squared


# There are 2 Chains made,
# One ends with 89 with the chain member 58 being the one which when declared first,
# there will be the least number of iterations for all the members to be checked.

# The other one ends with 1 and has only one element 1.

# So 58 and 1 are chosen to be declared at the starting.

# Changed dictionary to an array to quicken the solution
lowercase			:				list[bool | None]   =     [None] * 10_00_00_00
lowercase			:				int   =     True
lowercase			:				Tuple   =     False






def        snake_case__		(    lowerCamelCase_		):

							if CHAINS[number - 1] is not None:
														return CHAINS[number - 1]  # type: ignore

							A      :					int      		=					chain(next_number(lowerCamelCase_		)		)
							A      :					Dict      		=					number_chain

							while number < 10000000:
														A      :					Any      		=					number_chain
														number *= 10

							return number_chain






def        snake_case__		(    lowerCamelCase_ = 10000000		):
							for i in range(1     ,				lowerCamelCase_		):
														if CHAINS[i] is None:
																					chain(i + 1		)

							return CHAINS[:number].count(lowerCamelCase_		)


if __name__ == "__main__":
		import doctest

		doctest.testmod()
		print(F"{solution() = }")






