
'''simple docstring'''
import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


__lowerCAmelCase												=	re.compile(r'\b(a|an|the)\b', re.UNICODE)

__lowerCAmelCase												=	None






def  __SCREAMING_SNAKE_CASE				(  ):
							_snake_case       =  argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0."""				)
							parser.add_argument("""data_file"""							, metavar="""data.json"""							, help="""Input data JSON file."""				)
							parser.add_argument("""pred_file"""							, metavar="""pred.json"""							, help="""Model predictions."""				)
							parser.add_argument(
							    """--out-file"""							, """-o"""							, metavar="""eval.json"""							, help="""Write accuracy metrics to file (default is stdout)."""				)
							parser.add_argument(
							    """--na-prob-file"""							, """-n"""							, metavar="""na_prob.json"""							, help="""Model estimates of probability of no answer."""				)
							parser.add_argument(
							    """--na-prob-thresh"""							, """-t"""							, type=_SCREAMING_SNAKE_CASE							, default=1.0							, help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""							, )
							parser.add_argument(
							    """--out-image-dir"""							, """-p"""							, metavar="""out_images"""							, default=_SCREAMING_SNAKE_CASE							, help="""Save precision-recall curves to directory."""				)
							parser.add_argument("""--verbose"""							, """-v"""							, action="""store_true"""				)
							if len(sys.argv				) == 1:
														parser.print_help()
														sys.exit(1				)
							return parser.parse_args()






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE				):
							_snake_case       =  {}
							for article in dataset:
														for p in article["paragraphs"]:
																					for qa in p["qas"]:
																												_snake_case       =  bool(qa["""answers"""]["""text"""]				)
							return qid_to_has_ans






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE				):

							def remove_articles(_SCREAMING_SNAKE_CASE				):
														return ARTICLES_REGEX.sub(""" """							, _SCREAMING_SNAKE_CASE				)

							def white_space_fix(_SCREAMING_SNAKE_CASE				):
														return " ".join(text.split()				)

							def remove_punc(_SCREAMING_SNAKE_CASE				):
														_snake_case       =  set(string.punctuation				)
														return "".join(ch for ch in text if ch not in exclude				)

							def lower(_SCREAMING_SNAKE_CASE				):
														return text.lower()

							return white_space_fix(remove_articles(remove_punc(lower(_SCREAMING_SNAKE_CASE				)				)				)				)






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE				):
							if not s:
														return []
							return normalize_answer(_SCREAMING_SNAKE_CASE				).split()






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):
							return int(normalize_answer(_SCREAMING_SNAKE_CASE				) == normalize_answer(_SCREAMING_SNAKE_CASE				)				)






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):
							_snake_case       =  get_tokens(_SCREAMING_SNAKE_CASE				)
							_snake_case       =  get_tokens(_SCREAMING_SNAKE_CASE				)
							_snake_case       =  collections.Counter(_SCREAMING_SNAKE_CASE				) & collections.Counter(_SCREAMING_SNAKE_CASE				)
							_snake_case       =  sum(common.values()				)
							if len(_SCREAMING_SNAKE_CASE				) == 0 or len(_SCREAMING_SNAKE_CASE				) == 0:
														# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
														return int(gold_toks == pred_toks				)
							if num_same == 0:
														return 0
							_snake_case       =  1.0 * num_same / len(_SCREAMING_SNAKE_CASE				)
							_snake_case       =  1.0 * num_same / len(_SCREAMING_SNAKE_CASE				)
							_snake_case       =  (2 * precision * recall) / (precision + recall)
							return fa






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):
							_snake_case       =  {}
							_snake_case       =  {}
							for article in dataset:
														for p in article["paragraphs"]:
																					for qa in p["qas"]:
																												_snake_case       =  qa["""id"""]
																												_snake_case       =  [t for t in qa["""answers"""]["""text"""] if normalize_answer(_SCREAMING_SNAKE_CASE				)]
																												if not gold_answers:
																																			# For unanswerable questions, only correct answer is empty string
																																			_snake_case       =  [""""""]
																												if qid not in preds:
																																			print(f"""Missing prediction for {qid}"""				)
																																			continue
																												_snake_case       =  preds[qid]
																												# Take max over all gold answers
																												_snake_case       =  max(compute_exact(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				) for a in gold_answers				)
																												_snake_case       =  max(compute_fa(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				) for a in gold_answers				)
							return exact_scores, fa_scores






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):
							_snake_case       =  {}
							for qid, s in scores.items():
														_snake_case       =  na_probs[qid] > na_prob_thresh
														if pred_na:
																					_snake_case       =  float(not qid_to_has_ans[qid]				)
														else:
																					_snake_case       =  s
							return new_scores






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE=None				):
							if not qid_list:
														_snake_case       =  len(_SCREAMING_SNAKE_CASE				)
														return collections.OrderedDict(
														    [
														        ("""exact""", 100.0 * sum(exact_scores.values()				) / total),
														        ("""f1""", 100.0 * sum(fa_scores.values()				) / total),
														        ("""total""", total),
														    ]				)
							else:
														_snake_case       =  len(_SCREAMING_SNAKE_CASE				)
														return collections.OrderedDict(
														    [
														        ("""exact""", 100.0 * sum(exact_scores[k] for k in qid_list				) / total),
														        ("""f1""", 100.0 * sum(fa_scores[k] for k in qid_list				) / total),
														        ("""total""", total),
														    ]				)






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):
							for k in new_eval:
														_snake_case       =  new_eval[k]






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):
							plt.step(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, color="""b"""							, alpha=0.2							, where="""post"""				)
							plt.fill_between(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, step="""post"""							, alpha=0.2							, color="""b"""				)
							plt.xlabel("""Recall"""				)
							plt.ylabel("""Precision"""				)
							plt.xlim([0.0, 1.05]				)
							plt.ylim([0.0, 1.05]				)
							plt.title(_SCREAMING_SNAKE_CASE				)
							plt.savefig(_SCREAMING_SNAKE_CASE				)
							plt.clf()






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE=None							, _SCREAMING_SNAKE_CASE=None				):
							_snake_case       =  sorted(_SCREAMING_SNAKE_CASE							, key=lambda _SCREAMING_SNAKE_CASE				: na_probs[k]				)
							_snake_case       =  0.0
							_snake_case       =  1.0
							_snake_case       =  0.0
							_snake_case       =  [1.0]
							_snake_case       =  [0.0]
							_snake_case       =  0.0
							for i, qid in enumerate(_SCREAMING_SNAKE_CASE				):
														if qid_to_has_ans[qid]:
																					true_pos += scores[qid]
														_snake_case       =  true_pos / float(i + 1				)
														_snake_case       =  true_pos / float(_SCREAMING_SNAKE_CASE				)
														if i == len(_SCREAMING_SNAKE_CASE				) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
																					# i.e., if we can put a threshold after this point
																					avg_prec += cur_p * (cur_r - recalls[-1])
																					precisions.append(_SCREAMING_SNAKE_CASE				)
																					recalls.append(_SCREAMING_SNAKE_CASE				)
							if out_image:
														plot_pr_curve(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				)
							return {"ap": 100.0 * avg_prec}






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):
							if out_image_dir and not os.path.exists(_SCREAMING_SNAKE_CASE				):
														os.makedirs(_SCREAMING_SNAKE_CASE				)
							_snake_case       =  sum(1 for v in qid_to_has_ans.values() if v				)
							if num_true_pos == 0:
														return
							_snake_case       =  make_precision_recall_eval(
							    _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, out_image=os.path.join(_SCREAMING_SNAKE_CASE							, """pr_exact.png"""				)							, title="""Precision-Recall curve for Exact Match score"""							, )
							_snake_case       =  make_precision_recall_eval(
							    _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, out_image=os.path.join(_SCREAMING_SNAKE_CASE							, """pr_f1.png"""				)							, title="""Precision-Recall curve for F1 score"""							, )
							_snake_case       =  {k: float(_SCREAMING_SNAKE_CASE				) for k, v in qid_to_has_ans.items()}
							_snake_case       =  make_precision_recall_eval(
							    _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, out_image=os.path.join(_SCREAMING_SNAKE_CASE							, """pr_oracle.png"""				)							, title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""							, )
							merge_eval(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, """pr_exact"""				)
							merge_eval(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, """pr_f1"""				)
							merge_eval(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, """pr_oracle"""				)






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):
							if not qid_list:
														return
							_snake_case       =  [na_probs[k] for k in qid_list]
							_snake_case       =  np.ones_like(_SCREAMING_SNAKE_CASE				) / float(len(_SCREAMING_SNAKE_CASE				)				)
							plt.hist(_SCREAMING_SNAKE_CASE							, weights=_SCREAMING_SNAKE_CASE							, bins=20							, range=(0.0, 1.0)				)
							plt.xlabel("""Model probability of no-answer"""				)
							plt.ylabel("""Proportion of dataset"""				)
							plt.title(f"""Histogram of no-answer probability: {name}"""				)
							plt.savefig(os.path.join(_SCREAMING_SNAKE_CASE							, f"""na_prob_hist_{name}.png"""				)				)
							plt.clf()






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):
							_snake_case       =  sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]				)
							_snake_case       =  num_no_ans
							_snake_case       =  cur_score
							_snake_case       =  0.0
							_snake_case       =  sorted(_SCREAMING_SNAKE_CASE							, key=lambda _SCREAMING_SNAKE_CASE				: na_probs[k]				)
							for i, qid in enumerate(_SCREAMING_SNAKE_CASE				):
														if qid not in scores:
																					continue
														if qid_to_has_ans[qid]:
																					_snake_case       =  scores[qid]
														else:
																					if preds[qid]:
																												_snake_case       =  -1
																					else:
																												_snake_case       =  0
														cur_score += diff
														if cur_score > best_score:
																					_snake_case       =  cur_score
																					_snake_case       =  na_probs[qid]
							return 100.0 * best_score / len(_SCREAMING_SNAKE_CASE				), best_thresh






def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):
							_snake_case,							_snake_case       =  find_best_thresh(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				)
							_snake_case,							_snake_case       =  find_best_thresh(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				)
							_snake_case       =  best_exact
							_snake_case       =  exact_thresh
							_snake_case       =  best_fa
							_snake_case       =  fa_thresh






def  __SCREAMING_SNAKE_CASE				(  ):
							with open(OPTS.data_file				) as f:
														_snake_case       =  json.load(_SCREAMING_SNAKE_CASE				)
														_snake_case       =  dataset_json["""data"""]
							with open(OPTS.pred_file				) as f:
														_snake_case       =  json.load(_SCREAMING_SNAKE_CASE				)
							if OPTS.na_prob_file:
														with open(OPTS.na_prob_file				) as f:
																					_snake_case       =  json.load(_SCREAMING_SNAKE_CASE				)
							else:
														_snake_case       =  {k: 0.0 for k in preds}
							_snake_case       =  make_qid_to_has_ans(_SCREAMING_SNAKE_CASE				)  # maps qid to True/False
							_snake_case       =  [k for k, v in qid_to_has_ans.items() if v]
							_snake_case       =  [k for k, v in qid_to_has_ans.items() if not v]
							_snake_case,							_snake_case       =  get_raw_scores(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				)
							_snake_case       =  apply_no_ans_threshold(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, OPTS.na_prob_thresh				)
							_snake_case       =  apply_no_ans_threshold(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, OPTS.na_prob_thresh				)
							_snake_case       =  make_eval_dict(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				)
							if has_ans_qids:
														_snake_case       =  make_eval_dict(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, qid_list=_SCREAMING_SNAKE_CASE				)
														merge_eval(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, """HasAns"""				)
							if no_ans_qids:
														_snake_case       =  make_eval_dict(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, qid_list=_SCREAMING_SNAKE_CASE				)
														merge_eval(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, """NoAns"""				)
							if OPTS.na_prob_file:
														find_all_best_thresh(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				)
							if OPTS.na_prob_file and OPTS.out_image_dir:
														run_precision_recall_analysis(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, OPTS.out_image_dir				)
														histogram_na_prob(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, OPTS.out_image_dir							, """hasAns"""				)
														histogram_na_prob(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE							, OPTS.out_image_dir							, """noAns"""				)
							if OPTS.out_file:
														with open(OPTS.out_file							, """w"""				) as f:
																					json.dump(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				)
							else:
														print(json.dumps(_SCREAMING_SNAKE_CASE							, indent=2				)				)


if __name__ == "__main__":
				__lowerCAmelCase												=	parse_args()
				if OPTS.out_image_dir:
								import matplotlib

								matplotlib.use('Agg')
								import matplotlib.pyplot as plt
				main()

'''simple docstring'''
def  __SCREAMING_SNAKE_CASE				(  _SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				):

							if not (isinstance(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				) and isinstance(_SCREAMING_SNAKE_CASE							, _SCREAMING_SNAKE_CASE				)):
														raise ValueError("""longest_common_substring() takes two strings for inputs"""				)

							_snake_case       =  len(_SCREAMING_SNAKE_CASE				)
							_snake_case       =  len(_SCREAMING_SNAKE_CASE				)

							_snake_case       =  [[0] * (texta_length + 1) for _ in range(texta_length + 1				)]
							_snake_case       =  0
							_snake_case       =  0

							for i in range(1							, texta_length + 1				):
														for j in range(1							, texta_length + 1				):
																					if texta[i - 1] == texta[j - 1]:
																												_snake_case       =  1 + dp[i - 1][j - 1]
																												if dp[i][j] > ans_length:
																																			_snake_case       =  i
																																			_snake_case       =  dp[i][j]

							return texta[ans_index - ans_length : ans_index]


if __name__ == "__main__":
				import doctest

				doctest.testmod()