

import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


A									=  re.compile(R'\b(a|an|the)\b', re.UNICODE)

A									=  None





def a():





	'''simple docstring'''
	snake_case_   		=			argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'						)
	parser.add_argument('data_file'      ,			metavar='data.json'      ,			help='Input data JSON file.'						)
	parser.add_argument('pred_file'      ,			metavar='pred.json'      ,			help='Model predictions.'						)
	parser.add_argument(
	    '--out-file'      ,			'-o'      ,			metavar='eval.json'      ,			help='Write accuracy metrics to file (default is stdout).'						)
	parser.add_argument(
	    '--na-prob-file'      ,			'-n'      ,			metavar='na_prob.json'      ,			help='Model estimates of probability of no answer.'						)
	parser.add_argument(
	    '--na-prob-thresh'      ,			'-t'      ,			type=lowercase__      ,			default=1.0      ,			help='Predict "" if no-answer probability exceeds this (default = 1.0).'      ,			)
	parser.add_argument(
	    '--out-image-dir'      ,			'-p'      ,			metavar='out_images'      ,			default=lowercase__      ,			help='Save precision-recall curves to directory.'						)
	parser.add_argument('--verbose'      ,			'-v'      ,			action='store_true'						)
	if len(sys.argv						) == 1:
		parser.print_help()
		sys.exit(1						)
	return parser.parse_args()





def a(lowercase__						):





	'''simple docstring'''
	snake_case_   		=			{}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				snake_case_   		=			bool(qa['answers']['text']						)
	return qid_to_has_ans





def a(lowercase__						):





	'''simple docstring'''

	def remove_articles(lowercase__						):
		return ARTICLES_REGEX.sub(' '      ,			lowercase__						)

	def white_space_fix(lowercase__						):
		return " ".join(text.split()						)

	def remove_punc(lowercase__						):
		snake_case_   		=			set(string.punctuation						)
		return "".join(ch for ch in text if ch not in exclude						)

	def lower(lowercase__						):
		return text.lower()

	return white_space_fix(remove_articles(remove_punc(lower(lowercase__						)						)						)						)





def a(lowercase__						):





	'''simple docstring'''
	if not s:
		return []
	return normalize_answer(lowercase__						).split()





def a(lowercase__      ,			lowercase__						):





	'''simple docstring'''
	return int(normalize_answer(lowercase__						) == normalize_answer(lowercase__						)						)





def a(lowercase__      ,			lowercase__						):





	'''simple docstring'''
	snake_case_   		=			get_tokens(lowercase__						)
	snake_case_   		=			get_tokens(lowercase__						)
	snake_case_   		=			collections.Counter(lowercase__						) & collections.Counter(lowercase__						)
	snake_case_   		=			sum(common.values()						)
	if len(lowercase__						) == 0 or len(lowercase__						) == 0:
		# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
		return int(gold_toks == pred_toks						)
	if num_same == 0:
		return 0
	snake_case_   		=			1.0 * num_same / len(lowercase__						)
	snake_case_   		=			1.0 * num_same / len(lowercase__						)
	snake_case_   		=			(2 * precision * recall) / (precision + recall)
	return fa





def a(lowercase__      ,			lowercase__						):





	'''simple docstring'''
	snake_case_   		=			{}
	snake_case_   		=			{}
	for article in dataset:
		for p in article["paragraphs"]:
			for qa in p["qas"]:
				snake_case_   		=			qa['id']
				snake_case_   		=			[t for t in qa['answers']['text'] if normalize_answer(lowercase__						)]
				if not gold_answers:
					# For unanswerable questions, only correct answer is empty string
					snake_case_   		=			['']
				if qid not in preds:
					print(f"""Missing prediction for {qid}"""						)
					continue
				snake_case_   		=			preds[qid]
				# Take max over all gold answers
				snake_case_   		=			max(compute_exact(lowercase__      ,			lowercase__						) for a in gold_answers						)
				snake_case_   		=			max(compute_fa(lowercase__      ,			lowercase__						) for a in gold_answers						)
	return exact_scores, fa_scores





def a(lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__						):





	'''simple docstring'''
	snake_case_   		=			{}
	for qid, s in scores.items():
		snake_case_   		=			na_probs[qid] > na_prob_thresh
		if pred_na:
			snake_case_   		=			float(not qid_to_has_ans[qid]						)
		else:
			snake_case_   		=			s
	return new_scores





def a(lowercase__      ,			lowercase__      ,			lowercase__=None						):





	'''simple docstring'''
	if not qid_list:
		snake_case_   		=			len(lowercase__						)
		return collections.OrderedDict(
		    [
		        ('exact', 100.0 * sum(exact_scores.values()						) / total),
		        ('f1', 100.0 * sum(fa_scores.values()						) / total),
		        ('total', total),
		    ]						)
	else:
		snake_case_   		=			len(lowercase__						)
		return collections.OrderedDict(
		    [
		        ('exact', 100.0 * sum(exact_scores[k] for k in qid_list						) / total),
		        ('f1', 100.0 * sum(fa_scores[k] for k in qid_list						) / total),
		        ('total', total),
		    ]						)





def a(lowercase__      ,			lowercase__      ,			lowercase__						):





	'''simple docstring'''
	for k in new_eval:
		snake_case_   		=			new_eval[k]





def a(lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__						):





	'''simple docstring'''
	plt.step(lowercase__      ,			lowercase__      ,			color='b'      ,			alpha=0.2      ,			where='post'						)
	plt.fill_between(lowercase__      ,			lowercase__      ,			step='post'      ,			alpha=0.2      ,			color='b'						)
	plt.xlabel('Recall'						)
	plt.ylabel('Precision'						)
	plt.xlim([0.0, 1.05]						)
	plt.ylim([0.0, 1.05]						)
	plt.title(lowercase__						)
	plt.savefig(lowercase__						)
	plt.clf()





def a(lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__=None      ,			lowercase__=None						):





	'''simple docstring'''
	snake_case_   		=			sorted(lowercase__      ,			key=lambda lowercase__						: na_probs[k]						)
	snake_case_   		=			0.0
	snake_case_   		=			1.0
	snake_case_   		=			0.0
	snake_case_   		=			[1.0]
	snake_case_   		=			[0.0]
	snake_case_   		=			0.0
	for i, qid in enumerate(lowercase__						):
		if qid_to_has_ans[qid]:
			true_pos += scores[qid]
		snake_case_   		=			true_pos / float(i + 1						)
		snake_case_   		=			true_pos / float(lowercase__						)
		if i == len(lowercase__						) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
			# i.e., if we can put a threshold after this point
			avg_prec += cur_p * (cur_r - recalls[-1])
			precisions.append(lowercase__						)
			recalls.append(lowercase__						)
	if out_image:
		plot_pr_curve(lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__						)
	return {"ap": 100.0 * avg_prec}





def a(lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__						):





	'''simple docstring'''
	if out_image_dir and not os.path.exists(lowercase__						):
		os.makedirs(lowercase__						)
	snake_case_   		=			sum(1 for v in qid_to_has_ans.values() if v						)
	if num_true_pos == 0:
		return
	snake_case_   		=			make_precision_recall_eval(
	    lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__      ,			out_image=os.path.join(lowercase__      ,			'pr_exact.png'						)      ,			title='Precision-Recall curve for Exact Match score'      ,			)
	snake_case_   		=			make_precision_recall_eval(
	    lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__      ,			out_image=os.path.join(lowercase__      ,			'pr_f1.png'						)      ,			title='Precision-Recall curve for F1 score'      ,			)
	snake_case_   		=			{k: float(lowercase__						) for k, v in qid_to_has_ans.items()}
	snake_case_   		=			make_precision_recall_eval(
	    lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__      ,			out_image=os.path.join(lowercase__      ,			'pr_oracle.png'						)      ,			title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'      ,			)
	merge_eval(lowercase__      ,			lowercase__      ,			'pr_exact'						)
	merge_eval(lowercase__      ,			lowercase__      ,			'pr_f1'						)
	merge_eval(lowercase__      ,			lowercase__      ,			'pr_oracle'						)





def a(lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__						):





	'''simple docstring'''
	if not qid_list:
		return
	snake_case_   		=			[na_probs[k] for k in qid_list]
	snake_case_   		=			np.ones_like(lowercase__						) / float(len(lowercase__						)						)
	plt.hist(lowercase__      ,			weights=lowercase__      ,			bins=20      ,			range=(0.0, 1.0)						)
	plt.xlabel('Model probability of no-answer'						)
	plt.ylabel('Proportion of dataset'						)
	plt.title(f"""Histogram of no-answer probability: {name}"""						)
	plt.savefig(os.path.join(lowercase__      ,			f"""na_prob_hist_{name}.png"""						)						)
	plt.clf()





def a(lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__						):





	'''simple docstring'''
	snake_case_   		=			sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]						)
	snake_case_   		=			num_no_ans
	snake_case_   		=			cur_score
	snake_case_   		=			0.0
	snake_case_   		=			sorted(lowercase__      ,			key=lambda lowercase__						: na_probs[k]						)
	for i, qid in enumerate(lowercase__						):
		if qid not in scores:
			continue
		if qid_to_has_ans[qid]:
			snake_case_   		=			scores[qid]
		else:
			if preds[qid]:
				snake_case_   		=			-1
			else:
				snake_case_   		=			0
		cur_score += diff
		if cur_score > best_score:
			snake_case_   		=			cur_score
			snake_case_   		=			na_probs[qid]
	return 100.0 * best_score / len(lowercase__						), best_thresh





def a(lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__						):





	'''simple docstring'''
	snake_case_		,							snake_case_   		=			find_best_thresh(lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__						)
	snake_case_		,							snake_case_   		=			find_best_thresh(lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__						)
	snake_case_   		=			best_exact
	snake_case_   		=			exact_thresh
	snake_case_   		=			best_fa
	snake_case_   		=			fa_thresh





def a():





	'''simple docstring'''
	with open(OPTS.data_file						) as f:
		snake_case_   		=			json.load(lowercase__						)
		snake_case_   		=			dataset_json['data']
	with open(OPTS.pred_file						) as f:
		snake_case_   		=			json.load(lowercase__						)
	if OPTS.na_prob_file:
		with open(OPTS.na_prob_file						) as f:
			snake_case_   		=			json.load(lowercase__						)
	else:
		snake_case_   		=			{k: 0.0 for k in preds}
	snake_case_   		=			make_qid_to_has_ans(lowercase__						)  # maps qid to True/False
	snake_case_   		=			[k for k, v in qid_to_has_ans.items() if v]
	snake_case_   		=			[k for k, v in qid_to_has_ans.items() if not v]
	snake_case_		,							snake_case_   		=			get_raw_scores(lowercase__      ,			lowercase__						)
	snake_case_   		=			apply_no_ans_threshold(lowercase__      ,			lowercase__      ,			lowercase__      ,			OPTS.na_prob_thresh						)
	snake_case_   		=			apply_no_ans_threshold(lowercase__      ,			lowercase__      ,			lowercase__      ,			OPTS.na_prob_thresh						)
	snake_case_   		=			make_eval_dict(lowercase__      ,			lowercase__						)
	if has_ans_qids:
		snake_case_   		=			make_eval_dict(lowercase__      ,			lowercase__      ,			qid_list=lowercase__						)
		merge_eval(lowercase__      ,			lowercase__      ,			'HasAns'						)
	if no_ans_qids:
		snake_case_   		=			make_eval_dict(lowercase__      ,			lowercase__      ,			qid_list=lowercase__						)
		merge_eval(lowercase__      ,			lowercase__      ,			'NoAns'						)
	if OPTS.na_prob_file:
		find_all_best_thresh(lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__						)
	if OPTS.na_prob_file and OPTS.out_image_dir:
		run_precision_recall_analysis(lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__      ,			lowercase__      ,			OPTS.out_image_dir						)
		histogram_na_prob(lowercase__      ,			lowercase__      ,			OPTS.out_image_dir      ,			'hasAns'						)
		histogram_na_prob(lowercase__      ,			lowercase__      ,			OPTS.out_image_dir      ,			'noAns'						)
	if OPTS.out_file:
		with open(OPTS.out_file      ,			'w'						) as f:
			json.dump(lowercase__      ,			lowercase__						)
	else:
		print(json.dumps(lowercase__      ,			indent=2						)						)


if __name__ == "__main__":
		A									=  parse_args()
		if OPTS.out_image_dir:
				import matplotlib

				matplotlib.use('Agg')
				import matplotlib.pyplot as plt
		main()



class SCREAMING_SNAKE_CASE	:






			"""simple docstring"""




			def __init__(     self			,					__UpperCamelCase				):

				"""simple docstring"""


				snake_case_   		=			val
				snake_case_   		=			None
				snake_case_   		=			None







			def       __lowerCAmelCase			(     self			,					__UpperCamelCase				):

				"""simple docstring"""


				if self.val:
					if val < self.val:
						if self.left is None:
							snake_case_   		=			Node(__UpperCamelCase				)
						else:
							self.left.insert(__UpperCamelCase				)
					elif val > self.val:
						if self.right is None:
							snake_case_   		=			Node(__UpperCamelCase				)
						else:
							self.right.insert(__UpperCamelCase				)
				else:
					snake_case_   		=			val







def a(lowercase__      ,			lowercase__						):





	'''simple docstring'''
	# Recursive traversal
	if root:
		inorder(root.left      ,			lowercase__						)
		res.append(root.val						)
		inorder(root.right      ,			lowercase__						)





def a(lowercase__						):





	'''simple docstring'''
	# Build BST
	if len(lowercase__						) == 0:
		return arr
	snake_case_   		=			Node(arr[0]						)
	for i in range(1      ,			len(lowercase__						)						):
		root.insert(arr[i]						)
	# Traverse BST in order.
	snake_case_   		=			[]
	inorder(lowercase__      ,			lowercase__						)
	return res


if __name__ == "__main__":
		print(tree_sort([10, 1, 3, 2, 9, 14, 13]))
