






import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


a_		:    Optional[Any]   							=     re.compile(R'\b(a|an|the)\b', re.UNICODE)

a_		:    List[str]   							=     None






def 					lowerCamelCase__					():
			SCREAMING_SNAKE_CASE							=			argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.')
			parser.add_argument('data_file'				,							metavar='data.json'				,							help='Input data JSON file.')
			parser.add_argument('pred_file'				,							metavar='pred.json'				,							help='Model predictions.')
			parser.add_argument(
			    '--out-file'				,							'-o'				,							metavar='eval.json'				,							help='Write accuracy metrics to file (default is stdout).')
			parser.add_argument(
			    '--na-prob-file'				,							'-n'				,							metavar='na_prob.json'				,							help='Model estimates of probability of no answer.')
			parser.add_argument(
			    '--na-prob-thresh'				,							'-t'				,							type=_UpperCAmelCase				,							default=1.0				,							help='Predict "" if no-answer probability exceeds this (default = 1.0).'				,							)
			parser.add_argument(
			    '--out-image-dir'				,							'-p'				,							metavar='out_images'				,							default=_UpperCAmelCase				,							help='Save precision-recall curves to directory.')
			parser.add_argument('--verbose'				,							'-v'				,							action='store_true')
			if len(sys.argv) == 1:
						parser.print_help()
						sys.exit(1)
			return parser.parse_args()






def 					lowerCamelCase__					(_UpperCAmelCase):
			SCREAMING_SNAKE_CASE							=			{}
			for article in dataset:
						for p in article["paragraphs"]:
									for qa in p["qas"]:
												SCREAMING_SNAKE_CASE							=			bool(qa['answers']['text'])
			return qid_to_has_ans






def 					lowerCamelCase__					(_UpperCAmelCase):

			def remove_articles(_UpperCAmelCase):
						return ARTICLES_REGEX.sub(' '				,							_UpperCAmelCase)

			def white_space_fix(_UpperCAmelCase):
						return " ".join(text.split())

			def remove_punc(_UpperCAmelCase):
						SCREAMING_SNAKE_CASE							=			set(string.punctuation)
						return "".join(ch for ch in text if ch not in exclude)

			def lower(_UpperCAmelCase):
						return text.lower()

			return white_space_fix(remove_articles(remove_punc(lower(_UpperCAmelCase))))






def 					lowerCamelCase__					(_UpperCAmelCase):
			if not s:
						return []
			return normalize_answer(_UpperCAmelCase).split()






def 					lowerCamelCase__					(_UpperCAmelCase				,							_UpperCAmelCase):
			return int(normalize_answer(_UpperCAmelCase) == normalize_answer(_UpperCAmelCase))






def 					lowerCamelCase__					(_UpperCAmelCase				,							_UpperCAmelCase):
			SCREAMING_SNAKE_CASE							=			get_tokens(_UpperCAmelCase)
			SCREAMING_SNAKE_CASE							=			get_tokens(_UpperCAmelCase)
			SCREAMING_SNAKE_CASE							=			collections.Counter(_UpperCAmelCase) & collections.Counter(_UpperCAmelCase)
			SCREAMING_SNAKE_CASE							=			sum(common.values())
			if len(_UpperCAmelCase) == 0 or len(_UpperCAmelCase) == 0:
						# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
						return int(gold_toks == pred_toks)
			if num_same == 0:
						return 0
			SCREAMING_SNAKE_CASE							=			1.0 * num_same / len(_UpperCAmelCase)
			SCREAMING_SNAKE_CASE							=			1.0 * num_same / len(_UpperCAmelCase)
			SCREAMING_SNAKE_CASE							=			(2 * precision * recall) / (precision + recall)
			return fa






def 					lowerCamelCase__					(_UpperCAmelCase				,							_UpperCAmelCase):
			SCREAMING_SNAKE_CASE							=			{}
			SCREAMING_SNAKE_CASE							=			{}
			for article in dataset:
						for p in article["paragraphs"]:
									for qa in p["qas"]:
												SCREAMING_SNAKE_CASE							=			qa['id']
												SCREAMING_SNAKE_CASE							=			[t for t in qa['answers']['text'] if normalize_answer(_UpperCAmelCase)]
												if not gold_answers:
															# For unanswerable questions, only correct answer is empty string
															SCREAMING_SNAKE_CASE							=			['']
												if qid not in preds:
															print(F'''Missing prediction for {qid}''')
															continue
												SCREAMING_SNAKE_CASE							=			preds[qid]
												# Take max over all gold answers
												SCREAMING_SNAKE_CASE							=			max(compute_exact(_UpperCAmelCase				,							_UpperCAmelCase) for a in gold_answers)
												SCREAMING_SNAKE_CASE							=			max(compute_fa(_UpperCAmelCase				,							_UpperCAmelCase) for a in gold_answers)
			return exact_scores, fa_scores






def 					lowerCamelCase__					(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase):
			SCREAMING_SNAKE_CASE							=			{}
			for qid, s in scores.items():
						SCREAMING_SNAKE_CASE							=			na_probs[qid] > na_prob_thresh
						if pred_na:
									SCREAMING_SNAKE_CASE							=			float(not qid_to_has_ans[qid])
						else:
									SCREAMING_SNAKE_CASE							=			s
			return new_scores






def 					lowerCamelCase__					(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase=None):
			if not qid_list:
						SCREAMING_SNAKE_CASE							=			len(_UpperCAmelCase)
						return collections.OrderedDict(
						    [
						        ('exact', 1_00.0 * sum(exact_scores.values()) / total),
						        ('f1', 1_00.0 * sum(fa_scores.values()) / total),
						        ('total', total),
						    ])
			else:
						SCREAMING_SNAKE_CASE							=			len(_UpperCAmelCase)
						return collections.OrderedDict(
						    [
						        ('exact', 1_00.0 * sum(exact_scores[k] for k in qid_list) / total),
						        ('f1', 1_00.0 * sum(fa_scores[k] for k in qid_list) / total),
						        ('total', total),
						    ])






def 					lowerCamelCase__					(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase):
			for k in new_eval:
						SCREAMING_SNAKE_CASE							=			new_eval[k]






def 					lowerCamelCase__					(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase):
			plt.step(_UpperCAmelCase				,							_UpperCAmelCase				,							color='b'				,							alpha=0.2				,							where='post')
			plt.fill_between(_UpperCAmelCase				,							_UpperCAmelCase				,							step='post'				,							alpha=0.2				,							color='b')
			plt.xlabel('Recall')
			plt.ylabel('Precision')
			plt.xlim([0.0, 1.05])
			plt.ylim([0.0, 1.05])
			plt.title(_UpperCAmelCase)
			plt.savefig(_UpperCAmelCase)
			plt.clf()






def 					lowerCamelCase__					(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase=None				,							_UpperCAmelCase=None):
			SCREAMING_SNAKE_CASE							=			sorted(_UpperCAmelCase				,							key=lambda _UpperCAmelCase: na_probs[k])
			SCREAMING_SNAKE_CASE							=			0.0
			SCREAMING_SNAKE_CASE							=			1.0
			SCREAMING_SNAKE_CASE							=			0.0
			SCREAMING_SNAKE_CASE							=			[1.0]
			SCREAMING_SNAKE_CASE							=			[0.0]
			SCREAMING_SNAKE_CASE							=			0.0
			for i, qid in enumerate(_UpperCAmelCase):
						if qid_to_has_ans[qid]:
									true_pos += scores[qid]
						SCREAMING_SNAKE_CASE							=			true_pos / float(i + 1)
						SCREAMING_SNAKE_CASE							=			true_pos / float(_UpperCAmelCase)
						if i == len(_UpperCAmelCase) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
									# i.e., if we can put a threshold after this point
									avg_prec += cur_p * (cur_r - recalls[-1])
									precisions.append(_UpperCAmelCase)
									recalls.append(_UpperCAmelCase)
			if out_image:
						plot_pr_curve(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase)
			return {"ap": 1_00.0 * avg_prec}






def 					lowerCamelCase__					(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase):
			if out_image_dir and not os.path.exists(_UpperCAmelCase):
						os.makedirs(_UpperCAmelCase)
			SCREAMING_SNAKE_CASE							=			sum(1 for v in qid_to_has_ans.values() if v)
			if num_true_pos == 0:
						return
			SCREAMING_SNAKE_CASE							=			make_precision_recall_eval(
			    _UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							out_image=os.path.join(_UpperCAmelCase				,							'pr_exact.png')				,							title='Precision-Recall curve for Exact Match score'				,							)
			SCREAMING_SNAKE_CASE							=			make_precision_recall_eval(
			    _UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							out_image=os.path.join(_UpperCAmelCase				,							'pr_f1.png')				,							title='Precision-Recall curve for F1 score'				,							)
			SCREAMING_SNAKE_CASE							=			{k: float(_UpperCAmelCase) for k, v in qid_to_has_ans.items()}
			SCREAMING_SNAKE_CASE							=			make_precision_recall_eval(
			    _UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							out_image=os.path.join(_UpperCAmelCase				,							'pr_oracle.png')				,							title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'				,							)
			merge_eval(_UpperCAmelCase				,							_UpperCAmelCase				,							'pr_exact')
			merge_eval(_UpperCAmelCase				,							_UpperCAmelCase				,							'pr_f1')
			merge_eval(_UpperCAmelCase				,							_UpperCAmelCase				,							'pr_oracle')






def 					lowerCamelCase__					(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase):
			if not qid_list:
						return
			SCREAMING_SNAKE_CASE							=			[na_probs[k] for k in qid_list]
			SCREAMING_SNAKE_CASE							=			np.ones_like(_UpperCAmelCase) / float(len(_UpperCAmelCase))
			plt.hist(_UpperCAmelCase				,							weights=_UpperCAmelCase				,							bins=20				,							range=(0.0, 1.0))
			plt.xlabel('Model probability of no-answer')
			plt.ylabel('Proportion of dataset')
			plt.title(F'''Histogram of no-answer probability: {name}''')
			plt.savefig(os.path.join(_UpperCAmelCase				,							F'''na_prob_hist_{name}.png'''))
			plt.clf()






def 					lowerCamelCase__					(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase):
			SCREAMING_SNAKE_CASE							=			sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k])
			SCREAMING_SNAKE_CASE							=			num_no_ans
			SCREAMING_SNAKE_CASE							=			cur_score
			SCREAMING_SNAKE_CASE							=			0.0
			SCREAMING_SNAKE_CASE							=			sorted(_UpperCAmelCase				,							key=lambda _UpperCAmelCase: na_probs[k])
			for i, qid in enumerate(_UpperCAmelCase):
						if qid not in scores:
									continue
						if qid_to_has_ans[qid]:
									SCREAMING_SNAKE_CASE							=			scores[qid]
						else:
									if preds[qid]:
												SCREAMING_SNAKE_CASE							=			-1
									else:
												SCREAMING_SNAKE_CASE							=			0
						cur_score += diff
						if cur_score > best_score:
									SCREAMING_SNAKE_CASE							=			cur_score
									SCREAMING_SNAKE_CASE							=			na_probs[qid]
			return 1_00.0 * best_score / len(_UpperCAmelCase), best_thresh






def 					lowerCamelCase__					(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase):
			SCREAMING_SNAKE_CASE					,    SCREAMING_SNAKE_CASE							=			find_best_thresh(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase)
			SCREAMING_SNAKE_CASE					,    SCREAMING_SNAKE_CASE							=			find_best_thresh(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase)
			SCREAMING_SNAKE_CASE							=			best_exact
			SCREAMING_SNAKE_CASE							=			exact_thresh
			SCREAMING_SNAKE_CASE							=			best_fa
			SCREAMING_SNAKE_CASE							=			fa_thresh






def 					lowerCamelCase__					():
			with open(OPTS.data_file) as f:
						SCREAMING_SNAKE_CASE							=			json.load(_UpperCAmelCase)
						SCREAMING_SNAKE_CASE							=			dataset_json['data']
			with open(OPTS.pred_file) as f:
						SCREAMING_SNAKE_CASE							=			json.load(_UpperCAmelCase)
			if OPTS.na_prob_file:
						with open(OPTS.na_prob_file) as f:
									SCREAMING_SNAKE_CASE							=			json.load(_UpperCAmelCase)
			else:
						SCREAMING_SNAKE_CASE							=			{k: 0.0 for k in preds}
			SCREAMING_SNAKE_CASE							=			make_qid_to_has_ans(_UpperCAmelCase)  # maps qid to True/False
			SCREAMING_SNAKE_CASE							=			[k for k, v in qid_to_has_ans.items() if v]
			SCREAMING_SNAKE_CASE							=			[k for k, v in qid_to_has_ans.items() if not v]
			SCREAMING_SNAKE_CASE					,    SCREAMING_SNAKE_CASE							=			get_raw_scores(_UpperCAmelCase				,							_UpperCAmelCase)
			SCREAMING_SNAKE_CASE							=			apply_no_ans_threshold(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							OPTS.na_prob_thresh)
			SCREAMING_SNAKE_CASE							=			apply_no_ans_threshold(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							OPTS.na_prob_thresh)
			SCREAMING_SNAKE_CASE							=			make_eval_dict(_UpperCAmelCase				,							_UpperCAmelCase)
			if has_ans_qids:
						SCREAMING_SNAKE_CASE							=			make_eval_dict(_UpperCAmelCase				,							_UpperCAmelCase				,							qid_list=_UpperCAmelCase)
						merge_eval(_UpperCAmelCase				,							_UpperCAmelCase				,							'HasAns')
			if no_ans_qids:
						SCREAMING_SNAKE_CASE							=			make_eval_dict(_UpperCAmelCase				,							_UpperCAmelCase				,							qid_list=_UpperCAmelCase)
						merge_eval(_UpperCAmelCase				,							_UpperCAmelCase				,							'NoAns')
			if OPTS.na_prob_file:
						find_all_best_thresh(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase)
			if OPTS.na_prob_file and OPTS.out_image_dir:
						run_precision_recall_analysis(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							OPTS.out_image_dir)
						histogram_na_prob(_UpperCAmelCase				,							_UpperCAmelCase				,							OPTS.out_image_dir				,							'hasAns')
						histogram_na_prob(_UpperCAmelCase				,							_UpperCAmelCase				,							OPTS.out_image_dir				,							'noAns')
			if OPTS.out_file:
						with open(OPTS.out_file				,							'w') as f:
									json.dump(_UpperCAmelCase				,							_UpperCAmelCase)
			else:
						print(json.dumps(_UpperCAmelCase				,							indent=2))


if __name__ == "__main__":
				a_		:    Any   							=     parse_args()
				if OPTS.out_image_dir:
								import matplotlib

								matplotlib.use('Agg')
								import matplotlib.pyplot as plt
				main()







def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE   ):
      '''simple docstring'''







      if len(SCREAMING_SNAKE_CASE   ) < 2:
            return collection

      def circle_sort_util(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   ) -> bool:
            A_							       =		False

            if low == high:
                  return swapped

            A_							       =		low
            A_							       =		high

            while left < right:
                  if collection[left] > collection[right]:
                        A_       ,A_							       =		(
                            collection[right],
                            collection[left],
                        )
                        A_							       =		True

                  left += 1
                  right -= 1

            if left == right and collection[left] > collection[right + 1]:
                  A_       ,A_							       =		(
                      collection[right + 1],
                      collection[left],
                  )

                  A_							       =		True

            A_							       =		low + int((high - low) / 2   )
            A_							       =		circle_sort_util(SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE		,				SCREAMING_SNAKE_CASE   )
            A_							       =		circle_sort_util(SCREAMING_SNAKE_CASE		,				mid + 1		,				SCREAMING_SNAKE_CASE   )

            return swapped or left_swap or right_swap

      A_							       =		True

      while is_not_sorted is True:
            A_							       =		circle_sort_util(SCREAMING_SNAKE_CASE		,				0		,				len(SCREAMING_SNAKE_CASE   ) - 1   )

      return collection


if __name__ == "__main__":
      __lowercase				      =						input("""Enter numbers separated by a comma:\n""").strip()
      __lowercase				      =						[int(item) for item in user_input.split(""",""")]
      print(circle_sort(unsorted))





