


import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


snake_case							:       Optional[Any]										=	re.compile(R'''\b(a|an|the)\b''', re.UNICODE)

snake_case							:       List[str]										=	None
def 							__lowerCamelCase		(					):




      """simple docstring"""





      a		:Optional[Any]									=			argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''		)
      parser.add_argument('''data_file'''					,    metavar='''data.json'''					,    help='''Input data JSON file.'''		)
      parser.add_argument('''pred_file'''					,    metavar='''pred.json'''					,    help='''Model predictions.'''		)
      parser.add_argument(
          '''--out-file'''					,    '''-o'''					,    metavar='''eval.json'''					,    help='''Write accuracy metrics to file (default is stdout).'''		)
      parser.add_argument(
          '''--na-prob-file'''					,    '''-n'''					,    metavar='''na_prob.json'''					,    help='''Model estimates of probability of no answer.'''		)
      parser.add_argument(
          '''--na-prob-thresh'''					,    '''-t'''					,    type=UpperCAmelCase_					,    default=1.0					,    help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''					,    )
      parser.add_argument(
          '''--out-image-dir'''					,    '''-p'''					,    metavar='''out_images'''					,    default=UpperCAmelCase_					,    help='''Save precision-recall curves to directory.'''		)
      parser.add_argument('''--verbose'''					,    '''-v'''					,    action='''store_true'''		)
      if len(sys.argv		) == 1:
            parser.print_help()
            sys.exit(1		)
      return parser.parse_args()
def 							__lowerCamelCase		(					UpperCAmelCase_		:						Optional[int]		):




      """simple docstring"""





      a		:Optional[Any]									=			{}
      for article in dataset:
            for p in article["paragraphs"]:
                  for qa in p["qas"]:
                        a		:Union[str, Any]									=			bool(qa['''answers''']['''text''']		)
      return qid_to_has_ans
def 							__lowerCamelCase		(					UpperCAmelCase_		:						Dict		):




      """simple docstring"""





      def remove_articles(UpperCAmelCase_		:						Union[str, Any]		):
            return ARTICLES_REGEX.sub(''' '''					,    UpperCAmelCase_		)

      def white_space_fix(UpperCAmelCase_		:						List[Any]		):
            return " ".join(text.split()		)

      def remove_punc(UpperCAmelCase_		:						Optional[int]		):
            a		:List[Any]									=			set(string.punctuation		)
            return "".join(ch for ch in text if ch not in exclude		)

      def lower(UpperCAmelCase_		:						Dict		):
            return text.lower()

      return white_space_fix(remove_articles(remove_punc(lower(UpperCAmelCase_		)		)		)		)
def 							__lowerCamelCase		(					UpperCAmelCase_		:						Optional[int]		):




      """simple docstring"""





      if not s:
            return []
      return normalize_answer(UpperCAmelCase_		).split()
def 							__lowerCamelCase		(					UpperCAmelCase_		:						Union[str, Any]					,    UpperCAmelCase_		:						List[Any]		):




      """simple docstring"""





      return int(normalize_answer(UpperCAmelCase_		) == normalize_answer(UpperCAmelCase_		)		)
def 							__lowerCamelCase		(					UpperCAmelCase_		:						Dict					,    UpperCAmelCase_		:						List[Any]		):




      """simple docstring"""





      a		:Optional[int]									=			get_tokens(UpperCAmelCase_		)
      a		:List[str]									=			get_tokens(UpperCAmelCase_		)
      a		:List[str]									=			collections.Counter(UpperCAmelCase_		) & collections.Counter(UpperCAmelCase_		)
      a		:List[str]									=			sum(common.values()		)
      if len(UpperCAmelCase_		) == 0 or len(UpperCAmelCase_		) == 0:
            # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
            return int(gold_toks == pred_toks		)
      if num_same == 0:
            return 0
      a		:List[str]									=			1.0 * num_same / len(UpperCAmelCase_		)
      a		:Tuple									=			1.0 * num_same / len(UpperCAmelCase_		)
      a		:Optional[int]									=			(2 * precision * recall) / (precision + recall)
      return fa
def 							__lowerCamelCase		(					UpperCAmelCase_		:						List[Any]					,    UpperCAmelCase_		:						str		):




      """simple docstring"""





      a		:Optional[int]									=			{}
      a		:List[str]									=			{}
      for article in dataset:
            for p in article["paragraphs"]:
                  for qa in p["qas"]:
                        a		:List[Any]									=			qa['''id''']
                        a		:Union[str, Any]									=			[t for t in qa['''answers''']['''text'''] if normalize_answer(UpperCAmelCase_		)]
                        if not gold_answers:
                              # For unanswerable questions, only correct answer is empty string
                              a		:Any									=			['''''']
                        if qid not in preds:
                              print(F'''Missing prediction for {qid}'''		)
                              continue
                        a		:Optional[Any]									=			preds[qid]
                        # Take max over all gold answers
                        a		:Union[str, Any]									=			max(compute_exact(UpperCAmelCase_					,    UpperCAmelCase_		) for a in gold_answers		)
                        a		:Union[str, Any]									=			max(compute_fa(UpperCAmelCase_					,    UpperCAmelCase_		) for a in gold_answers		)
      return exact_scores, fa_scores
def 							__lowerCamelCase		(					UpperCAmelCase_		:						int					,    UpperCAmelCase_		:						List[Any]					,    UpperCAmelCase_		:						str					,    UpperCAmelCase_		:						Dict		):




      """simple docstring"""





      a		:Tuple									=			{}
      for qid, s in scores.items():
            a		:List[str]									=			na_probs[qid] > na_prob_thresh
            if pred_na:
                  a		:str									=			float(not qid_to_has_ans[qid]		)
            else:
                  a		:Tuple									=			s
      return new_scores
def 							__lowerCamelCase		(					UpperCAmelCase_		:						Union[str, Any]					,    UpperCAmelCase_		:						Tuple					,    UpperCAmelCase_		:						Any=None		):




      """simple docstring"""





      if not qid_list:
            a		:Dict									=			len(UpperCAmelCase_		)
            return collections.OrderedDict(
                [
                    ('''exact''', 100.0 * sum(exact_scores.values()		) / total),
                    ('''f1''', 100.0 * sum(fa_scores.values()		) / total),
                    ('''total''', total),
                ]		)
      else:
            a		:int									=			len(UpperCAmelCase_		)
            return collections.OrderedDict(
                [
                    ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list		) / total),
                    ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list		) / total),
                    ('''total''', total),
                ]		)
def 							__lowerCamelCase		(					UpperCAmelCase_		:						Tuple					,    UpperCAmelCase_		:						Optional[int]					,    UpperCAmelCase_		:						Dict		):




      """simple docstring"""





      for k in new_eval:
            a		:str									=			new_eval[k]
def 							__lowerCamelCase		(					UpperCAmelCase_		:						int					,    UpperCAmelCase_		:						Any					,    UpperCAmelCase_		:						Any					,    UpperCAmelCase_		:						Tuple		):




      """simple docstring"""





      plt.step(UpperCAmelCase_					,    UpperCAmelCase_					,    color='''b'''					,    alpha=0.2					,    where='''post'''		)
      plt.fill_between(UpperCAmelCase_					,    UpperCAmelCase_					,    step='''post'''					,    alpha=0.2					,    color='''b'''		)
      plt.xlabel('''Recall'''		)
      plt.ylabel('''Precision'''		)
      plt.xlim([0.0, 1.05]		)
      plt.ylim([0.0, 1.05]		)
      plt.title(UpperCAmelCase_		)
      plt.savefig(UpperCAmelCase_		)
      plt.clf()
def 							__lowerCamelCase		(					UpperCAmelCase_		:						List[str]					,    UpperCAmelCase_		:						List[str]					,    UpperCAmelCase_		:						Optional[Any]					,    UpperCAmelCase_		:						Tuple					,    UpperCAmelCase_		:						Any=None					,    UpperCAmelCase_		:						Dict=None		):




      """simple docstring"""





      a		:Optional[Any]									=			sorted(UpperCAmelCase_					,    key=lambda UpperCAmelCase_		: na_probs[k]		)
      a		:Optional[int]									=			0.0
      a		:List[Any]									=			1.0
      a		:Dict									=			0.0
      a		:Any									=			[1.0]
      a		:Optional[int]									=			[0.0]
      a		:int									=			0.0
      for i, qid in enumerate(UpperCAmelCase_		):
            if qid_to_has_ans[qid]:
                  true_pos += scores[qid]
            a		:List[str]									=			true_pos / float(i + 1		)
            a		:Dict									=			true_pos / float(UpperCAmelCase_		)
            if i == len(UpperCAmelCase_		) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                  # i.e., if we can put a threshold after this point
                  avg_prec += cur_p * (cur_r - recalls[-1])
                  precisions.append(UpperCAmelCase_		)
                  recalls.append(UpperCAmelCase_		)
      if out_image:
            plot_pr_curve(UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_		)
      return {"ap": 100.0 * avg_prec}
def 							__lowerCamelCase		(					UpperCAmelCase_		:						Optional[Any]					,    UpperCAmelCase_		:						Tuple					,    UpperCAmelCase_		:						Tuple					,    UpperCAmelCase_		:						Optional[int]					,    UpperCAmelCase_		:						Union[str, Any]					,    UpperCAmelCase_		:						Optional[Any]		):




      """simple docstring"""





      if out_image_dir and not os.path.exists(UpperCAmelCase_		):
            os.makedirs(UpperCAmelCase_		)
      a		:int									=			sum(1 for v in qid_to_has_ans.values() if v		)
      if num_true_pos == 0:
            return
      a		:Optional[int]									=			make_precision_recall_eval(
          UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    out_image=os.path.join(UpperCAmelCase_					,    '''pr_exact.png'''		)					,    title='''Precision-Recall curve for Exact Match score'''					,    )
      a		:Any									=			make_precision_recall_eval(
          UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    out_image=os.path.join(UpperCAmelCase_					,    '''pr_f1.png'''		)					,    title='''Precision-Recall curve for F1 score'''					,    )
      a		:int									=			{k: float(UpperCAmelCase_		) for k, v in qid_to_has_ans.items()}
      a		:int									=			make_precision_recall_eval(
          UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    out_image=os.path.join(UpperCAmelCase_					,    '''pr_oracle.png'''		)					,    title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''					,    )
      merge_eval(UpperCAmelCase_					,    UpperCAmelCase_					,    '''pr_exact'''		)
      merge_eval(UpperCAmelCase_					,    UpperCAmelCase_					,    '''pr_f1'''		)
      merge_eval(UpperCAmelCase_					,    UpperCAmelCase_					,    '''pr_oracle'''		)
def 							__lowerCamelCase		(					UpperCAmelCase_		:						Optional[Any]					,    UpperCAmelCase_		:						Optional[Any]					,    UpperCAmelCase_		:						Tuple					,    UpperCAmelCase_		:						Tuple		):




      """simple docstring"""





      if not qid_list:
            return
      a		:List[Any]									=			[na_probs[k] for k in qid_list]
      a		:Dict									=			np.ones_like(UpperCAmelCase_		) / float(len(UpperCAmelCase_		)		)
      plt.hist(UpperCAmelCase_					,    weights=UpperCAmelCase_					,    bins=20					,    range=(0.0, 1.0)		)
      plt.xlabel('''Model probability of no-answer'''		)
      plt.ylabel('''Proportion of dataset'''		)
      plt.title(F'''Histogram of no-answer probability: {name}'''		)
      plt.savefig(os.path.join(UpperCAmelCase_					,    F'''na_prob_hist_{name}.png'''		)		)
      plt.clf()
def 							__lowerCamelCase		(					UpperCAmelCase_		:						List[Any]					,    UpperCAmelCase_		:						Optional[Any]					,    UpperCAmelCase_		:						Tuple					,    UpperCAmelCase_		:						List[Any]		):




      """simple docstring"""





      a		:Optional[int]									=			sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]		)
      a		:str									=			num_no_ans
      a		:List[Any]									=			cur_score
      a		:Dict									=			0.0
      a		:Optional[int]									=			sorted(UpperCAmelCase_					,    key=lambda UpperCAmelCase_		: na_probs[k]		)
      for i, qid in enumerate(UpperCAmelCase_		):
            if qid not in scores:
                  continue
            if qid_to_has_ans[qid]:
                  a		:List[str]									=			scores[qid]
            else:
                  if preds[qid]:
                        a		:Optional[Any]									=			-1
                  else:
                        a		:Dict									=			0
            cur_score += diff
            if cur_score > best_score:
                  a		:Dict									=			cur_score
                  a		:Dict									=			na_probs[qid]
      return 100.0 * best_score / len(UpperCAmelCase_		), best_thresh
def 							__lowerCamelCase		(					UpperCAmelCase_		:						int					,    UpperCAmelCase_		:						Union[str, Any]					,    UpperCAmelCase_		:						str					,    UpperCAmelCase_		:						Optional[int]					,    UpperCAmelCase_		:						Optional[Any]					,    UpperCAmelCase_		:						Tuple		):




      """simple docstring"""





      a				,							a		:List[Any]									=			find_best_thresh(UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_		)
      a				,							a		:List[Any]									=			find_best_thresh(UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_		)
      a		:List[str]									=			best_exact
      a		:Optional[Any]									=			exact_thresh
      a		:Optional[int]									=			best_fa
      a		:int									=			fa_thresh
def 							__lowerCamelCase		(					):




      """simple docstring"""





      with open(OPTS.data_file		) as f:
            a		:Optional[int]									=			json.load(UpperCAmelCase_		)
            a		:Any									=			dataset_json['''data''']
      with open(OPTS.pred_file		) as f:
            a		:str									=			json.load(UpperCAmelCase_		)
      if OPTS.na_prob_file:
            with open(OPTS.na_prob_file		) as f:
                  a		:Optional[int]									=			json.load(UpperCAmelCase_		)
      else:
            a		:List[str]									=			{k: 0.0 for k in preds}
      a		:Tuple									=			make_qid_to_has_ans(UpperCAmelCase_		)  # maps qid to True/False
      a		:Optional[int]									=			[k for k, v in qid_to_has_ans.items() if v]
      a		:int									=			[k for k, v in qid_to_has_ans.items() if not v]
      a				,							a		:str									=			get_raw_scores(UpperCAmelCase_					,    UpperCAmelCase_		)
      a		:Union[str, Any]									=			apply_no_ans_threshold(UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    OPTS.na_prob_thresh		)
      a		:Optional[Any]									=			apply_no_ans_threshold(UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    OPTS.na_prob_thresh		)
      a		:int									=			make_eval_dict(UpperCAmelCase_					,    UpperCAmelCase_		)
      if has_ans_qids:
            a		:str									=			make_eval_dict(UpperCAmelCase_					,    UpperCAmelCase_					,    qid_list=UpperCAmelCase_		)
            merge_eval(UpperCAmelCase_					,    UpperCAmelCase_					,    '''HasAns'''		)
      if no_ans_qids:
            a		:Optional[Any]									=			make_eval_dict(UpperCAmelCase_					,    UpperCAmelCase_					,    qid_list=UpperCAmelCase_		)
            merge_eval(UpperCAmelCase_					,    UpperCAmelCase_					,    '''NoAns'''		)
      if OPTS.na_prob_file:
            find_all_best_thresh(UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_		)
      if OPTS.na_prob_file and OPTS.out_image_dir:
            run_precision_recall_analysis(UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    UpperCAmelCase_					,    OPTS.out_image_dir		)
            histogram_na_prob(UpperCAmelCase_					,    UpperCAmelCase_					,    OPTS.out_image_dir					,    '''hasAns'''		)
            histogram_na_prob(UpperCAmelCase_					,    UpperCAmelCase_					,    OPTS.out_image_dir					,    '''noAns'''		)
      if OPTS.out_file:
            with open(OPTS.out_file					,    '''w'''		) as f:
                  json.dump(UpperCAmelCase_					,    UpperCAmelCase_		)
      else:
            print(json.dumps(UpperCAmelCase_					,    indent=2		)		)


if __name__ == "__main__":
     snake_case							:       int										=	parse_args()
     if OPTS.out_image_dir:
          import matplotlib

          matplotlib.use('''Agg''')
          import matplotlib.pyplot as plt
     main()









from __future__ import annotations

from typing import Any







class 						_snake_case    ( _snake_case				):
 pass







class 						_snake_case    :




 def __init__(						self   ,    _lowerCamelCase       ):
       a		:Any									=			data
       a		:Node | None									=			None




 def __iter__(						self       ):
       a		:Dict									=			self
       a		:Optional[int]									=			[]
       while node:
             if node in visited:
                   raise ContainsLoopError
             visited.append(_lowerCamelCase       )
             yield node.data
             a		:str									=			node.next_node




 @property
 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       try:
             list(self       )
             return False
       except ContainsLoopError:
             return True


if __name__ == "__main__":
     snake_case							:       int										=	Node(1)
     snake_case							:       int										=	Node(2)
     snake_case							:       Optional[int]										=	Node(3)
     snake_case							:       List[str]										=	Node(4)
     print(root_node.has_loop)  # False
     snake_case							:       Optional[Any]										=	root_node.next_node
     print(root_node.has_loop)  # True

     snake_case							:       Dict										=	Node(5)
     snake_case							:       List[Any]										=	Node(6)
     snake_case							:       Optional[int]										=	Node(5)
     snake_case							:       Union[str, Any]										=	Node(6)
     print(root_node.has_loop)  # False

     snake_case							:       Tuple										=	Node(1)
     print(root_node.has_loop)  # False





