


"""simple docstring"""
import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


UpperCAmelCase				=						re.compile(r"""\b(a|an|the)\b""", re.UNICODE)

UpperCAmelCase				=						None

def 				_snake_case      (   ):




   """simple docstring"""







   _lowerCamelCase	:					List[Any]   =		argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0."""   )
   parser.add_argument("""data_file"""						, metavar="""data.json"""						, help="""Input data JSON file."""   )
   parser.add_argument("""pred_file"""						, metavar="""pred.json"""						, help="""Model predictions."""   )
   parser.add_argument(
       """--out-file"""						, """-o"""						, metavar="""eval.json"""						, help="""Write accuracy metrics to file (default is stdout)."""   )
   parser.add_argument(
       """--na-prob-file"""						, """-n"""						, metavar="""na_prob.json"""						, help="""Model estimates of probability of no answer."""   )
   parser.add_argument(
       """--na-prob-thresh"""						, """-t"""						, type=__snake_case						, default=1.0						, help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""						, )
   parser.add_argument(
       """--out-image-dir"""						, """-p"""						, metavar="""out_images"""						, default=__snake_case						, help="""Save precision-recall curves to directory."""   )
   parser.add_argument("""--verbose"""						, """-v"""						, action="""store_true"""   )
   if len(sys.argv   ) == 1:
      parser.print_help()
      sys.exit(1   )
   return parser.parse_args()

def 				_snake_case      (   __snake_case							:      Dict   ):




   """simple docstring"""







   _lowerCamelCase	:					Dict   =		{}
   for article in dataset:
      for p in article["paragraphs"]:
         for qa in p["qas"]:
            _lowerCamelCase	:					Optional[Any]   =		bool(qa["""answers"""]["""text"""]   )
   return qid_to_has_ans

def 				_snake_case      (   __snake_case							:      Union[str, Any]   ):




   """simple docstring"""







   def remove_articles(__snake_case							:      List[str]   ):
      return ARTICLES_REGEX.sub(""" """						, __snake_case   )

   def white_space_fix(__snake_case							:      Optional[int]   ):
      return " ".join(text.split()   )

   def remove_punc(__snake_case							:      Optional[Any]   ):
      _lowerCamelCase	:					str   =		set(string.punctuation   )
      return "".join(ch for ch in text if ch not in exclude   )

   def lower(__snake_case							:      Tuple   ):
      return text.lower()

   return white_space_fix(remove_articles(remove_punc(lower(__snake_case   )   )   )   )

def 				_snake_case      (   __snake_case							:      Any   ):




   """simple docstring"""







   if not s:
      return []
   return normalize_answer(__snake_case   ).split()

def 				_snake_case      (   __snake_case							:      Dict						, __snake_case							:      int   ):




   """simple docstring"""







   return int(normalize_answer(__snake_case   ) == normalize_answer(__snake_case   )   )

def 				_snake_case      (   __snake_case							:      Tuple						, __snake_case							:      Any   ):




   """simple docstring"""







   _lowerCamelCase	:					Optional[int]   =		get_tokens(__snake_case   )
   _lowerCamelCase	:					int   =		get_tokens(__snake_case   )
   _lowerCamelCase	:					Optional[Any]   =		collections.Counter(__snake_case   ) & collections.Counter(__snake_case   )
   _lowerCamelCase	:					int   =		sum(common.values()   )
   if len(__snake_case   ) == 0 or len(__snake_case   ) == 0:
      # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
      return int(gold_toks == pred_toks   )
   if num_same == 0:
      return 0
   _lowerCamelCase	:					List[str]   =		1.0 * num_same / len(__snake_case   )
   _lowerCamelCase	:					Dict   =		1.0 * num_same / len(__snake_case   )
   _lowerCamelCase	:					int   =		(2 * precision * recall) / (precision + recall)
   return fa

def 				_snake_case      (   __snake_case							:      Optional[Any]						, __snake_case							:      str   ):




   """simple docstring"""







   _lowerCamelCase	:					List[Any]   =		{}
   _lowerCamelCase	:					Optional[int]   =		{}
   for article in dataset:
      for p in article["paragraphs"]:
         for qa in p["qas"]:
            _lowerCamelCase	:					Tuple   =		qa["""id"""]
            _lowerCamelCase	:					Optional[int]   =		[t for t in qa["""answers"""]["""text"""] if normalize_answer(__snake_case   )]
            if not gold_answers:
               # For unanswerable questions, only correct answer is empty string
               _lowerCamelCase	:					List[str]   =		[""""""]
            if qid not in preds:
               print(F'Missing prediction for {qid}'   )
               continue
            _lowerCamelCase	:					Any   =		preds[qid]
            # Take max over all gold answers
            _lowerCamelCase	:					Tuple   =		max(compute_exact(__snake_case						, __snake_case   ) for a in gold_answers   )
            _lowerCamelCase	:					List[str]   =		max(compute_fa(__snake_case						, __snake_case   ) for a in gold_answers   )
   return exact_scores, fa_scores

def 				_snake_case      (   __snake_case							:      List[Any]						, __snake_case							:      Any						, __snake_case							:      Union[str, Any]						, __snake_case							:      str   ):




   """simple docstring"""







   _lowerCamelCase	:					str   =		{}
   for qid, s in scores.items():
      _lowerCamelCase	:					Optional[Any]   =		na_probs[qid] > na_prob_thresh
      if pred_na:
         _lowerCamelCase	:					Any   =		float(not qid_to_has_ans[qid]   )
      else:
         _lowerCamelCase	:					List[str]   =		s
   return new_scores

def 				_snake_case      (   __snake_case							:      Any						, __snake_case							:      int						, __snake_case							:      Any=None   ):




   """simple docstring"""







   if not qid_list:
      _lowerCamelCase	:					int   =		len(__snake_case   )
      return collections.OrderedDict(
          [
              ("""exact""", 100.0 * sum(exact_scores.values()   ) / total),
              ("""f1""", 100.0 * sum(fa_scores.values()   ) / total),
              ("""total""", total),
          ]   )
   else:
      _lowerCamelCase	:					List[str]   =		len(__snake_case   )
      return collections.OrderedDict(
          [
              ("""exact""", 100.0 * sum(exact_scores[k] for k in qid_list   ) / total),
              ("""f1""", 100.0 * sum(fa_scores[k] for k in qid_list   ) / total),
              ("""total""", total),
          ]   )

def 				_snake_case      (   __snake_case							:      Optional[Any]						, __snake_case							:      Optional[Any]						, __snake_case							:      int   ):




   """simple docstring"""







   for k in new_eval:
      _lowerCamelCase	:					Optional[int]   =		new_eval[k]

def 				_snake_case      (   __snake_case							:      Tuple						, __snake_case							:      Any						, __snake_case							:      Union[str, Any]						, __snake_case							:      Tuple   ):




   """simple docstring"""







   plt.step(__snake_case						, __snake_case						, color="""b"""						, alpha=0.2						, where="""post"""   )
   plt.fill_between(__snake_case						, __snake_case						, step="""post"""						, alpha=0.2						, color="""b"""   )
   plt.xlabel("""Recall"""   )
   plt.ylabel("""Precision"""   )
   plt.xlim([0.0, 1.05]   )
   plt.ylim([0.0, 1.05]   )
   plt.title(__snake_case   )
   plt.savefig(__snake_case   )
   plt.clf()

def 				_snake_case      (   __snake_case							:      int						, __snake_case							:      Dict						, __snake_case							:      int						, __snake_case							:      Union[str, Any]						, __snake_case							:      Union[str, Any]=None						, __snake_case							:      List[Any]=None   ):




   """simple docstring"""







   _lowerCamelCase	:					Any   =		sorted(__snake_case						, key=lambda __snake_case   : na_probs[k]   )
   _lowerCamelCase	:					List[Any]   =		0.0
   _lowerCamelCase	:					Optional[Any]   =		1.0
   _lowerCamelCase	:					Tuple   =		0.0
   _lowerCamelCase	:					Tuple   =		[1.0]
   _lowerCamelCase	:					List[Any]   =		[0.0]
   _lowerCamelCase	:					Optional[int]   =		0.0
   for i, qid in enumerate(__snake_case   ):
      if qid_to_has_ans[qid]:
         true_pos += scores[qid]
      _lowerCamelCase	:					List[str]   =		true_pos / float(i + 1   )
      _lowerCamelCase	:					Optional[int]   =		true_pos / float(__snake_case   )
      if i == len(__snake_case   ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
         # i.e., if we can put a threshold after this point
         avg_prec += cur_p * (cur_r - recalls[-1])
         precisions.append(__snake_case   )
         recalls.append(__snake_case   )
   if out_image:
      plot_pr_curve(__snake_case						, __snake_case						, __snake_case						, __snake_case   )
   return {"ap": 100.0 * avg_prec}

def 				_snake_case      (   __snake_case							:      Union[str, Any]						, __snake_case							:      List[str]						, __snake_case							:      Optional[int]						, __snake_case							:      Optional[int]						, __snake_case							:      Tuple						, __snake_case							:      Optional[Any]   ):




   """simple docstring"""







   if out_image_dir and not os.path.exists(__snake_case   ):
      os.makedirs(__snake_case   )
   _lowerCamelCase	:					str   =		sum(1 for v in qid_to_has_ans.values() if v   )
   if num_true_pos == 0:
      return
   _lowerCamelCase	:					List[str]   =		make_precision_recall_eval(
       __snake_case						, __snake_case						, __snake_case						, __snake_case						, out_image=os.path.join(__snake_case						, """pr_exact.png"""   )						, title="""Precision-Recall curve for Exact Match score"""						, )
   _lowerCamelCase	:					int   =		make_precision_recall_eval(
       __snake_case						, __snake_case						, __snake_case						, __snake_case						, out_image=os.path.join(__snake_case						, """pr_f1.png"""   )						, title="""Precision-Recall curve for F1 score"""						, )
   _lowerCamelCase	:					Union[str, Any]   =		{k: float(__snake_case   ) for k, v in qid_to_has_ans.items()}
   _lowerCamelCase	:					Optional[Any]   =		make_precision_recall_eval(
       __snake_case						, __snake_case						, __snake_case						, __snake_case						, out_image=os.path.join(__snake_case						, """pr_oracle.png"""   )						, title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""						, )
   merge_eval(__snake_case						, __snake_case						, """pr_exact"""   )
   merge_eval(__snake_case						, __snake_case						, """pr_f1"""   )
   merge_eval(__snake_case						, __snake_case						, """pr_oracle"""   )

def 				_snake_case      (   __snake_case							:      Union[str, Any]						, __snake_case							:      Optional[int]						, __snake_case							:      str						, __snake_case							:      Optional[int]   ):




   """simple docstring"""







   if not qid_list:
      return
   _lowerCamelCase	:					Tuple   =		[na_probs[k] for k in qid_list]
   _lowerCamelCase	:					int   =		np.ones_like(__snake_case   ) / float(len(__snake_case   )   )
   plt.hist(__snake_case						, weights=__snake_case						, bins=20						, range=(0.0, 1.0)   )
   plt.xlabel("""Model probability of no-answer"""   )
   plt.ylabel("""Proportion of dataset"""   )
   plt.title(F'Histogram of no-answer probability: {name}'   )
   plt.savefig(os.path.join(__snake_case						, F'na_prob_hist_{name}.png'   )   )
   plt.clf()

def 				_snake_case      (   __snake_case							:      Optional[Any]						, __snake_case							:      List[Any]						, __snake_case							:      List[str]						, __snake_case							:      Any   ):




   """simple docstring"""







   _lowerCamelCase	:					int   =		sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]   )
   _lowerCamelCase	:					Optional[Any]   =		num_no_ans
   _lowerCamelCase	:					str   =		cur_score
   _lowerCamelCase	:					Dict   =		0.0
   _lowerCamelCase	:					Optional[int]   =		sorted(__snake_case						, key=lambda __snake_case   : na_probs[k]   )
   for i, qid in enumerate(__snake_case   ):
      if qid not in scores:
         continue
      if qid_to_has_ans[qid]:
         _lowerCamelCase	:					List[str]   =		scores[qid]
      else:
         if preds[qid]:
            _lowerCamelCase	:					str   =		-1
         else:
            _lowerCamelCase	:					List[str]   =		0
      cur_score += diff
      if cur_score > best_score:
         _lowerCamelCase	:					Any   =		cur_score
         _lowerCamelCase	:					Dict   =		na_probs[qid]
   return 100.0 * best_score / len(__snake_case   ), best_thresh

def 				_snake_case      (   __snake_case							:      Union[str, Any]						, __snake_case							:      str						, __snake_case							:      Union[str, Any]						, __snake_case							:      List[Any]						, __snake_case							:      str						, __snake_case							:      Dict   ):




   """simple docstring"""







   _lowerCamelCase   ,    _lowerCamelCase	:					str   =		find_best_thresh(__snake_case						, __snake_case						, __snake_case						, __snake_case   )
   _lowerCamelCase   ,    _lowerCamelCase	:					List[Any]   =		find_best_thresh(__snake_case						, __snake_case						, __snake_case						, __snake_case   )
   _lowerCamelCase	:					Dict   =		best_exact
   _lowerCamelCase	:					Optional[int]   =		exact_thresh
   _lowerCamelCase	:					Any   =		best_fa
   _lowerCamelCase	:					Any   =		fa_thresh

def 				_snake_case      (   ):




   """simple docstring"""







   with open(OPTS.data_file   ) as f:
      _lowerCamelCase	:					Union[str, Any]   =		json.load(__snake_case   )
      _lowerCamelCase	:					Dict   =		dataset_json["""data"""]
   with open(OPTS.pred_file   ) as f:
      _lowerCamelCase	:					Union[str, Any]   =		json.load(__snake_case   )
   if OPTS.na_prob_file:
      with open(OPTS.na_prob_file   ) as f:
         _lowerCamelCase	:					Any   =		json.load(__snake_case   )
   else:
      _lowerCamelCase	:					Optional[Any]   =		{k: 0.0 for k in preds}
   _lowerCamelCase	:					List[Any]   =		make_qid_to_has_ans(__snake_case   )  # maps qid to True/False
   _lowerCamelCase	:					Union[str, Any]   =		[k for k, v in qid_to_has_ans.items() if v]
   _lowerCamelCase	:					Optional[int]   =		[k for k, v in qid_to_has_ans.items() if not v]
   _lowerCamelCase   ,    _lowerCamelCase	:					Union[str, Any]   =		get_raw_scores(__snake_case						, __snake_case   )
   _lowerCamelCase	:					Dict   =		apply_no_ans_threshold(__snake_case						, __snake_case						, __snake_case						, OPTS.na_prob_thresh   )
   _lowerCamelCase	:					List[str]   =		apply_no_ans_threshold(__snake_case						, __snake_case						, __snake_case						, OPTS.na_prob_thresh   )
   _lowerCamelCase	:					Optional[Any]   =		make_eval_dict(__snake_case						, __snake_case   )
   if has_ans_qids:
      _lowerCamelCase	:					Any   =		make_eval_dict(__snake_case						, __snake_case						, qid_list=__snake_case   )
      merge_eval(__snake_case						, __snake_case						, """HasAns"""   )
   if no_ans_qids:
      _lowerCamelCase	:					List[Any]   =		make_eval_dict(__snake_case						, __snake_case						, qid_list=__snake_case   )
      merge_eval(__snake_case						, __snake_case						, """NoAns"""   )
   if OPTS.na_prob_file:
      find_all_best_thresh(__snake_case						, __snake_case						, __snake_case						, __snake_case						, __snake_case						, __snake_case   )
   if OPTS.na_prob_file and OPTS.out_image_dir:
      run_precision_recall_analysis(__snake_case						, __snake_case						, __snake_case						, __snake_case						, __snake_case						, OPTS.out_image_dir   )
      histogram_na_prob(__snake_case						, __snake_case						, OPTS.out_image_dir						, """hasAns"""   )
      histogram_na_prob(__snake_case						, __snake_case						, OPTS.out_image_dir						, """noAns"""   )
   if OPTS.out_file:
      with open(OPTS.out_file						, """w"""   ) as f:
         json.dump(__snake_case						, __snake_case   )
   else:
      print(json.dumps(__snake_case						, indent=2   )   )


if __name__ == "__main__":
 UpperCAmelCase				=						parse_args()
 if OPTS.out_image_dir:
  import matplotlib

  matplotlib.use("""Agg""")
  import matplotlib.pyplot as plt
 main()








import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp





def  __magic_name__  (				)				-> str:

							"""simple docstring"""



							lowercase_	:   Optional[int]  				=     ArgumentParser(
							    description=(
							        """PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes"""
							    )				)

							# Optional arguments for the launch helper
							parser.add_argument("""--num_cores"""				,       type=lowercase				,       default=1				,       help="""Number of TPU cores to use (1 or 8)."""				)

							# positional
							parser.add_argument(
							    """training_script"""				,       type=lowercase				,       help=(
							        """The full path to the single TPU training """
							        """program/script to be launched in parallel, """
							        """followed by all the arguments for the """
							        """training script"""
							    )				,       )

							# rest from the training program
							parser.add_argument("""training_script_args"""				,       nargs=lowercase				)

							return parser.parse_args()





def  __magic_name__  (				)				-> List[Any]:

							"""simple docstring"""



							lowercase_	:   Union[str, Any]  				=     parse_args()

							# Import training_script as a module.
							lowercase_	:   List[Any]  				=     Path(args.training_script				)
							sys.path.append(str(script_fpath.parent.resolve()				)				)
							lowercase_	:   Tuple  				=     script_fpath.stem
							lowercase_	:   List[str]  				=     importlib.import_module(lowercase				)

							# Patch sys.argv
							lowercase_	:   Optional[int]  				=     [args.training_script] + args.training_script_args + ["""--tpu_num_cores""", str(args.num_cores				)]

							xmp.spawn(mod._mp_fn				,       args=()				,       nprocs=args.num_cores				)


if __name__ == "__main__":
							main()