






import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCamelCase__       		=  re.compile(R"""\b(a|an|the)\b""", re.UNICODE)

lowerCamelCase__       		=  None






def 	UpperCamelCase	(							):




     '''simple docstring'''






     __snake_case      :str 							=    argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0."""	)
     parser.add_argument("""data_file"""						,metavar="""data.json"""						,help="""Input data JSON file."""	)
     parser.add_argument("""pred_file"""						,metavar="""pred.json"""						,help="""Model predictions."""	)
     parser.add_argument(
         """--out-file"""						,"""-o"""						,metavar="""eval.json"""						,help="""Write accuracy metrics to file (default is stdout)."""	)
     parser.add_argument(
         """--na-prob-file"""						,"""-n"""						,metavar="""na_prob.json"""						,help="""Model estimates of probability of no answer."""	)
     parser.add_argument(
         """--na-prob-thresh"""						,"""-t"""						,type=snake_case__						,default=1.0						,help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""						,)
     parser.add_argument(
         """--out-image-dir"""						,"""-p"""						,metavar="""out_images"""						,default=snake_case__						,help="""Save precision-recall curves to directory."""	)
     parser.add_argument("""--verbose"""						,"""-v"""						,action="""store_true"""	)
     if len(sys.argv	) == 1:
          parser.print_help()
          sys.exit(1	)
     return parser.parse_args()






def 	UpperCamelCase	(							snake_case__  : Optional[Any]	):




     '''simple docstring'''






     __snake_case      :Union[str, Any] 							=    {}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    __snake_case      :Union[str, Any] 							=    bool(qa["""answers"""]["""text"""]	)
     return qid_to_has_ans






def 	UpperCamelCase	(							snake_case__  : Optional[int]	):




     '''simple docstring'''






     def remove_articles(snake_case__  : Tuple	):
          return ARTICLES_REGEX.sub(""" """						,snake_case__	)

     def white_space_fix(snake_case__  : List[str]	):
          return " ".join(text.split()	)

     def remove_punc(snake_case__  : int	):
          __snake_case      :Optional[Any] 							=    set(string.punctuation	)
          return "".join(ch for ch in text if ch not in exclude	)

     def lower(snake_case__  : List[str]	):
          return text.lower()

     return white_space_fix(remove_articles(remove_punc(lower(snake_case__	)	)	)	)






def 	UpperCamelCase	(							snake_case__  : Optional[int]	):




     '''simple docstring'''






     if not s:
          return []
     return normalize_answer(snake_case__	).split()






def 	UpperCamelCase	(							snake_case__  : Optional[Any]						,snake_case__  : Tuple	):




     '''simple docstring'''






     return int(normalize_answer(snake_case__	) == normalize_answer(snake_case__	)	)






def 	UpperCamelCase	(							snake_case__  : int						,snake_case__  : Any	):




     '''simple docstring'''






     __snake_case      :List[str] 							=    get_tokens(snake_case__	)
     __snake_case      :int 							=    get_tokens(snake_case__	)
     __snake_case      :Optional[int] 							=    collections.Counter(snake_case__	) & collections.Counter(snake_case__	)
     __snake_case      :Dict 							=    sum(common.values()	)
     if len(snake_case__	) == 0 or len(snake_case__	) == 0:
          # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
          return int(gold_toks == pred_toks	)
     if num_same == 0:
          return 0
     __snake_case      :List[str] 							=    1.0 * num_same / len(snake_case__	)
     __snake_case      :int 							=    1.0 * num_same / len(snake_case__	)
     __snake_case      :Optional[int] 							=    (2 * precision * recall) / (precision + recall)
     return fa






def 	UpperCamelCase	(							snake_case__  : str						,snake_case__  : Any	):




     '''simple docstring'''






     __snake_case      :Any 							=    {}
     __snake_case      :Union[str, Any] 							=    {}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    __snake_case      :Optional[Any] 							=    qa["""id"""]
                    __snake_case      :Any 							=    [t for t in qa["""answers"""]["""text"""] if normalize_answer(snake_case__	)]
                    if not gold_answers:
                         # For unanswerable questions, only correct answer is empty string
                         __snake_case      :Tuple 							=    [""""""]
                    if qid not in preds:
                         print(f'''Missing prediction for {qid}'''	)
                         continue
                    __snake_case      :Tuple 							=    preds[qid]
                    # Take max over all gold answers
                    __snake_case      :Tuple 							=    max(compute_exact(snake_case__						,snake_case__	) for a in gold_answers	)
                    __snake_case      :int 							=    max(compute_fa(snake_case__						,snake_case__	) for a in gold_answers	)
     return exact_scores, fa_scores






def 	UpperCamelCase	(							snake_case__  : Optional[Any]						,snake_case__  : Dict						,snake_case__  : Optional[Any]						,snake_case__  : List[Any]	):




     '''simple docstring'''






     __snake_case      :List[Any] 							=    {}
     for qid, s in scores.items():
          __snake_case      :Union[str, Any] 							=    na_probs[qid] > na_prob_thresh
          if pred_na:
               __snake_case      :Tuple 							=    float(not qid_to_has_ans[qid]	)
          else:
               __snake_case      :Any 							=    s
     return new_scores






def 	UpperCamelCase	(							snake_case__  : Optional[Any]						,snake_case__  : Dict						,snake_case__  : List[str]=None	):




     '''simple docstring'''






     if not qid_list:
          __snake_case      :Optional[int] 							=    len(snake_case__	)
          return collections.OrderedDict(
              [
                  ("""exact""", 1_0_0.0 * sum(exact_scores.values()	) / total),
                  ("""f1""", 1_0_0.0 * sum(fa_scores.values()	) / total),
                  ("""total""", total),
              ]	)
     else:
          __snake_case      :Tuple 							=    len(snake_case__	)
          return collections.OrderedDict(
              [
                  ("""exact""", 1_0_0.0 * sum(exact_scores[k] for k in qid_list	) / total),
                  ("""f1""", 1_0_0.0 * sum(fa_scores[k] for k in qid_list	) / total),
                  ("""total""", total),
              ]	)






def 	UpperCamelCase	(							snake_case__  : List[str]						,snake_case__  : Any						,snake_case__  : Optional[Any]	):




     '''simple docstring'''






     for k in new_eval:
          __snake_case      :List[str] 							=    new_eval[k]






def 	UpperCamelCase	(							snake_case__  : str						,snake_case__  : Any						,snake_case__  : Any						,snake_case__  : List[Any]	):




     '''simple docstring'''






     plt.step(snake_case__						,snake_case__						,color="""b"""						,alpha=0.2						,where="""post"""	)
     plt.fill_between(snake_case__						,snake_case__						,step="""post"""						,alpha=0.2						,color="""b"""	)
     plt.xlabel("""Recall"""	)
     plt.ylabel("""Precision"""	)
     plt.xlim([0.0, 1.0_5]	)
     plt.ylim([0.0, 1.0_5]	)
     plt.title(snake_case__	)
     plt.savefig(snake_case__	)
     plt.clf()






def 	UpperCamelCase	(							snake_case__  : int						,snake_case__  : int						,snake_case__  : int						,snake_case__  : Tuple						,snake_case__  : Union[str, Any]=None						,snake_case__  : Any=None	):




     '''simple docstring'''






     __snake_case      :str 							=    sorted(snake_case__						,key=lambda snake_case__	: na_probs[k]	)
     __snake_case      :Tuple 							=    0.0
     __snake_case      :List[Any] 							=    1.0
     __snake_case      :Optional[Any] 							=    0.0
     __snake_case      :Any 							=    [1.0]
     __snake_case      :Union[str, Any] 							=    [0.0]
     __snake_case      :List[Any] 							=    0.0
     for i, qid in enumerate(snake_case__	):
          if qid_to_has_ans[qid]:
               true_pos += scores[qid]
          __snake_case      :Any 							=    true_pos / float(i + 1	)
          __snake_case      :Dict 							=    true_pos / float(snake_case__	)
          if i == len(snake_case__	) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
               # i.e., if we can put a threshold after this point
               avg_prec += cur_p * (cur_r - recalls[-1])
               precisions.append(snake_case__	)
               recalls.append(snake_case__	)
     if out_image:
          plot_pr_curve(snake_case__						,snake_case__						,snake_case__						,snake_case__	)
     return {"ap": 1_0_0.0 * avg_prec}






def 	UpperCamelCase	(							snake_case__  : List[Any]						,snake_case__  : Tuple						,snake_case__  : int						,snake_case__  : List[str]						,snake_case__  : str						,snake_case__  : Optional[int]	):




     '''simple docstring'''






     if out_image_dir and not os.path.exists(snake_case__	):
          os.makedirs(snake_case__	)
     __snake_case      :Union[str, Any] 							=    sum(1 for v in qid_to_has_ans.values() if v	)
     if num_true_pos == 0:
          return
     __snake_case      :Tuple 							=    make_precision_recall_eval(
         snake_case__						,snake_case__						,snake_case__						,snake_case__						,out_image=os.path.join(snake_case__						,"""pr_exact.png"""	)						,title="""Precision-Recall curve for Exact Match score"""						,)
     __snake_case      :Any 							=    make_precision_recall_eval(
         snake_case__						,snake_case__						,snake_case__						,snake_case__						,out_image=os.path.join(snake_case__						,"""pr_f1.png"""	)						,title="""Precision-Recall curve for F1 score"""						,)
     __snake_case      :Tuple 							=    {k: float(snake_case__	) for k, v in qid_to_has_ans.items()}
     __snake_case      :int 							=    make_precision_recall_eval(
         snake_case__						,snake_case__						,snake_case__						,snake_case__						,out_image=os.path.join(snake_case__						,"""pr_oracle.png"""	)						,title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""						,)
     merge_eval(snake_case__						,snake_case__						,"""pr_exact"""	)
     merge_eval(snake_case__						,snake_case__						,"""pr_f1"""	)
     merge_eval(snake_case__						,snake_case__						,"""pr_oracle"""	)






def 	UpperCamelCase	(							snake_case__  : str						,snake_case__  : Any						,snake_case__  : List[str]						,snake_case__  : Optional[Any]	):




     '''simple docstring'''






     if not qid_list:
          return
     __snake_case      :int 							=    [na_probs[k] for k in qid_list]
     __snake_case      :List[Any] 							=    np.ones_like(snake_case__	) / float(len(snake_case__	)	)
     plt.hist(snake_case__						,weights=snake_case__						,bins=20						,range=(0.0, 1.0)	)
     plt.xlabel("""Model probability of no-answer"""	)
     plt.ylabel("""Proportion of dataset"""	)
     plt.title(f'''Histogram of no-answer probability: {name}'''	)
     plt.savefig(os.path.join(snake_case__						,f'''na_prob_hist_{name}.png'''	)	)
     plt.clf()






def 	UpperCamelCase	(							snake_case__  : str						,snake_case__  : str						,snake_case__  : Union[str, Any]						,snake_case__  : Optional[int]	):




     '''simple docstring'''






     __snake_case      :Dict 							=    sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]	)
     __snake_case      :List[str] 							=    num_no_ans
     __snake_case      :Union[str, Any] 							=    cur_score
     __snake_case      :Any 							=    0.0
     __snake_case      :Dict 							=    sorted(snake_case__						,key=lambda snake_case__	: na_probs[k]	)
     for i, qid in enumerate(snake_case__	):
          if qid not in scores:
               continue
          if qid_to_has_ans[qid]:
               __snake_case      :List[Any] 							=    scores[qid]
          else:
               if preds[qid]:
                    __snake_case      :Union[str, Any] 							=    -1
               else:
                    __snake_case      :List[Any] 							=    0
          cur_score += diff
          if cur_score > best_score:
               __snake_case      :Union[str, Any] 							=    cur_score
               __snake_case      :Dict 							=    na_probs[qid]
     return 1_0_0.0 * best_score / len(snake_case__	), best_thresh






def 	UpperCamelCase	(							snake_case__  : Dict						,snake_case__  : str						,snake_case__  : Optional[Any]						,snake_case__  : Optional[Any]						,snake_case__  : Optional[Any]						,snake_case__  : Any	):




     '''simple docstring'''






     __snake_case							,			__snake_case      :Dict 							=    find_best_thresh(snake_case__						,snake_case__						,snake_case__						,snake_case__	)
     __snake_case							,			__snake_case      :Optional[int] 							=    find_best_thresh(snake_case__						,snake_case__						,snake_case__						,snake_case__	)
     __snake_case      :Dict 							=    best_exact
     __snake_case      :List[str] 							=    exact_thresh
     __snake_case      :str 							=    best_fa
     __snake_case      :int 							=    fa_thresh






def 	UpperCamelCase	(							):




     '''simple docstring'''






     with open(OPTS.data_file	) as f:
          __snake_case      :Any 							=    json.load(snake_case__	)
          __snake_case      :List[Any] 							=    dataset_json["""data"""]
     with open(OPTS.pred_file	) as f:
          __snake_case      :List[Any] 							=    json.load(snake_case__	)
     if OPTS.na_prob_file:
          with open(OPTS.na_prob_file	) as f:
               __snake_case      :Dict 							=    json.load(snake_case__	)
     else:
          __snake_case      :Union[str, Any] 							=    {k: 0.0 for k in preds}
     __snake_case      :Union[str, Any] 							=    make_qid_to_has_ans(snake_case__	)  # maps qid to True/False
     __snake_case      :int 							=    [k for k, v in qid_to_has_ans.items() if v]
     __snake_case      :Tuple 							=    [k for k, v in qid_to_has_ans.items() if not v]
     __snake_case							,			__snake_case      :List[Any] 							=    get_raw_scores(snake_case__						,snake_case__	)
     __snake_case      :Optional[int] 							=    apply_no_ans_threshold(snake_case__						,snake_case__						,snake_case__						,OPTS.na_prob_thresh	)
     __snake_case      :int 							=    apply_no_ans_threshold(snake_case__						,snake_case__						,snake_case__						,OPTS.na_prob_thresh	)
     __snake_case      :Optional[int] 							=    make_eval_dict(snake_case__						,snake_case__	)
     if has_ans_qids:
          __snake_case      :Optional[Any] 							=    make_eval_dict(snake_case__						,snake_case__						,qid_list=snake_case__	)
          merge_eval(snake_case__						,snake_case__						,"""HasAns"""	)
     if no_ans_qids:
          __snake_case      :Optional[Any] 							=    make_eval_dict(snake_case__						,snake_case__						,qid_list=snake_case__	)
          merge_eval(snake_case__						,snake_case__						,"""NoAns"""	)
     if OPTS.na_prob_file:
          find_all_best_thresh(snake_case__						,snake_case__						,snake_case__						,snake_case__						,snake_case__						,snake_case__	)
     if OPTS.na_prob_file and OPTS.out_image_dir:
          run_precision_recall_analysis(snake_case__						,snake_case__						,snake_case__						,snake_case__						,snake_case__						,OPTS.out_image_dir	)
          histogram_na_prob(snake_case__						,snake_case__						,OPTS.out_image_dir						,"""hasAns"""	)
          histogram_na_prob(snake_case__						,snake_case__						,OPTS.out_image_dir						,"""noAns"""	)
     if OPTS.out_file:
          with open(OPTS.out_file						,"""w"""	) as f:
               json.dump(snake_case__						,snake_case__	)
     else:
          print(json.dumps(snake_case__						,indent=2	)	)


if __name__ == "__main__":
     lowerCamelCase__       		=  parse_args()
     if OPTS.out_image_dir:
          import matplotlib

          matplotlib.use("""Agg""")
          import matplotlib.pyplot as plt
     main()










# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
     import torch




class       snake_case__ (				TensorFormatter[Mapping, "torch.Tensor", Mapping]):

       '''simple docstring'''



       def __init__(				self      ,    a__=None      ,    **a__     )			->   Tuple:






            '''simple docstring'''





            super().__init__(features=a__     )
            __snake_case      :int 							=    torch_tensor_kwargs
            import torch  # noqa import torch at initialization



       def        __lowercase    (				self      ,    a__     )			->   List[str]:






            '''simple docstring'''





            import torch

            if isinstance(a__      ,    a__     ) and column:
                 if all(
                     isinstance(a__      ,    torch.Tensor     ) and x.shape == column[0].shape and x.dtype == column[0].dtype
                     for x in column     ):
                      return torch.stack(a__     )
            return column



       def        __lowercase    (				self      ,    a__     )			->   Union[str, Any]:






            '''simple docstring'''





            import torch

            if isinstance(a__      ,    (str, bytes, type(a__     ))     ):
                 return value
            elif isinstance(a__      ,    (np.character, np.ndarray)     ) and np.issubdtype(value.dtype      ,    np.character     ):
                 return value.tolist()

            __snake_case      :Optional[int] 							=    {}

            if isinstance(a__      ,    (np.number, np.ndarray)     ) and np.issubdtype(value.dtype      ,    np.integer     ):
                 __snake_case      :List[Any] 							=    {"""dtype""": torch.intaa}
            elif isinstance(a__      ,    (np.number, np.ndarray)     ) and np.issubdtype(value.dtype      ,    np.floating     ):
                 __snake_case      :Tuple 							=    {"""dtype""": torch.floataa}
            elif config.PIL_AVAILABLE and "PIL" in sys.modules:
                 import PIL.Image

                 if isinstance(a__      ,    PIL.Image.Image     ):
                      __snake_case      :Union[str, Any] 							=    np.asarray(a__     )
            return torch.tensor(a__      ,    **{**default_dtype, **self.torch_tensor_kwargs}     )



       def        __lowercase    (				self      ,    a__     )			->   str:






            '''simple docstring'''





            import torch

            # support for torch, tf, jax etc.
            if hasattr(a__      ,    """__array__"""     ) and not isinstance(a__      ,    torch.Tensor     ):
                 __snake_case      :Optional[Any] 							=    data_struct.__array__()
            # support for nested types like struct of list of struct
            if isinstance(a__      ,    np.ndarray     ):
                 if data_struct.dtype == object:  # torch tensors cannot be instantied from an array of objects
                      return self._consolidate([self.recursive_tensorize(a__     ) for substruct in data_struct]     )
            elif isinstance(a__      ,    (list, tuple)     ):
                 return self._consolidate([self.recursive_tensorize(a__     ) for substruct in data_struct]     )
            return self._tensorize(a__     )



       def        __lowercase    (				self      ,    a__     )			->   Union[str, Any]:






            '''simple docstring'''





            return map_nested(self._recursive_tensorize      ,    a__      ,    map_list=a__     )



       def        __lowercase    (				self      ,    a__     )			->   Mapping:






            '''simple docstring'''





            __snake_case      :Tuple 							=    self.numpy_arrow_extractor().extract_row(a__     )
            __snake_case      :Any 							=    self.python_features_decoder.decode_row(a__     )
            return self.recursive_tensorize(a__     )



       def        __lowercase    (				self      ,    a__     )			->   "torch.Tensor":






            '''simple docstring'''





            __snake_case      :List[str] 							=    self.numpy_arrow_extractor().extract_column(a__     )
            __snake_case      :List[Any] 							=    self.python_features_decoder.decode_column(a__      ,    pa_table.column_names[0]     )
            __snake_case      :Optional[Any] 							=    self.recursive_tensorize(a__     )
            __snake_case      :Any 							=    self._consolidate(a__     )
            return column



       def        __lowercase    (				self      ,    a__     )			->   Mapping:






            '''simple docstring'''





            __snake_case      :Optional[int] 							=    self.numpy_arrow_extractor().extract_batch(a__     )
            __snake_case      :Tuple 							=    self.python_features_decoder.decode_batch(a__     )
            __snake_case      :Tuple 							=    self.recursive_tensorize(a__     )
            for column_name in batch:
                 __snake_case      :str 							=    self._consolidate(batch[column_name]     )
            return batch


