


'''simple docstring'''

import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowercase__				     =		re.compile(R"\b(a|an|the)\b", re.UNICODE)

lowercase__				     =		None






def __UpperCamelCase   (			)      ->     Union[str, Any]:







     '''simple docstring'''






     _a						=		argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."				)
     parser.add_argument("data_file"      ,  metavar="data.json"      ,  help="Input data JSON file."				)
     parser.add_argument("pred_file"      ,  metavar="pred.json"      ,  help="Model predictions."				)
     parser.add_argument(
         "--out-file"      ,  "-o"      ,  metavar="eval.json"      ,  help="Write accuracy metrics to file (default is stdout)."				)
     parser.add_argument(
         "--na-prob-file"      ,  "-n"      ,  metavar="na_prob.json"      ,  help="Model estimates of probability of no answer."				)
     parser.add_argument(
         "--na-prob-thresh"      ,  "-t"      ,  type=__lowerCamelCase      ,  default=1.0      ,  help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."      ,  )
     parser.add_argument(
         "--out-image-dir"      ,  "-p"      ,  metavar="out_images"      ,  default=__lowerCamelCase      ,  help="Save precision-recall curves to directory."				)
     parser.add_argument("--verbose"      ,  "-v"      ,  action="store_true"				)
     if len(sys.argv				) == 1:
          parser.print_help()
          sys.exit(1				)
     return parser.parse_args()






def __UpperCamelCase   (			__lowerCamelCase		: Optional[Any]				)      ->     Optional[Any]:







     '''simple docstring'''






     _a						=		{}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    _a						=		bool(qa["answers"]["text"]				)
     return qid_to_has_ans






def __UpperCamelCase   (			__lowerCamelCase		: List[Any]				)      ->     Union[str, Any]:







     '''simple docstring'''






     def remove_articles(__lowerCamelCase		: Optional[int]				):
          return ARTICLES_REGEX.sub(" "      ,  __lowerCamelCase				)

     def white_space_fix(__lowerCamelCase		: Tuple				):
          return " ".join(text.split()				)

     def remove_punc(__lowerCamelCase		: str				):
          _a						=		set(string.punctuation				)
          return "".join(ch for ch in text if ch not in exclude				)

     def lower(__lowerCamelCase		: Optional[Any]				):
          return text.lower()

     return white_space_fix(remove_articles(remove_punc(lower(__lowerCamelCase				)				)				)				)






def __UpperCamelCase   (			__lowerCamelCase		: Tuple				)      ->     List[Any]:







     '''simple docstring'''






     if not s:
          return []
     return normalize_answer(__lowerCamelCase				).split()






def __UpperCamelCase   (			__lowerCamelCase		: int      ,  __lowerCamelCase		: List[str]				)      ->     Tuple:







     '''simple docstring'''






     return int(normalize_answer(__lowerCamelCase				) == normalize_answer(__lowerCamelCase				)				)






def __UpperCamelCase   (			__lowerCamelCase		: Tuple      ,  __lowerCamelCase		: Optional[Any]				)      ->     int:







     '''simple docstring'''






     _a						=		get_tokens(__lowerCamelCase				)
     _a						=		get_tokens(__lowerCamelCase				)
     _a						=		collections.Counter(__lowerCamelCase				) & collections.Counter(__lowerCamelCase				)
     _a						=		sum(common.values()				)
     if len(__lowerCamelCase				) == 0 or len(__lowerCamelCase				) == 0:
          # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
          return int(gold_toks == pred_toks				)
     if num_same == 0:
          return 0
     _a						=		1.0 * num_same / len(__lowerCamelCase				)
     _a						=		1.0 * num_same / len(__lowerCamelCase				)
     _a						=		(2 * precision * recall) / (precision + recall)
     return fa






def __UpperCamelCase   (			__lowerCamelCase		: Union[str, Any]      ,  __lowerCamelCase		: List[Any]				)      ->     List[Any]:







     '''simple docstring'''






     _a						=		{}
     _a						=		{}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    _a						=		qa["id"]
                    _a						=		[t for t in qa["answers"]["text"] if normalize_answer(__lowerCamelCase				)]
                    if not gold_answers:
                         # For unanswerable questions, only correct answer is empty string
                         _a						=		[""]
                    if qid not in preds:
                         print(F"Missing prediction for {qid}"				)
                         continue
                    _a						=		preds[qid]
                    # Take max over all gold answers
                    _a						=		max(compute_exact(__lowerCamelCase      ,  __lowerCamelCase				) for a in gold_answers				)
                    _a						=		max(compute_fa(__lowerCamelCase      ,  __lowerCamelCase				) for a in gold_answers				)
     return exact_scores, fa_scores






def __UpperCamelCase   (			__lowerCamelCase		: Optional[Any]      ,  __lowerCamelCase		: Optional[int]      ,  __lowerCamelCase		: Optional[int]      ,  __lowerCamelCase		: Union[str, Any]				)      ->     Union[str, Any]:







     '''simple docstring'''






     _a						=		{}
     for qid, s in scores.items():
          _a						=		na_probs[qid] > na_prob_thresh
          if pred_na:
               _a						=		float(not qid_to_has_ans[qid]				)
          else:
               _a						=		s
     return new_scores






def __UpperCamelCase   (			__lowerCamelCase		: Tuple      ,  __lowerCamelCase		: List[str]      ,  __lowerCamelCase		: Any=None				)      ->     str:







     '''simple docstring'''






     if not qid_list:
          _a						=		len(__lowerCamelCase				)
          return collections.OrderedDict(
              [
                  ("exact", 1_00.0 * sum(exact_scores.values()				) / total),
                  ("f1", 1_00.0 * sum(fa_scores.values()				) / total),
                  ("total", total),
              ]				)
     else:
          _a						=		len(__lowerCamelCase				)
          return collections.OrderedDict(
              [
                  ("exact", 1_00.0 * sum(exact_scores[k] for k in qid_list				) / total),
                  ("f1", 1_00.0 * sum(fa_scores[k] for k in qid_list				) / total),
                  ("total", total),
              ]				)






def __UpperCamelCase   (			__lowerCamelCase		: int      ,  __lowerCamelCase		: List[str]      ,  __lowerCamelCase		: Tuple				)      ->     Optional[int]:







     '''simple docstring'''






     for k in new_eval:
          _a						=		new_eval[k]






def __UpperCamelCase   (			__lowerCamelCase		: str      ,  __lowerCamelCase		: Tuple      ,  __lowerCamelCase		: List[Any]      ,  __lowerCamelCase		: Union[str, Any]				)      ->     Tuple:







     '''simple docstring'''






     plt.step(__lowerCamelCase      ,  __lowerCamelCase      ,  color="b"      ,  alpha=0.2      ,  where="post"				)
     plt.fill_between(__lowerCamelCase      ,  __lowerCamelCase      ,  step="post"      ,  alpha=0.2      ,  color="b"				)
     plt.xlabel("Recall"				)
     plt.ylabel("Precision"				)
     plt.xlim([0.0, 1.05]				)
     plt.ylim([0.0, 1.05]				)
     plt.title(__lowerCamelCase				)
     plt.savefig(__lowerCamelCase				)
     plt.clf()






def __UpperCamelCase   (			__lowerCamelCase		: Dict      ,  __lowerCamelCase		: Dict      ,  __lowerCamelCase		: Optional[Any]      ,  __lowerCamelCase		: Any      ,  __lowerCamelCase		: int=None      ,  __lowerCamelCase		: int=None				)      ->     Optional[int]:







     '''simple docstring'''






     _a						=		sorted(__lowerCamelCase      ,  key=lambda __lowerCamelCase				: na_probs[k]				)
     _a						=		0.0
     _a						=		1.0
     _a						=		0.0
     _a						=		[1.0]
     _a						=		[0.0]
     _a						=		0.0
     for i, qid in enumerate(__lowerCamelCase				):
          if qid_to_has_ans[qid]:
               true_pos += scores[qid]
          _a						=		true_pos / float(i + 1				)
          _a						=		true_pos / float(__lowerCamelCase				)
          if i == len(__lowerCamelCase				) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
               # i.e., if we can put a threshold after this point
               avg_prec += cur_p * (cur_r - recalls[-1])
               precisions.append(__lowerCamelCase				)
               recalls.append(__lowerCamelCase				)
     if out_image:
          plot_pr_curve(__lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase				)
     return {"ap": 1_00.0 * avg_prec}






def __UpperCamelCase   (			__lowerCamelCase		: int      ,  __lowerCamelCase		: Dict      ,  __lowerCamelCase		: int      ,  __lowerCamelCase		: int      ,  __lowerCamelCase		: Tuple      ,  __lowerCamelCase		: Tuple				)      ->     Optional[Any]:







     '''simple docstring'''






     if out_image_dir and not os.path.exists(__lowerCamelCase				):
          os.makedirs(__lowerCamelCase				)
     _a						=		sum(1 for v in qid_to_has_ans.values() if v				)
     if num_true_pos == 0:
          return
     _a						=		make_precision_recall_eval(
         __lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  out_image=os.path.join(__lowerCamelCase      ,  "pr_exact.png"				)      ,  title="Precision-Recall curve for Exact Match score"      ,  )
     _a						=		make_precision_recall_eval(
         __lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  out_image=os.path.join(__lowerCamelCase      ,  "pr_f1.png"				)      ,  title="Precision-Recall curve for F1 score"      ,  )
     _a						=		{k: float(__lowerCamelCase				) for k, v in qid_to_has_ans.items()}
     _a						=		make_precision_recall_eval(
         __lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  out_image=os.path.join(__lowerCamelCase      ,  "pr_oracle.png"				)      ,  title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"      ,  )
     merge_eval(__lowerCamelCase      ,  __lowerCamelCase      ,  "pr_exact"				)
     merge_eval(__lowerCamelCase      ,  __lowerCamelCase      ,  "pr_f1"				)
     merge_eval(__lowerCamelCase      ,  __lowerCamelCase      ,  "pr_oracle"				)






def __UpperCamelCase   (			__lowerCamelCase		: Any      ,  __lowerCamelCase		: Union[str, Any]      ,  __lowerCamelCase		: List[str]      ,  __lowerCamelCase		: List[str]				)      ->     Any:







     '''simple docstring'''






     if not qid_list:
          return
     _a						=		[na_probs[k] for k in qid_list]
     _a						=		np.ones_like(__lowerCamelCase				) / float(len(__lowerCamelCase				)				)
     plt.hist(__lowerCamelCase      ,  weights=__lowerCamelCase      ,  bins=20      ,  range=(0.0, 1.0)				)
     plt.xlabel("Model probability of no-answer"				)
     plt.ylabel("Proportion of dataset"				)
     plt.title(F"Histogram of no-answer probability: {name}"				)
     plt.savefig(os.path.join(__lowerCamelCase      ,  F"na_prob_hist_{name}.png"				)				)
     plt.clf()






def __UpperCamelCase   (			__lowerCamelCase		: int      ,  __lowerCamelCase		: int      ,  __lowerCamelCase		: int      ,  __lowerCamelCase		: Tuple				)      ->     int:







     '''simple docstring'''






     _a						=		sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]				)
     _a						=		num_no_ans
     _a						=		cur_score
     _a						=		0.0
     _a						=		sorted(__lowerCamelCase      ,  key=lambda __lowerCamelCase				: na_probs[k]				)
     for i, qid in enumerate(__lowerCamelCase				):
          if qid not in scores:
               continue
          if qid_to_has_ans[qid]:
               _a						=		scores[qid]
          else:
               if preds[qid]:
                    _a						=		-1
               else:
                    _a						=		0
          cur_score += diff
          if cur_score > best_score:
               _a						=		cur_score
               _a						=		na_probs[qid]
     return 1_00.0 * best_score / len(__lowerCamelCase				), best_thresh






def __UpperCamelCase   (			__lowerCamelCase		: int      ,  __lowerCamelCase		: Any      ,  __lowerCamelCase		: int      ,  __lowerCamelCase		: int      ,  __lowerCamelCase		: Any      ,  __lowerCamelCase		: Optional[int]				)      ->     List[Any]:







     '''simple docstring'''






     _a			,			_a						=		find_best_thresh(__lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase				)
     _a			,			_a						=		find_best_thresh(__lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase				)
     _a						=		best_exact
     _a						=		exact_thresh
     _a						=		best_fa
     _a						=		fa_thresh






def __UpperCamelCase   (			)      ->     Optional[Any]:







     '''simple docstring'''






     with open(OPTS.data_file				) as f:
          _a						=		json.load(__lowerCamelCase				)
          _a						=		dataset_json["data"]
     with open(OPTS.pred_file				) as f:
          _a						=		json.load(__lowerCamelCase				)
     if OPTS.na_prob_file:
          with open(OPTS.na_prob_file				) as f:
               _a						=		json.load(__lowerCamelCase				)
     else:
          _a						=		{k: 0.0 for k in preds}
     _a						=		make_qid_to_has_ans(__lowerCamelCase				)  # maps qid to True/False
     _a						=		[k for k, v in qid_to_has_ans.items() if v]
     _a						=		[k for k, v in qid_to_has_ans.items() if not v]
     _a			,			_a						=		get_raw_scores(__lowerCamelCase      ,  __lowerCamelCase				)
     _a						=		apply_no_ans_threshold(__lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  OPTS.na_prob_thresh				)
     _a						=		apply_no_ans_threshold(__lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  OPTS.na_prob_thresh				)
     _a						=		make_eval_dict(__lowerCamelCase      ,  __lowerCamelCase				)
     if has_ans_qids:
          _a						=		make_eval_dict(__lowerCamelCase      ,  __lowerCamelCase      ,  qid_list=__lowerCamelCase				)
          merge_eval(__lowerCamelCase      ,  __lowerCamelCase      ,  "HasAns"				)
     if no_ans_qids:
          _a						=		make_eval_dict(__lowerCamelCase      ,  __lowerCamelCase      ,  qid_list=__lowerCamelCase				)
          merge_eval(__lowerCamelCase      ,  __lowerCamelCase      ,  "NoAns"				)
     if OPTS.na_prob_file:
          find_all_best_thresh(__lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase				)
     if OPTS.na_prob_file and OPTS.out_image_dir:
          run_precision_recall_analysis(__lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  __lowerCamelCase      ,  OPTS.out_image_dir				)
          histogram_na_prob(__lowerCamelCase      ,  __lowerCamelCase      ,  OPTS.out_image_dir      ,  "hasAns"				)
          histogram_na_prob(__lowerCamelCase      ,  __lowerCamelCase      ,  OPTS.out_image_dir      ,  "noAns"				)
     if OPTS.out_file:
          with open(OPTS.out_file      ,  "w"				) as f:
               json.dump(__lowerCamelCase      ,  __lowerCamelCase				)
     else:
          print(json.dumps(__lowerCamelCase      ,  indent=2				)				)


if __name__ == "__main__":
   lowercase__				     =		parse_args()
   if OPTS.out_image_dir:
      import matplotlib

      matplotlib.use("Agg")
      import matplotlib.pyplot as plt
   main()





'''simple docstring'''

from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowercase__				     =		{
    "configuration_mctct": ["MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP", "MCTCTConfig"],
    "feature_extraction_mctct": ["MCTCTFeatureExtractor"],
    "processing_mctct": ["MCTCTProcessor"],
}


try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   lowercase__				     =		[
       "MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST",
       "MCTCTForCTC",
       "MCTCTModel",
       "MCTCTPreTrainedModel",
   ]


if TYPE_CHECKING:
   from .configuration_mctct import MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP, MCTCTConfig
   from .feature_extraction_mctct import MCTCTFeatureExtractor
   from .processing_mctct import MCTCTProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_mctct import MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST, MCTCTForCTC, MCTCTModel, MCTCTPreTrainedModel

else:
   import sys

   lowercase__				     =		_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

