



"""simple docstring"""





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCamelCase  			=		re.compile(r"""\b(a|an|the)\b""", re.UNICODE)

lowerCamelCase  			=		None
def   a__      ( ):
      UpperCAmelCase_     					=				argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0." )
      parser.add_argument("data_file"      ,     metavar="data.json"      ,     help="Input data JSON file." )
      parser.add_argument("pred_file"      ,     metavar="pred.json"      ,     help="Model predictions." )
      parser.add_argument(
          "--out-file"      ,     "-o"      ,     metavar="eval.json"      ,     help="Write accuracy metrics to file (default is stdout)." )
      parser.add_argument(
          "--na-prob-file"      ,     "-n"      ,     metavar="na_prob.json"      ,     help="Model estimates of probability of no answer." )
      parser.add_argument(
          "--na-prob-thresh"      ,     "-t"      ,     type=lowerCAmelCase__      ,     default=1.0      ,     help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."      ,     )
      parser.add_argument(
          "--out-image-dir"      ,     "-p"      ,     metavar="out_images"      ,     default=lowerCAmelCase__      ,     help="Save precision-recall curves to directory." )
      parser.add_argument("--verbose"      ,     "-v"      ,     action="store_true" )
      if len(sys.argv ) == 1:
            parser.print_help()
            sys.exit(1 )
      return parser.parse_args()
def   a__      ( lowerCAmelCase__ ):
      UpperCAmelCase_     					=				{}
      for article in dataset:
            for p in article["paragraphs"]:
                  for qa in p["qas"]:
                        UpperCAmelCase_     					=				bool(qa["answers"]["text"] )
      return qid_to_has_ans
def   a__      ( lowerCAmelCase__ ):

      def remove_articles(lowerCAmelCase__ ):
            return ARTICLES_REGEX.sub(" "      ,     lowerCAmelCase__ )

      def white_space_fix(lowerCAmelCase__ ):
            return " ".join(text.split() )

      def remove_punc(lowerCAmelCase__ ):
            UpperCAmelCase_     					=				set(string.punctuation )
            return "".join(ch for ch in text if ch not in exclude )

      def lower(lowerCAmelCase__ ):
            return text.lower()

      return white_space_fix(remove_articles(remove_punc(lower(lowerCAmelCase__ ) ) ) )
def   a__      ( lowerCAmelCase__ ):
      if not s:
            return []
      return normalize_answer(lowerCAmelCase__ ).split()
def   a__      ( lowerCAmelCase__      ,     lowerCAmelCase__ ):
      return int(normalize_answer(lowerCAmelCase__ ) == normalize_answer(lowerCAmelCase__ ) )
def   a__      ( lowerCAmelCase__      ,     lowerCAmelCase__ ):
      UpperCAmelCase_     					=				get_tokens(lowerCAmelCase__ )
      UpperCAmelCase_     					=				get_tokens(lowerCAmelCase__ )
      UpperCAmelCase_     					=				collections.Counter(lowerCAmelCase__ ) & collections.Counter(lowerCAmelCase__ )
      UpperCAmelCase_     					=				sum(common.values() )
      if len(lowerCAmelCase__ ) == 0 or len(lowerCAmelCase__ ) == 0:
            # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
            return int(gold_toks == pred_toks )
      if num_same == 0:
            return 0
      UpperCAmelCase_     					=				1.0 * num_same / len(lowerCAmelCase__ )
      UpperCAmelCase_     					=				1.0 * num_same / len(lowerCAmelCase__ )
      UpperCAmelCase_     					=				(2 * precision * recall) / (precision + recall)
      return fa
def   a__      ( lowerCAmelCase__      ,     lowerCAmelCase__ ):
      UpperCAmelCase_     					=				{}
      UpperCAmelCase_     					=				{}
      for article in dataset:
            for p in article["paragraphs"]:
                  for qa in p["qas"]:
                        UpperCAmelCase_     					=				qa["id"]
                        UpperCAmelCase_     					=				[t for t in qa["answers"]["text"] if normalize_answer(lowerCAmelCase__ )]
                        if not gold_answers:
                              # For unanswerable questions, only correct answer is empty string
                              UpperCAmelCase_     					=				[""]
                        if qid not in preds:
                              print(f"""Missing prediction for {qid}""" )
                              continue
                        UpperCAmelCase_     					=				preds[qid]
                        # Take max over all gold answers
                        UpperCAmelCase_     					=				max(compute_exact(lowerCAmelCase__      ,     lowerCAmelCase__ ) for a in gold_answers )
                        UpperCAmelCase_     					=				max(compute_fa(lowerCAmelCase__      ,     lowerCAmelCase__ ) for a in gold_answers )
      return exact_scores, fa_scores
def   a__      ( lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__ ):
      UpperCAmelCase_     					=				{}
      for qid, s in scores.items():
            UpperCAmelCase_     					=				na_probs[qid] > na_prob_thresh
            if pred_na:
                  UpperCAmelCase_     					=				float(not qid_to_has_ans[qid] )
            else:
                  UpperCAmelCase_     					=				s
      return new_scores
def   a__      ( lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__=None ):
      if not qid_list:
            UpperCAmelCase_     					=				len(lowerCAmelCase__ )
            return collections.OrderedDict(
                [
                    ("exact", 100.0 * sum(exact_scores.values() ) / total),
                    ("f1", 100.0 * sum(fa_scores.values() ) / total),
                    ("total", total),
                ] )
      else:
            UpperCAmelCase_     					=				len(lowerCAmelCase__ )
            return collections.OrderedDict(
                [
                    ("exact", 100.0 * sum(exact_scores[k] for k in qid_list ) / total),
                    ("f1", 100.0 * sum(fa_scores[k] for k in qid_list ) / total),
                    ("total", total),
                ] )
def   a__      ( lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__ ):
      for k in new_eval:
            UpperCAmelCase_     					=				new_eval[k]
def   a__      ( lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__ ):
      plt.step(lowerCAmelCase__      ,     lowerCAmelCase__      ,     color="b"      ,     alpha=0.2      ,     where="post" )
      plt.fill_between(lowerCAmelCase__      ,     lowerCAmelCase__      ,     step="post"      ,     alpha=0.2      ,     color="b" )
      plt.xlabel("Recall" )
      plt.ylabel("Precision" )
      plt.xlim([0.0, 1.05] )
      plt.ylim([0.0, 1.05] )
      plt.title(lowerCAmelCase__ )
      plt.savefig(lowerCAmelCase__ )
      plt.clf()
def   a__      ( lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__=None      ,     lowerCAmelCase__=None ):
      UpperCAmelCase_     					=				sorted(lowerCAmelCase__      ,     key=lambda lowerCAmelCase__ : na_probs[k] )
      UpperCAmelCase_     					=				0.0
      UpperCAmelCase_     					=				1.0
      UpperCAmelCase_     					=				0.0
      UpperCAmelCase_     					=				[1.0]
      UpperCAmelCase_     					=				[0.0]
      UpperCAmelCase_     					=				0.0
      for i, qid in enumerate(lowerCAmelCase__ ):
            if qid_to_has_ans[qid]:
                  true_pos += scores[qid]
            UpperCAmelCase_     					=				true_pos / float(i + 1 )
            UpperCAmelCase_     					=				true_pos / float(lowerCAmelCase__ )
            if i == len(lowerCAmelCase__ ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                  # i.e., if we can put a threshold after this point
                  avg_prec += cur_p * (cur_r - recalls[-1])
                  precisions.append(lowerCAmelCase__ )
                  recalls.append(lowerCAmelCase__ )
      if out_image:
            plot_pr_curve(lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__ )
      return {"ap": 100.0 * avg_prec}
def   a__      ( lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__ ):
      if out_image_dir and not os.path.exists(lowerCAmelCase__ ):
            os.makedirs(lowerCAmelCase__ )
      UpperCAmelCase_     					=				sum(1 for v in qid_to_has_ans.values() if v )
      if num_true_pos == 0:
            return
      UpperCAmelCase_     					=				make_precision_recall_eval(
          lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     out_image=os.path.join(lowerCAmelCase__      ,     "pr_exact.png" )      ,     title="Precision-Recall curve for Exact Match score"      ,     )
      UpperCAmelCase_     					=				make_precision_recall_eval(
          lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     out_image=os.path.join(lowerCAmelCase__      ,     "pr_f1.png" )      ,     title="Precision-Recall curve for F1 score"      ,     )
      UpperCAmelCase_     					=				{k: float(lowerCAmelCase__ ) for k, v in qid_to_has_ans.items()}
      UpperCAmelCase_     					=				make_precision_recall_eval(
          lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     out_image=os.path.join(lowerCAmelCase__      ,     "pr_oracle.png" )      ,     title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"      ,     )
      merge_eval(lowerCAmelCase__      ,     lowerCAmelCase__      ,     "pr_exact" )
      merge_eval(lowerCAmelCase__      ,     lowerCAmelCase__      ,     "pr_f1" )
      merge_eval(lowerCAmelCase__      ,     lowerCAmelCase__      ,     "pr_oracle" )
def   a__      ( lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__ ):
      if not qid_list:
            return
      UpperCAmelCase_     					=				[na_probs[k] for k in qid_list]
      UpperCAmelCase_     					=				np.ones_like(lowerCAmelCase__ ) / float(len(lowerCAmelCase__ ) )
      plt.hist(lowerCAmelCase__      ,     weights=lowerCAmelCase__      ,     bins=20      ,     range=(0.0, 1.0) )
      plt.xlabel("Model probability of no-answer" )
      plt.ylabel("Proportion of dataset" )
      plt.title(f"""Histogram of no-answer probability: {name}""" )
      plt.savefig(os.path.join(lowerCAmelCase__      ,     f"""na_prob_hist_{name}.png""" ) )
      plt.clf()
def   a__      ( lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__ ):
      UpperCAmelCase_     					=				sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k] )
      UpperCAmelCase_     					=				num_no_ans
      UpperCAmelCase_     					=				cur_score
      UpperCAmelCase_     					=				0.0
      UpperCAmelCase_     					=				sorted(lowerCAmelCase__      ,     key=lambda lowerCAmelCase__ : na_probs[k] )
      for i, qid in enumerate(lowerCAmelCase__ ):
            if qid not in scores:
                  continue
            if qid_to_has_ans[qid]:
                  UpperCAmelCase_     					=				scores[qid]
            else:
                  if preds[qid]:
                        UpperCAmelCase_     					=				-1
                  else:
                        UpperCAmelCase_     					=				0
            cur_score += diff
            if cur_score > best_score:
                  UpperCAmelCase_     					=				cur_score
                  UpperCAmelCase_     					=				na_probs[qid]
      return 100.0 * best_score / len(lowerCAmelCase__ ), best_thresh
def   a__      ( lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__ ):
      UpperCAmelCase_					,       UpperCAmelCase_     					=				find_best_thresh(lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__ )
      UpperCAmelCase_					,       UpperCAmelCase_     					=				find_best_thresh(lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__ )
      UpperCAmelCase_     					=				best_exact
      UpperCAmelCase_     					=				exact_thresh
      UpperCAmelCase_     					=				best_fa
      UpperCAmelCase_     					=				fa_thresh
def   a__      ( ):
      with open(OPTS.data_file ) as f:
            UpperCAmelCase_     					=				json.load(lowerCAmelCase__ )
            UpperCAmelCase_     					=				dataset_json["data"]
      with open(OPTS.pred_file ) as f:
            UpperCAmelCase_     					=				json.load(lowerCAmelCase__ )
      if OPTS.na_prob_file:
            with open(OPTS.na_prob_file ) as f:
                  UpperCAmelCase_     					=				json.load(lowerCAmelCase__ )
      else:
            UpperCAmelCase_     					=				{k: 0.0 for k in preds}
      UpperCAmelCase_     					=				make_qid_to_has_ans(lowerCAmelCase__ )  # maps qid to True/False
      UpperCAmelCase_     					=				[k for k, v in qid_to_has_ans.items() if v]
      UpperCAmelCase_     					=				[k for k, v in qid_to_has_ans.items() if not v]
      UpperCAmelCase_					,       UpperCAmelCase_     					=				get_raw_scores(lowerCAmelCase__      ,     lowerCAmelCase__ )
      UpperCAmelCase_     					=				apply_no_ans_threshold(lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     OPTS.na_prob_thresh )
      UpperCAmelCase_     					=				apply_no_ans_threshold(lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     OPTS.na_prob_thresh )
      UpperCAmelCase_     					=				make_eval_dict(lowerCAmelCase__      ,     lowerCAmelCase__ )
      if has_ans_qids:
            UpperCAmelCase_     					=				make_eval_dict(lowerCAmelCase__      ,     lowerCAmelCase__      ,     qid_list=lowerCAmelCase__ )
            merge_eval(lowerCAmelCase__      ,     lowerCAmelCase__      ,     "HasAns" )
      if no_ans_qids:
            UpperCAmelCase_     					=				make_eval_dict(lowerCAmelCase__      ,     lowerCAmelCase__      ,     qid_list=lowerCAmelCase__ )
            merge_eval(lowerCAmelCase__      ,     lowerCAmelCase__      ,     "NoAns" )
      if OPTS.na_prob_file:
            find_all_best_thresh(lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__ )
      if OPTS.na_prob_file and OPTS.out_image_dir:
            run_precision_recall_analysis(lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     lowerCAmelCase__      ,     OPTS.out_image_dir )
            histogram_na_prob(lowerCAmelCase__      ,     lowerCAmelCase__      ,     OPTS.out_image_dir      ,     "hasAns" )
            histogram_na_prob(lowerCAmelCase__      ,     lowerCAmelCase__      ,     OPTS.out_image_dir      ,     "noAns" )
      if OPTS.out_file:
            with open(OPTS.out_file      ,     "w" ) as f:
                  json.dump(lowerCAmelCase__      ,     lowerCAmelCase__ )
      else:
            print(json.dumps(lowerCAmelCase__      ,     indent=2 ) )


if __name__ == "__main__":
  lowerCamelCase  			=		parse_args()
  if OPTS.out_image_dir:
    import matplotlib

    matplotlib.use("""Agg""")
    import matplotlib.pyplot as plt
  main()









'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


snake_case_					:      Union[str, Any]    			=    {
    'configuration_transfo_xl': ['TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP', 'TransfoXLConfig'],
    'tokenization_transfo_xl': ['TransfoXLCorpus', 'TransfoXLTokenizer'],
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					snake_case_					:      str    			=    [
					    'TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST',
					    'AdaptiveEmbedding',
					    'TransfoXLForSequenceClassification',
					    'TransfoXLLMHeadModel',
					    'TransfoXLModel',
					    'TransfoXLPreTrainedModel',
					    'load_tf_weights_in_transfo_xl',
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					snake_case_					:      Any    			=    [
					    'TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST',
					    'TFAdaptiveEmbedding',
					    'TFTransfoXLForSequenceClassification',
					    'TFTransfoXLLMHeadModel',
					    'TFTransfoXLMainLayer',
					    'TFTransfoXLModel',
					    'TFTransfoXLPreTrainedModel',
					]


if TYPE_CHECKING:
					from .configuration_transfo_xl import TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, TransfoXLConfig
					from .tokenization_transfo_xl import TransfoXLCorpus, TransfoXLTokenizer

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_transfo_xl import (
										    TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
										    AdaptiveEmbedding,
										    TransfoXLForSequenceClassification,
										    TransfoXLLMHeadModel,
										    TransfoXLModel,
										    TransfoXLPreTrainedModel,
										    load_tf_weights_in_transfo_xl,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_transfo_xl import (
										    TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFAdaptiveEmbedding,
										    TFTransfoXLForSequenceClassification,
										    TFTransfoXLLMHeadModel,
										    TFTransfoXLMainLayer,
										    TFTransfoXLModel,
										    TFTransfoXLPreTrainedModel,
										)

else:
					import sys

					snake_case_					:      Dict    			=    _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


