




import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCamelCase_    		=  re.compile(r"\b(a|an|the)\b", re.UNICODE)

lowerCamelCase_    		=  None



def       UpperCAmelCase_       (      ):
    SCREAMING_SNAKE_CASE__        =argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0."""      )
    parser.add_argument("""data_file""",							metavar="""data.json""",							help="""Input data JSON file."""      )
    parser.add_argument("""pred_file""",							metavar="""pred.json""",							help="""Model predictions."""      )
    parser.add_argument(
        """--out-file""",							"""-o""",							metavar="""eval.json""",							help="""Write accuracy metrics to file (default is stdout)."""      )
    parser.add_argument(
        """--na-prob-file""",							"""-n""",							metavar="""na_prob.json""",							help="""Model estimates of probability of no answer."""      )
    parser.add_argument(
        """--na-prob-thresh""",							"""-t""",							type=__snake_case,							default=1.0,							help="""Predict \"\" if no-answer probability exceeds this (default = 1.0).""",							)
    parser.add_argument(
        """--out-image-dir""",							"""-p""",							metavar="""out_images""",							default=__snake_case,							help="""Save precision-recall curves to directory."""      )
    parser.add_argument("""--verbose""",							"""-v""",							action="""store_true"""      )
    if len(sys.argv      ) == 1:
        parser.print_help()
        sys.exit(1      )
    return parser.parse_args()



def       UpperCAmelCase_       (      __UpperCamelCase      ):
    SCREAMING_SNAKE_CASE__        ={}
    for article in dataset:
        for p in article["paragraphs"]:
            for qa in p["qas"]:
                SCREAMING_SNAKE_CASE__        =bool(qa["""answers"""]["""text"""]      )
    return qid_to_has_ans



def       UpperCAmelCase_       (      __UpperCamelCase      ):







    def remove_articles(__UpperCamelCase      ):
        return ARTICLES_REGEX.sub(""" """,							__snake_case      )

    def white_space_fix(__UpperCamelCase      ):
        return " ".join(text.split()      )

    def remove_punc(__UpperCamelCase      ):
        SCREAMING_SNAKE_CASE__        =set(string.punctuation      )
        return "".join(ch for ch in text if ch not in exclude      )

    def lower(__UpperCamelCase      ):
        return text.lower()

    return white_space_fix(remove_articles(remove_punc(lower(__snake_case      )      )      )      )



def       UpperCAmelCase_       (      __UpperCamelCase      ):







    if not s:
        return []
    return normalize_answer(__snake_case      ).split()



def       UpperCAmelCase_       (      __UpperCamelCase,							__UpperCamelCase      ):
    return int(normalize_answer(__snake_case      ) == normalize_answer(__snake_case      )      )



def       UpperCAmelCase_       (      __UpperCamelCase,							__UpperCamelCase      ):
    SCREAMING_SNAKE_CASE__        =get_tokens(__snake_case      )
    SCREAMING_SNAKE_CASE__        =get_tokens(__snake_case      )
    SCREAMING_SNAKE_CASE__        =collections.Counter(__snake_case      ) & collections.Counter(__snake_case      )
    SCREAMING_SNAKE_CASE__        =sum(common.values()      )
    if len(__snake_case      ) == 0 or len(__snake_case      ) == 0:
        # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
        return int(gold_toks == pred_toks      )
    if num_same == 0:
        return 0
    SCREAMING_SNAKE_CASE__        =1.0 * num_same / len(__snake_case      )
    SCREAMING_SNAKE_CASE__        =1.0 * num_same / len(__snake_case      )
    SCREAMING_SNAKE_CASE__        =(2 * precision * recall) / (precision + recall)
    return fa



def       UpperCAmelCase_       (      __UpperCamelCase,							__UpperCamelCase      ):
    SCREAMING_SNAKE_CASE__        ={}
    SCREAMING_SNAKE_CASE__        ={}
    for article in dataset:
        for p in article["paragraphs"]:
            for qa in p["qas"]:
                SCREAMING_SNAKE_CASE__        =qa["""id"""]
                SCREAMING_SNAKE_CASE__        =[t for t in qa["""answers"""]["""text"""] if normalize_answer(__snake_case      )]
                if not gold_answers:
                    # For unanswerable questions, only correct answer is empty string
                    SCREAMING_SNAKE_CASE__        =[""""""]
                if qid not in preds:
                    print(f"""Missing prediction for {qid}"""      )
                    continue
                SCREAMING_SNAKE_CASE__        =preds[qid]
                # Take max over all gold answers
                SCREAMING_SNAKE_CASE__        =max(compute_exact(__snake_case,							__snake_case      ) for a in gold_answers      )
                SCREAMING_SNAKE_CASE__        =max(compute_fa(__snake_case,							__snake_case      ) for a in gold_answers      )
    return exact_scores, fa_scores



def       UpperCAmelCase_       (      __UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase      ):
    SCREAMING_SNAKE_CASE__        ={}
    for qid, s in scores.items():
        SCREAMING_SNAKE_CASE__        =na_probs[qid] > na_prob_thresh
        if pred_na:
            SCREAMING_SNAKE_CASE__        =float(not qid_to_has_ans[qid]      )
        else:
            SCREAMING_SNAKE_CASE__        =s
    return new_scores



def       UpperCAmelCase_       (      __UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase=None      ):







    if not qid_list:
        SCREAMING_SNAKE_CASE__        =len(__snake_case      )
        return collections.OrderedDict(
            [
                ("""exact""", 100.0 * sum(exact_scores.values()      ) / total),
                ("""f1""", 100.0 * sum(fa_scores.values()      ) / total),
                ("""total""", total),
            ]      )
    else:
        SCREAMING_SNAKE_CASE__        =len(__snake_case      )
        return collections.OrderedDict(
            [
                ("""exact""", 100.0 * sum(exact_scores[k] for k in qid_list      ) / total),
                ("""f1""", 100.0 * sum(fa_scores[k] for k in qid_list      ) / total),
                ("""total""", total),
            ]      )



def       UpperCAmelCase_       (      __UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase      ):







    for k in new_eval:
        SCREAMING_SNAKE_CASE__        =new_eval[k]



def       UpperCAmelCase_       (      __UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase      ):
    plt.step(__snake_case,							__snake_case,							color="""b""",							alpha=0.2,							where="""post"""      )
    plt.fill_between(__snake_case,							__snake_case,							step="""post""",							alpha=0.2,							color="""b"""      )
    plt.xlabel("""Recall"""      )
    plt.ylabel("""Precision"""      )
    plt.xlim([0.0, 1.05]      )
    plt.ylim([0.0, 1.05]      )
    plt.title(__snake_case      )
    plt.savefig(__snake_case      )
    plt.clf()



def       UpperCAmelCase_       (      __UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase=None,							__UpperCamelCase=None      ):
    SCREAMING_SNAKE_CASE__        =sorted(__snake_case,							key=lambda __UpperCamelCase      : na_probs[k]      )
    SCREAMING_SNAKE_CASE__        =0.0
    SCREAMING_SNAKE_CASE__        =1.0
    SCREAMING_SNAKE_CASE__        =0.0
    SCREAMING_SNAKE_CASE__        =[1.0]
    SCREAMING_SNAKE_CASE__        =[0.0]
    SCREAMING_SNAKE_CASE__        =0.0
    for i, qid in enumerate(__snake_case      ):
        if qid_to_has_ans[qid]:
            true_pos += scores[qid]
        SCREAMING_SNAKE_CASE__        =true_pos / float(i + 1      )
        SCREAMING_SNAKE_CASE__        =true_pos / float(__snake_case      )
        if i == len(__snake_case      ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
            # i.e., if we can put a threshold after this point
            avg_prec += cur_p * (cur_r - recalls[-1])
            precisions.append(__snake_case      )
            recalls.append(__snake_case      )
    if out_image:
        plot_pr_curve(__snake_case,							__snake_case,							__snake_case,							__snake_case      )
    return {"ap": 100.0 * avg_prec}



def       UpperCAmelCase_       (      __UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase      ):







    if out_image_dir and not os.path.exists(__snake_case      ):
        os.makedirs(__snake_case      )
    SCREAMING_SNAKE_CASE__        =sum(1 for v in qid_to_has_ans.values() if v      )
    if num_true_pos == 0:
        return
    SCREAMING_SNAKE_CASE__        =make_precision_recall_eval(
        __snake_case,							__snake_case,							__snake_case,							__snake_case,							out_image=os.path.join(__snake_case,							"""pr_exact.png"""      ),							title="""Precision-Recall curve for Exact Match score""",							)
    SCREAMING_SNAKE_CASE__        =make_precision_recall_eval(
        __snake_case,							__snake_case,							__snake_case,							__snake_case,							out_image=os.path.join(__snake_case,							"""pr_f1.png"""      ),							title="""Precision-Recall curve for F1 score""",							)
    SCREAMING_SNAKE_CASE__        ={k: float(__snake_case      ) for k, v in qid_to_has_ans.items()}
    SCREAMING_SNAKE_CASE__        =make_precision_recall_eval(
        __snake_case,							__snake_case,							__snake_case,							__snake_case,							out_image=os.path.join(__snake_case,							"""pr_oracle.png"""      ),							title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)""",							)
    merge_eval(__snake_case,							__snake_case,							"""pr_exact"""      )
    merge_eval(__snake_case,							__snake_case,							"""pr_f1"""      )
    merge_eval(__snake_case,							__snake_case,							"""pr_oracle"""      )



def       UpperCAmelCase_       (      __UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase      ):







    if not qid_list:
        return
    SCREAMING_SNAKE_CASE__        =[na_probs[k] for k in qid_list]
    SCREAMING_SNAKE_CASE__        =np.ones_like(__snake_case      ) / float(len(__snake_case      )      )
    plt.hist(__snake_case,							weights=__snake_case,							bins=20,							range=(0.0, 1.0)      )
    plt.xlabel("""Model probability of no-answer"""      )
    plt.ylabel("""Proportion of dataset"""      )
    plt.title(f"""Histogram of no-answer probability: {name}"""      )
    plt.savefig(os.path.join(__snake_case,							f"""na_prob_hist_{name}.png"""      )      )
    plt.clf()



def       UpperCAmelCase_       (      __UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase      ):
    SCREAMING_SNAKE_CASE__        =sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]      )
    SCREAMING_SNAKE_CASE__        =num_no_ans
    SCREAMING_SNAKE_CASE__        =cur_score
    SCREAMING_SNAKE_CASE__        =0.0
    SCREAMING_SNAKE_CASE__        =sorted(__snake_case,							key=lambda __UpperCamelCase      : na_probs[k]      )
    for i, qid in enumerate(__snake_case      ):
        if qid not in scores:
            continue
        if qid_to_has_ans[qid]:
            SCREAMING_SNAKE_CASE__        =scores[qid]
        else:
            if preds[qid]:
                SCREAMING_SNAKE_CASE__        =-1
            else:
                SCREAMING_SNAKE_CASE__        =0
        cur_score += diff
        if cur_score > best_score:
            SCREAMING_SNAKE_CASE__        =cur_score
            SCREAMING_SNAKE_CASE__        =na_probs[qid]
    return 100.0 * best_score / len(__snake_case      ), best_thresh



def       UpperCAmelCase_       (      __UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase,							__UpperCamelCase      ):
    SCREAMING_SNAKE_CASE__        =find_best_thresh(__snake_case,							__snake_case,							__snake_case,							__snake_case      )
    SCREAMING_SNAKE_CASE__        =find_best_thresh(__snake_case,							__snake_case,							__snake_case,							__snake_case      )
    SCREAMING_SNAKE_CASE__        =best_exact
    SCREAMING_SNAKE_CASE__        =exact_thresh
    SCREAMING_SNAKE_CASE__        =best_fa
    SCREAMING_SNAKE_CASE__        =fa_thresh



def       UpperCAmelCase_       (      ):







    with open(OPTS.data_file      ) as f:
        SCREAMING_SNAKE_CASE__        =json.load(__snake_case      )
        SCREAMING_SNAKE_CASE__        =dataset_json["""data"""]
    with open(OPTS.pred_file      ) as f:
        SCREAMING_SNAKE_CASE__        =json.load(__snake_case      )
    if OPTS.na_prob_file:
        with open(OPTS.na_prob_file      ) as f:
            SCREAMING_SNAKE_CASE__        =json.load(__snake_case      )
    else:
        SCREAMING_SNAKE_CASE__        ={k: 0.0 for k in preds}
    SCREAMING_SNAKE_CASE__        =make_qid_to_has_ans(__snake_case      )  # maps qid to True/False
    SCREAMING_SNAKE_CASE__        =[k for k, v in qid_to_has_ans.items() if v]
    SCREAMING_SNAKE_CASE__        =[k for k, v in qid_to_has_ans.items() if not v]
    SCREAMING_SNAKE_CASE__        =get_raw_scores(__snake_case,							__snake_case      )
    SCREAMING_SNAKE_CASE__        =apply_no_ans_threshold(__snake_case,							__snake_case,							__snake_case,							OPTS.na_prob_thresh      )
    SCREAMING_SNAKE_CASE__        =apply_no_ans_threshold(__snake_case,							__snake_case,							__snake_case,							OPTS.na_prob_thresh      )
    SCREAMING_SNAKE_CASE__        =make_eval_dict(__snake_case,							__snake_case      )
    if has_ans_qids:
        SCREAMING_SNAKE_CASE__        =make_eval_dict(__snake_case,							__snake_case,							qid_list=__snake_case      )
        merge_eval(__snake_case,							__snake_case,							"""HasAns"""      )
    if no_ans_qids:
        SCREAMING_SNAKE_CASE__        =make_eval_dict(__snake_case,							__snake_case,							qid_list=__snake_case      )
        merge_eval(__snake_case,							__snake_case,							"""NoAns"""      )
    if OPTS.na_prob_file:
        find_all_best_thresh(__snake_case,							__snake_case,							__snake_case,							__snake_case,							__snake_case,							__snake_case      )
    if OPTS.na_prob_file and OPTS.out_image_dir:
        run_precision_recall_analysis(__snake_case,							__snake_case,							__snake_case,							__snake_case,							__snake_case,							OPTS.out_image_dir      )
        histogram_na_prob(__snake_case,							__snake_case,							OPTS.out_image_dir,							"""hasAns"""      )
        histogram_na_prob(__snake_case,							__snake_case,							OPTS.out_image_dir,							"""noAns"""      )
    if OPTS.out_file:
        with open(OPTS.out_file,							"""w"""      ) as f:
            json.dump(__snake_case,							__snake_case      )
    else:
        print(json.dumps(__snake_case,							indent=2      )      )


if __name__ == "__main__":
    lowerCamelCase_    		=  parse_args()
    if OPTS.out_image_dir:
        import matplotlib

        matplotlib.use("Agg")
        import matplotlib.pyplot as plt
    main()










"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


UpperCAmelCase				=						{
    """configuration_vision_encoder_decoder""": ["""VisionEncoderDecoderConfig""", """VisionEncoderDecoderOnnxConfig"""]
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 UpperCAmelCase				=						["""VisionEncoderDecoderModel"""]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 UpperCAmelCase				=						["""TFVisionEncoderDecoderModel"""]

try:
 if not is_flax_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 UpperCAmelCase				=						["""FlaxVisionEncoderDecoderModel"""]

if TYPE_CHECKING:
 from .configuration_vision_encoder_decoder import VisionEncoderDecoderConfig, VisionEncoderDecoderOnnxConfig

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_vision_encoder_decoder import VisionEncoderDecoderModel

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_vision_encoder_decoder import TFVisionEncoderDecoderModel

 try:
  if not is_flax_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_flax_vision_encoder_decoder import FlaxVisionEncoderDecoderModel

else:
 import sys

 UpperCAmelCase				=						_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

