


'''simple docstring'''

import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCamelCase				:  Dict    		=      re.compile(R'\b(a|an|the)\b', re.UNICODE)

lowerCamelCase				:  Tuple    		=      None

def _SCREAMING_SNAKE_CASE ()			->			Union[str, Any]:






       """simple docstring"""







       lowercase__      =					argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''			)
       parser.add_argument('''data_file'''    ,					metavar='''data.json'''    ,					help='''Input data JSON file.'''			)
       parser.add_argument('''pred_file'''    ,					metavar='''pred.json'''    ,					help='''Model predictions.'''			)
       parser.add_argument(
           '''--out-file'''    ,					'''-o'''    ,					metavar='''eval.json'''    ,					help='''Write accuracy metrics to file (default is stdout).'''			)
       parser.add_argument(
           '''--na-prob-file'''    ,					'''-n'''    ,					metavar='''na_prob.json'''    ,					help='''Model estimates of probability of no answer.'''			)
       parser.add_argument(
           '''--na-prob-thresh'''    ,					'''-t'''    ,					type=A    ,					default=1.0    ,					help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''    ,					)
       parser.add_argument(
           '''--out-image-dir'''    ,					'''-p'''    ,					metavar='''out_images'''    ,					default=A    ,					help='''Save precision-recall curves to directory.'''			)
       parser.add_argument('''--verbose'''    ,					'''-v'''    ,					action='''store_true'''			)
       if len(sys.argv			) == 1:
              parser.print_help()
              sys.exit(1			)
       return parser.parse_args()

def _SCREAMING_SNAKE_CASE (A			)			->			Any:






       """simple docstring"""







       lowercase__      =					{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            lowercase__      =					bool(qa['''answers''']['''text''']			)
       return qid_to_has_ans

def _SCREAMING_SNAKE_CASE (A			)			->			Union[str, Any]:






       """simple docstring"""







       def remove_articles(A			):
              return ARTICLES_REGEX.sub(''' '''    ,					A			)

       def white_space_fix(A			):
              return " ".join(text.split()			)

       def remove_punc(A			):
              lowercase__      =					set(string.punctuation			)
              return "".join(ch for ch in text if ch not in exclude			)

       def lower(A			):
              return text.lower()

       return white_space_fix(remove_articles(remove_punc(lower(A			)			)			)			)

def _SCREAMING_SNAKE_CASE (A			)			->			List[str]:






       """simple docstring"""







       if not s:
              return []
       return normalize_answer(A			).split()

def _SCREAMING_SNAKE_CASE (A    ,					A			)			->			Tuple:






       """simple docstring"""







       return int(normalize_answer(A			) == normalize_answer(A			)			)

def _SCREAMING_SNAKE_CASE (A    ,					A			)			->			Any:






       """simple docstring"""







       lowercase__      =					get_tokens(A			)
       lowercase__      =					get_tokens(A			)
       lowercase__      =					collections.Counter(A			) & collections.Counter(A			)
       lowercase__      =					sum(common.values()			)
       if len(A			) == 0 or len(A			) == 0:
              # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
              return int(gold_toks == pred_toks			)
       if num_same == 0:
              return 0
       lowercase__      =					1.0 * num_same / len(A			)
       lowercase__      =					1.0 * num_same / len(A			)
       lowercase__      =					(2 * precision * recall) / (precision + recall)
       return fa

def _SCREAMING_SNAKE_CASE (A    ,					A			)			->			int:






       """simple docstring"""







       lowercase__      =					{}
       lowercase__      =					{}
       for article in dataset:
              for p in article["paragraphs"]:
                     for qa in p["qas"]:
                            lowercase__      =					qa['''id''']
                            lowercase__      =					[t for t in qa['''answers''']['''text'''] if normalize_answer(A			)]
                            if not gold_answers:
                                   # For unanswerable questions, only correct answer is empty string
                                   lowercase__      =					['''''']
                            if qid not in preds:
                                   print(f"Missing prediction for {qid}"			)
                                   continue
                            lowercase__      =					preds[qid]
                            # Take max over all gold answers
                            lowercase__      =					max(compute_exact(A    ,					A			) for a in gold_answers			)
                            lowercase__      =					max(compute_fa(A    ,					A			) for a in gold_answers			)
       return exact_scores, fa_scores

def _SCREAMING_SNAKE_CASE (A    ,					A    ,					A    ,					A			)			->			str:






       """simple docstring"""







       lowercase__      =					{}
       for qid, s in scores.items():
              lowercase__      =					na_probs[qid] > na_prob_thresh
              if pred_na:
                     lowercase__      =					float(not qid_to_has_ans[qid]			)
              else:
                     lowercase__      =					s
       return new_scores

def _SCREAMING_SNAKE_CASE (A    ,					A    ,					A=None			)			->			List[str]:






       """simple docstring"""







       if not qid_list:
              lowercase__      =					len(A			)
              return collections.OrderedDict(
                  [
                      ('''exact''', 100.0 * sum(exact_scores.values()			) / total),
                      ('''f1''', 100.0 * sum(fa_scores.values()			) / total),
                      ('''total''', total),
                  ]			)
       else:
              lowercase__      =					len(A			)
              return collections.OrderedDict(
                  [
                      ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list			) / total),
                      ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list			) / total),
                      ('''total''', total),
                  ]			)

def _SCREAMING_SNAKE_CASE (A    ,					A    ,					A			)			->			Dict:






       """simple docstring"""







       for k in new_eval:
              lowercase__      =					new_eval[k]

def _SCREAMING_SNAKE_CASE (A    ,					A    ,					A    ,					A			)			->			List[str]:






       """simple docstring"""







       plt.step(A    ,					A    ,					color='''b'''    ,					alpha=0.2    ,					where='''post'''			)
       plt.fill_between(A    ,					A    ,					step='''post'''    ,					alpha=0.2    ,					color='''b'''			)
       plt.xlabel('''Recall'''			)
       plt.ylabel('''Precision'''			)
       plt.xlim([0.0, 1.05]			)
       plt.ylim([0.0, 1.05]			)
       plt.title(A			)
       plt.savefig(A			)
       plt.clf()

def _SCREAMING_SNAKE_CASE (A    ,					A    ,					A    ,					A    ,					A=None    ,					A=None			)			->			List[Any]:






       """simple docstring"""







       lowercase__      =					sorted(A    ,					key=lambda A			: na_probs[k]			)
       lowercase__      =					0.0
       lowercase__      =					1.0
       lowercase__      =					0.0
       lowercase__      =					[1.0]
       lowercase__      =					[0.0]
       lowercase__      =					0.0
       for i, qid in enumerate(A			):
              if qid_to_has_ans[qid]:
                     true_pos += scores[qid]
              lowercase__      =					true_pos / float(i + 1			)
              lowercase__      =					true_pos / float(A			)
              if i == len(A			) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                     # i.e., if we can put a threshold after this point
                     avg_prec += cur_p * (cur_r - recalls[-1])
                     precisions.append(A			)
                     recalls.append(A			)
       if out_image:
              plot_pr_curve(A    ,					A    ,					A    ,					A			)
       return {"ap": 100.0 * avg_prec}

def _SCREAMING_SNAKE_CASE (A    ,					A    ,					A    ,					A    ,					A    ,					A			)			->			int:






       """simple docstring"""







       if out_image_dir and not os.path.exists(A			):
              os.makedirs(A			)
       lowercase__      =					sum(1 for v in qid_to_has_ans.values() if v			)
       if num_true_pos == 0:
              return
       lowercase__      =					make_precision_recall_eval(
           A    ,					A    ,					A    ,					A    ,					out_image=os.path.join(A    ,					'''pr_exact.png'''			)    ,					title='''Precision-Recall curve for Exact Match score'''    ,					)
       lowercase__      =					make_precision_recall_eval(
           A    ,					A    ,					A    ,					A    ,					out_image=os.path.join(A    ,					'''pr_f1.png'''			)    ,					title='''Precision-Recall curve for F1 score'''    ,					)
       lowercase__      =					{k: float(A			) for k, v in qid_to_has_ans.items()}
       lowercase__      =					make_precision_recall_eval(
           A    ,					A    ,					A    ,					A    ,					out_image=os.path.join(A    ,					'''pr_oracle.png'''			)    ,					title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''    ,					)
       merge_eval(A    ,					A    ,					'''pr_exact'''			)
       merge_eval(A    ,					A    ,					'''pr_f1'''			)
       merge_eval(A    ,					A    ,					'''pr_oracle'''			)

def _SCREAMING_SNAKE_CASE (A    ,					A    ,					A    ,					A			)			->			Tuple:






       """simple docstring"""







       if not qid_list:
              return
       lowercase__      =					[na_probs[k] for k in qid_list]
       lowercase__      =					np.ones_like(A			) / float(len(A			)			)
       plt.hist(A    ,					weights=A    ,					bins=20    ,					range=(0.0, 1.0)			)
       plt.xlabel('''Model probability of no-answer'''			)
       plt.ylabel('''Proportion of dataset'''			)
       plt.title(f"Histogram of no-answer probability: {name}"			)
       plt.savefig(os.path.join(A    ,					f"na_prob_hist_{name}.png"			)			)
       plt.clf()

def _SCREAMING_SNAKE_CASE (A    ,					A    ,					A    ,					A			)			->			List[str]:






       """simple docstring"""







       lowercase__      =					sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]			)
       lowercase__      =					num_no_ans
       lowercase__      =					cur_score
       lowercase__      =					0.0
       lowercase__      =					sorted(A    ,					key=lambda A			: na_probs[k]			)
       for i, qid in enumerate(A			):
              if qid not in scores:
                     continue
              if qid_to_has_ans[qid]:
                     lowercase__      =					scores[qid]
              else:
                     if preds[qid]:
                            lowercase__      =					-1
                     else:
                            lowercase__      =					0
              cur_score += diff
              if cur_score > best_score:
                     lowercase__      =					cur_score
                     lowercase__      =					na_probs[qid]
       return 100.0 * best_score / len(A			), best_thresh

def _SCREAMING_SNAKE_CASE (A    ,					A    ,					A    ,					A    ,					A    ,					A			)			->			str:






       """simple docstring"""







       lowercase__     ,lowercase__      =					find_best_thresh(A    ,					A    ,					A    ,					A			)
       lowercase__     ,lowercase__      =					find_best_thresh(A    ,					A    ,					A    ,					A			)
       lowercase__      =					best_exact
       lowercase__      =					exact_thresh
       lowercase__      =					best_fa
       lowercase__      =					fa_thresh

def _SCREAMING_SNAKE_CASE ()			->			Union[str, Any]:






       """simple docstring"""







       with open(OPTS.data_file			) as f:
              lowercase__      =					json.load(A			)
              lowercase__      =					dataset_json['''data''']
       with open(OPTS.pred_file			) as f:
              lowercase__      =					json.load(A			)
       if OPTS.na_prob_file:
              with open(OPTS.na_prob_file			) as f:
                     lowercase__      =					json.load(A			)
       else:
              lowercase__      =					{k: 0.0 for k in preds}
       lowercase__      =					make_qid_to_has_ans(A			)  # maps qid to True/False
       lowercase__      =					[k for k, v in qid_to_has_ans.items() if v]
       lowercase__      =					[k for k, v in qid_to_has_ans.items() if not v]
       lowercase__     ,lowercase__      =					get_raw_scores(A    ,					A			)
       lowercase__      =					apply_no_ans_threshold(A    ,					A    ,					A    ,					OPTS.na_prob_thresh			)
       lowercase__      =					apply_no_ans_threshold(A    ,					A    ,					A    ,					OPTS.na_prob_thresh			)
       lowercase__      =					make_eval_dict(A    ,					A			)
       if has_ans_qids:
              lowercase__      =					make_eval_dict(A    ,					A    ,					qid_list=A			)
              merge_eval(A    ,					A    ,					'''HasAns'''			)
       if no_ans_qids:
              lowercase__      =					make_eval_dict(A    ,					A    ,					qid_list=A			)
              merge_eval(A    ,					A    ,					'''NoAns'''			)
       if OPTS.na_prob_file:
              find_all_best_thresh(A    ,					A    ,					A    ,					A    ,					A    ,					A			)
       if OPTS.na_prob_file and OPTS.out_image_dir:
              run_precision_recall_analysis(A    ,					A    ,					A    ,					A    ,					A    ,					OPTS.out_image_dir			)
              histogram_na_prob(A    ,					A    ,					OPTS.out_image_dir    ,					'''hasAns'''			)
              histogram_na_prob(A    ,					A    ,					OPTS.out_image_dir    ,					'''noAns'''			)
       if OPTS.out_file:
              with open(OPTS.out_file    ,					'''w'''			) as f:
                     json.dump(A    ,					A			)
       else:
              print(json.dumps(A    ,					indent=2			)			)


if __name__ == "__main__":
 lowerCamelCase				:  str    		=      parse_args()
 if OPTS.out_image_dir:
  import matplotlib

  matplotlib.use('Agg')
  import matplotlib.pyplot as plt
 main()








'''simple docstring'''

from typing import Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature
from ...image_transforms import get_image_size, pad, rescale, to_channel_dimension_format
from ...image_utils import ChannelDimension, ImageInput, make_list_of_images, to_numpy_array, valid_images
from ...utils import TensorType, logging


lowerCamelCase				:  Optional[int]    		=      logging.get_logger(__name__)
class     __lowerCAmelCase		(lowercase_	):





      '''simple docstring'''






      lowerCAmelCase__   :	Optional[int]       =				["""pixel_values"""]



      def __init__(self       :       str  ,		UpperCamelCase       :       bool = True  ,		UpperCamelCase       :       Union[int, float] = 1 / 255  ,		UpperCamelCase       :       bool = True  ,		UpperCamelCase       :       int = 8  ,		**UpperCamelCase       :       Dict  ,		):

             '''simple docstring'''







             super().__init__(**UpperCamelCase					)

             lowercase__      =					do_rescale
             lowercase__      =					rescale_factor
             lowercase__      =					do_pad
             lowercase__      =					pad_size



      def  UpperCamelCase__						(self       :       Dict  ,		UpperCamelCase       :       np.ndarray  ,		UpperCamelCase       :       float  ,		UpperCamelCase       :       Optional[Union[str, ChannelDimension]] = None  ,		**UpperCamelCase       :       Tuple					):

             '''simple docstring'''







             return rescale(UpperCamelCase  ,		scale=UpperCamelCase  ,		data_format=UpperCamelCase  ,		**UpperCamelCase					)



      def  UpperCamelCase__						(self       :       Tuple  ,		UpperCamelCase       :       np.ndarray  ,		UpperCamelCase       :       int  ,		UpperCamelCase       :       Optional[Union[str, ChannelDimension]] = None					):

             '''simple docstring'''







             lowercase__     ,lowercase__      =					get_image_size(UpperCamelCase					)
             lowercase__      =					(old_height // size + 1) * size - old_height
             lowercase__      =					(old_width // size + 1) * size - old_width

             return pad(UpperCamelCase  ,		((0, pad_height), (0, pad_width))  ,		mode='''symmetric'''  ,		data_format=UpperCamelCase					)






      def  UpperCamelCase__						(self       :       Optional[int]  ,		UpperCamelCase       :       ImageInput  ,		UpperCamelCase       :       Optional[bool] = None  ,		UpperCamelCase       :       Optional[float] = None  ,		UpperCamelCase       :       Optional[bool] = None  ,		UpperCamelCase       :       Optional[int] = None  ,		UpperCamelCase       :       Optional[Union[str, TensorType]] = None  ,		UpperCamelCase       :       Union[str, ChannelDimension] = ChannelDimension.FIRST  ,		**UpperCamelCase       :       int  ,		):

             '''simple docstring'''







             lowercase__      =					do_rescale if do_rescale is not None else self.do_rescale
             lowercase__      =					rescale_factor if rescale_factor is not None else self.rescale_factor
             lowercase__      =					do_pad if do_pad is not None else self.do_pad
             lowercase__      =					pad_size if pad_size is not None else self.pad_size

             lowercase__      =					make_list_of_images(UpperCamelCase					)

             if not valid_images(UpperCamelCase					):
                    raise ValueError(
                        '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
                        '''torch.Tensor, tf.Tensor or jax.ndarray.'''					)

             if do_rescale and rescale_factor is None:
                    raise ValueError('''Rescale factor must be specified if do_rescale is True.'''					)

             # All transformations expect numpy arrays.
             lowercase__      =					[to_numpy_array(UpperCamelCase					) for image in images]

             if do_rescale:
                    lowercase__      =					[self.rescale(image=UpperCamelCase  ,		scale=UpperCamelCase					) for image in images]

             if do_pad:
                    lowercase__      =					[self.pad(UpperCamelCase  ,		size=UpperCamelCase					) for image in images]

             lowercase__      =					[to_channel_dimension_format(UpperCamelCase  ,		UpperCamelCase					) for image in images]

             lowercase__      =					{'''pixel_values''': images}
             return BatchFeature(data=UpperCamelCase  ,		tensor_type=UpperCamelCase					)




