





'''simple docstring'''






import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


__lowerCamelCase						: Tuple		=   re.compile(R'\b(a|an|the)\b', re.UNICODE)

__lowerCamelCase						: Optional[int]		=   None


def 					_a   ():


  """simple docstring"""

  _UpperCamelCase       			=argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''   )
  parser.add_argument('''data_file'''  ,      metavar='''data.json'''  ,      help='''Input data JSON file.'''   )
  parser.add_argument('''pred_file'''  ,      metavar='''pred.json'''  ,      help='''Model predictions.'''   )
  parser.add_argument(
      '''--out-file'''  ,      '''-o'''  ,      metavar='''eval.json'''  ,      help='''Write accuracy metrics to file (default is stdout).'''   )
  parser.add_argument(
      '''--na-prob-file'''  ,      '''-n'''  ,      metavar='''na_prob.json'''  ,      help='''Model estimates of probability of no answer.'''   )
  parser.add_argument(
      '''--na-prob-thresh'''  ,      '''-t'''  ,      type=__SCREAMING_SNAKE_CASE  ,      default=1.0  ,      help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''  ,      )
  parser.add_argument(
      '''--out-image-dir'''  ,      '''-p'''  ,      metavar='''out_images'''  ,      default=__SCREAMING_SNAKE_CASE  ,      help='''Save precision-recall curves to directory.'''   )
  parser.add_argument('''--verbose'''  ,      '''-v'''  ,      action='''store_true'''   )
  if len(sys.argv   ) == 1:
    parser.print_help()
    sys.exit(1   )
  return parser.parse_args()


def 					_a   (__SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  _UpperCamelCase       			={}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        _UpperCamelCase       			=bool(qa['''answers''']['''text''']   )
  return qid_to_has_ans


def 					_a   (__SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  def remove_articles(__SCREAMING_SNAKE_CASE   ):
    return ARTICLES_REGEX.sub(''' '''  ,      __SCREAMING_SNAKE_CASE   )

  def white_space_fix(__SCREAMING_SNAKE_CASE   ):
    return " ".join(text.split()   )

  def remove_punc(__SCREAMING_SNAKE_CASE   ):
    _UpperCamelCase       			=set(string.punctuation   )
    return "".join(ch for ch in text if ch not in exclude   )

  def lower(__SCREAMING_SNAKE_CASE   ):
    return text.lower()

  return white_space_fix(remove_articles(remove_punc(lower(__SCREAMING_SNAKE_CASE   )   )   )   )


def 					_a   (__SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  if not s:
    return []
  return normalize_answer(__SCREAMING_SNAKE_CASE   ).split()


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  return int(normalize_answer(__SCREAMING_SNAKE_CASE   ) == normalize_answer(__SCREAMING_SNAKE_CASE   )   )


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  _UpperCamelCase       			=get_tokens(__SCREAMING_SNAKE_CASE   )
  _UpperCamelCase       			=get_tokens(__SCREAMING_SNAKE_CASE   )
  _UpperCamelCase       			=collections.Counter(__SCREAMING_SNAKE_CASE   ) & collections.Counter(__SCREAMING_SNAKE_CASE   )
  _UpperCamelCase       			=sum(common.values()   )
  if len(__SCREAMING_SNAKE_CASE   ) == 0 or len(__SCREAMING_SNAKE_CASE   ) == 0:
    # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
    return int(gold_toks == pred_toks   )
  if num_same == 0:
    return 0
  _UpperCamelCase       			=1.0 * num_same / len(__SCREAMING_SNAKE_CASE   )
  _UpperCamelCase       			=1.0 * num_same / len(__SCREAMING_SNAKE_CASE   )
  _UpperCamelCase       			=(2 * precision * recall) / (precision + recall)
  return fa


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  _UpperCamelCase       			={}
  _UpperCamelCase       			={}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        _UpperCamelCase       			=qa['''id''']
        _UpperCamelCase       			=[t for t in qa['''answers''']['''text'''] if normalize_answer(__SCREAMING_SNAKE_CASE   )]
        if not gold_answers:
          # For unanswerable questions, only correct answer is empty string
          _UpperCamelCase       			=['''''']
        if qid not in preds:
          print(f'''Missing prediction for {qid}'''   )
          continue
        _UpperCamelCase       			=preds[qid]
        # Take max over all gold answers
        _UpperCamelCase       			=max(compute_exact(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ) for a in gold_answers   )
        _UpperCamelCase       			=max(compute_fa(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ) for a in gold_answers   )
  return exact_scores, fa_scores


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  _UpperCamelCase       			={}
  for qid, s in scores.items():
    _UpperCamelCase       			=na_probs[qid] > na_prob_thresh
    if pred_na:
      _UpperCamelCase       			=float(not qid_to_has_ans[qid]   )
    else:
      _UpperCamelCase       			=s
  return new_scores


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE=None   ):


  """simple docstring"""

  if not qid_list:
    _UpperCamelCase       			=len(__SCREAMING_SNAKE_CASE   )
    return collections.OrderedDict(
        [
            ('''exact''', 1_0_0.0 * sum(exact_scores.values()   ) / total),
            ('''f1''', 1_0_0.0 * sum(fa_scores.values()   ) / total),
            ('''total''', total),
        ]   )
  else:
    _UpperCamelCase       			=len(__SCREAMING_SNAKE_CASE   )
    return collections.OrderedDict(
        [
            ('''exact''', 1_0_0.0 * sum(exact_scores[k] for k in qid_list   ) / total),
            ('''f1''', 1_0_0.0 * sum(fa_scores[k] for k in qid_list   ) / total),
            ('''total''', total),
        ]   )


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  for k in new_eval:
    _UpperCamelCase       			=new_eval[k]


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  plt.step(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      color='''b'''  ,      alpha=0.2  ,      where='''post'''   )
  plt.fill_between(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      step='''post'''  ,      alpha=0.2  ,      color='''b'''   )
  plt.xlabel('''Recall'''   )
  plt.ylabel('''Precision'''   )
  plt.xlim([0.0, 1.0_5]   )
  plt.ylim([0.0, 1.0_5]   )
  plt.title(__SCREAMING_SNAKE_CASE   )
  plt.savefig(__SCREAMING_SNAKE_CASE   )
  plt.clf()


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE=None  ,      __SCREAMING_SNAKE_CASE=None   ):


  """simple docstring"""

  _UpperCamelCase       			=sorted(__SCREAMING_SNAKE_CASE  ,      key=lambda __SCREAMING_SNAKE_CASE   : na_probs[k]   )
  _UpperCamelCase       			=0.0
  _UpperCamelCase       			=1.0
  _UpperCamelCase       			=0.0
  _UpperCamelCase       			=[1.0]
  _UpperCamelCase       			=[0.0]
  _UpperCamelCase       			=0.0
  for i, qid in enumerate(__SCREAMING_SNAKE_CASE   ):
    if qid_to_has_ans[qid]:
      true_pos += scores[qid]
    _UpperCamelCase       			=true_pos / float(i + 1   )
    _UpperCamelCase       			=true_pos / float(__SCREAMING_SNAKE_CASE   )
    if i == len(__SCREAMING_SNAKE_CASE   ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
      # i.e., if we can put a threshold after this point
      avg_prec += cur_p * (cur_r - recalls[-1])
      precisions.append(__SCREAMING_SNAKE_CASE   )
      recalls.append(__SCREAMING_SNAKE_CASE   )
  if out_image:
    plot_pr_curve(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )
  return {"ap": 1_0_0.0 * avg_prec}


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  if out_image_dir and not os.path.exists(__SCREAMING_SNAKE_CASE   ):
    os.makedirs(__SCREAMING_SNAKE_CASE   )
  _UpperCamelCase       			=sum(1 for v in qid_to_has_ans.values() if v   )
  if num_true_pos == 0:
    return
  _UpperCamelCase       			=make_precision_recall_eval(
      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      out_image=os.path.join(__SCREAMING_SNAKE_CASE  ,      '''pr_exact.png'''   )  ,      title='''Precision-Recall curve for Exact Match score'''  ,      )
  _UpperCamelCase       			=make_precision_recall_eval(
      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      out_image=os.path.join(__SCREAMING_SNAKE_CASE  ,      '''pr_f1.png'''   )  ,      title='''Precision-Recall curve for F1 score'''  ,      )
  _UpperCamelCase       			={k: float(__SCREAMING_SNAKE_CASE   ) for k, v in qid_to_has_ans.items()}
  _UpperCamelCase       			=make_precision_recall_eval(
      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      out_image=os.path.join(__SCREAMING_SNAKE_CASE  ,      '''pr_oracle.png'''   )  ,      title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''  ,      )
  merge_eval(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      '''pr_exact'''   )
  merge_eval(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      '''pr_f1'''   )
  merge_eval(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      '''pr_oracle'''   )


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  if not qid_list:
    return
  _UpperCamelCase       			=[na_probs[k] for k in qid_list]
  _UpperCamelCase       			=np.ones_like(__SCREAMING_SNAKE_CASE   ) / float(len(__SCREAMING_SNAKE_CASE   )   )
  plt.hist(__SCREAMING_SNAKE_CASE  ,      weights=__SCREAMING_SNAKE_CASE  ,      bins=20  ,      range=(0.0, 1.0)   )
  plt.xlabel('''Model probability of no-answer'''   )
  plt.ylabel('''Proportion of dataset'''   )
  plt.title(f'''Histogram of no-answer probability: {name}'''   )
  plt.savefig(os.path.join(__SCREAMING_SNAKE_CASE  ,      f'''na_prob_hist_{name}.png'''   )   )
  plt.clf()


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  _UpperCamelCase       			=sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]   )
  _UpperCamelCase       			=num_no_ans
  _UpperCamelCase       			=cur_score
  _UpperCamelCase       			=0.0
  _UpperCamelCase       			=sorted(__SCREAMING_SNAKE_CASE  ,      key=lambda __SCREAMING_SNAKE_CASE   : na_probs[k]   )
  for i, qid in enumerate(__SCREAMING_SNAKE_CASE   ):
    if qid not in scores:
      continue
    if qid_to_has_ans[qid]:
      _UpperCamelCase       			=scores[qid]
    else:
      if preds[qid]:
        _UpperCamelCase       			=-1
      else:
        _UpperCamelCase       			=0
    cur_score += diff
    if cur_score > best_score:
      _UpperCamelCase       			=cur_score
      _UpperCamelCase       			=na_probs[qid]
  return 1_0_0.0 * best_score / len(__SCREAMING_SNAKE_CASE   ), best_thresh


def 					_a   (__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   ):


  """simple docstring"""

  _UpperCamelCase			,     _UpperCamelCase       			=find_best_thresh(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )
  _UpperCamelCase			,     _UpperCamelCase       			=find_best_thresh(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )
  _UpperCamelCase       			=best_exact
  _UpperCamelCase       			=exact_thresh
  _UpperCamelCase       			=best_fa
  _UpperCamelCase       			=fa_thresh


def 					_a   ():


  """simple docstring"""

  with open(OPTS.data_file   ) as f:
    _UpperCamelCase       			=json.load(__SCREAMING_SNAKE_CASE   )
    _UpperCamelCase       			=dataset_json['''data''']
  with open(OPTS.pred_file   ) as f:
    _UpperCamelCase       			=json.load(__SCREAMING_SNAKE_CASE   )
  if OPTS.na_prob_file:
    with open(OPTS.na_prob_file   ) as f:
      _UpperCamelCase       			=json.load(__SCREAMING_SNAKE_CASE   )
  else:
    _UpperCamelCase       			={k: 0.0 for k in preds}
  _UpperCamelCase       			=make_qid_to_has_ans(__SCREAMING_SNAKE_CASE   )  # maps qid to True/False
  _UpperCamelCase       			=[k for k, v in qid_to_has_ans.items() if v]
  _UpperCamelCase       			=[k for k, v in qid_to_has_ans.items() if not v]
  _UpperCamelCase			,     _UpperCamelCase       			=get_raw_scores(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )
  _UpperCamelCase       			=apply_no_ans_threshold(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      OPTS.na_prob_thresh   )
  _UpperCamelCase       			=apply_no_ans_threshold(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      OPTS.na_prob_thresh   )
  _UpperCamelCase       			=make_eval_dict(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )
  if has_ans_qids:
    _UpperCamelCase       			=make_eval_dict(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      qid_list=__SCREAMING_SNAKE_CASE   )
    merge_eval(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      '''HasAns'''   )
  if no_ans_qids:
    _UpperCamelCase       			=make_eval_dict(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      qid_list=__SCREAMING_SNAKE_CASE   )
    merge_eval(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      '''NoAns'''   )
  if OPTS.na_prob_file:
    find_all_best_thresh(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )
  if OPTS.na_prob_file and OPTS.out_image_dir:
    run_precision_recall_analysis(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      OPTS.out_image_dir   )
    histogram_na_prob(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      OPTS.out_image_dir  ,      '''hasAns'''   )
    histogram_na_prob(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE  ,      OPTS.out_image_dir  ,      '''noAns'''   )
  if OPTS.out_file:
    with open(OPTS.out_file  ,      '''w'''   ) as f:
      json.dump(__SCREAMING_SNAKE_CASE  ,      __SCREAMING_SNAKE_CASE   )
  else:
    print(json.dumps(__SCREAMING_SNAKE_CASE  ,      indent=2   )   )


if __name__ == "__main__":
      __lowerCamelCase						: Union[str, Any]		=   parse_args()
      if OPTS.out_image_dir:
            import matplotlib

            matplotlib.use('Agg')
            import matplotlib.pyplot as plt
      main()













'''simple docstring'''






from __future__ import annotations

import sys
from collections import deque
from typing import Generic, TypeVar

__lowerCamelCase						: Optional[int]		=   TypeVar('T')







class 		UpperCAmelCase   (				Generic[T]):


  """simple docstring"""







  lowerCAmelCase_					=				42  # Cache store of keys
  lowerCAmelCase_					=				42  # References of the keys in cache
  lowerCAmelCase_					=				10  # Maximum capacity of cache


  def __init__(				self    :					Union[str, Any]							,	UpperCamelCase__    :					int  )     ->						None:
    _UpperCamelCase       			=deque()
    _UpperCamelCase       			=set()
    if not n:
      _UpperCamelCase       			=sys.maxsize
    elif n < 0:
      raise ValueError('''n should be an integer greater than 0.'''  )
    else:
      _UpperCamelCase       			=n


  def   UpperCamelCase__      (				self    :					List[str]							,	UpperCamelCase__    :					T  )     ->						None:
    if x not in self.key_reference:
      if len(self.dq_store  ) == LRUCache._MAX_CAPACITY:
        _UpperCamelCase       			=self.dq_store.pop()
        self.key_reference.remove(UpperCamelCase__  )
    else:
      self.dq_store.remove(UpperCamelCase__  )

    self.dq_store.appendleft(UpperCamelCase__  )
    self.key_reference.add(UpperCamelCase__  )


  def   UpperCamelCase__      (				self    :					Union[str, Any]  )     ->						None:
    for k in self.dq_store:
      print(UpperCamelCase__  )




  def __repr__(				self    :					str  )     ->						str:
    return F'''LRUCache({self._MAX_CAPACITY}) => {list(self.dq_store  )}'''


if __name__ == "__main__":
      import doctest

      doctest.testmod()

      __lowerCamelCase						: LRUCache[str | int]		=   LRUCache(4)
      lru_cache.refer('A')
      lru_cache.refer(2)
      lru_cache.refer(3)
      lru_cache.refer('A')
      lru_cache.refer(4)
      lru_cache.refer(5)
      lru_cache.display()

      print(lru_cache)
      assert str(lru_cache) == "LRUCache(4) => [5, 4, 'A', 3]"






