




"""simple docstring"""







import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


a      =							re.compile(R'\b(a|an|the)\b', re.UNICODE)

a      =							None





def 							lowercase					()     ->  Optional[Any]:


				'''simple docstring'''






				lowerCAmelCase  =				argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0.""" )
				parser.add_argument("""data_file"""      ,       metavar="""data.json"""      ,       help="""Input data JSON file.""" )
				parser.add_argument("""pred_file"""      ,       metavar="""pred.json"""      ,       help="""Model predictions.""" )
				parser.add_argument(
				    """--out-file"""      ,       """-o"""      ,       metavar="""eval.json"""      ,       help="""Write accuracy metrics to file (default is stdout).""" )
				parser.add_argument(
				    """--na-prob-file"""      ,       """-n"""      ,       metavar="""na_prob.json"""      ,       help="""Model estimates of probability of no answer.""" )
				parser.add_argument(
				    """--na-prob-thresh"""      ,       """-t"""      ,       type=__lowerCAmelCase      ,       default=1.0      ,       help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""      ,       )
				parser.add_argument(
				    """--out-image-dir"""      ,       """-p"""      ,       metavar="""out_images"""      ,       default=__lowerCAmelCase      ,       help="""Save precision-recall curves to directory.""" )
				parser.add_argument("""--verbose"""      ,       """-v"""      ,       action="""store_true""" )
				if len(sys.argv ) == 1:
								parser.print_help()
								sys.exit(1 )
				return parser.parse_args()





def 							lowercase					(snake_case__   : List[str] )     ->  Optional[Any]:


				'''simple docstring'''






				lowerCAmelCase  =				{}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																lowerCAmelCase  =				bool(qa["""answers"""]["""text"""] )
				return qid_to_has_ans





def 							lowercase					(snake_case__   : List[Any] )     ->  str:


				'''simple docstring'''






				def remove_articles(snake_case__   : Union[str, Any] ):
								return ARTICLES_REGEX.sub(""" """      ,       __lowerCAmelCase )

				def white_space_fix(snake_case__   : Tuple ):
								return " ".join(text.split() )

				def remove_punc(snake_case__   : Union[str, Any] ):
								lowerCAmelCase  =				set(string.punctuation )
								return "".join(ch for ch in text if ch not in exclude )

				def lower(snake_case__   : str ):
								return text.lower()

				return white_space_fix(remove_articles(remove_punc(lower(__lowerCAmelCase ) ) ) )





def 							lowercase					(snake_case__   : Optional[Any] )     ->  Any:


				'''simple docstring'''






				if not s:
								return []
				return normalize_answer(__lowerCAmelCase ).split()





def 							lowercase					(snake_case__   : Optional[Any]      ,       snake_case__   : int )     ->  Dict:


				'''simple docstring'''






				return int(normalize_answer(__lowerCAmelCase ) == normalize_answer(__lowerCAmelCase ) )





def 							lowercase					(snake_case__   : Any      ,       snake_case__   : str )     ->  Union[str, Any]:


				'''simple docstring'''






				lowerCAmelCase  =				get_tokens(__lowerCAmelCase )
				lowerCAmelCase  =				get_tokens(__lowerCAmelCase )
				lowerCAmelCase  =				collections.Counter(__lowerCAmelCase ) & collections.Counter(__lowerCAmelCase )
				lowerCAmelCase  =				sum(common.values() )
				if len(__lowerCAmelCase ) == 0 or len(__lowerCAmelCase ) == 0:
								# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
								return int(gold_toks == pred_toks )
				if num_same == 0:
								return 0
				lowerCAmelCase  =				1.0 * num_same / len(__lowerCAmelCase )
				lowerCAmelCase  =				1.0 * num_same / len(__lowerCAmelCase )
				lowerCAmelCase  =				(2 * precision * recall) / (precision + recall)
				return fa





def 							lowercase					(snake_case__   : List[Any]      ,       snake_case__   : List[Any] )     ->  Dict:


				'''simple docstring'''






				lowerCAmelCase  =				{}
				lowerCAmelCase  =				{}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																lowerCAmelCase  =				qa["""id"""]
																lowerCAmelCase  =				[t for t in qa["""answers"""]["""text"""] if normalize_answer(__lowerCAmelCase )]
																if not gold_answers:
																				# For unanswerable questions, only correct answer is empty string
																				lowerCAmelCase  =				[""""""]
																if qid not in preds:
																				print(f'''Missing prediction for {qid}''' )
																				continue
																lowerCAmelCase  =				preds[qid]
																# Take max over all gold answers
																lowerCAmelCase  =				max(compute_exact(__lowerCAmelCase      ,       __lowerCAmelCase ) for a in gold_answers )
																lowerCAmelCase  =				max(compute_fa(__lowerCAmelCase      ,       __lowerCAmelCase ) for a in gold_answers )
				return exact_scores, fa_scores





def 							lowercase					(snake_case__   : str      ,       snake_case__   : List[Any]      ,       snake_case__   : Dict      ,       snake_case__   : Tuple )     ->  int:


				'''simple docstring'''






				lowerCAmelCase  =				{}
				for qid, s in scores.items():
								lowerCAmelCase  =				na_probs[qid] > na_prob_thresh
								if pred_na:
												lowerCAmelCase  =				float(not qid_to_has_ans[qid] )
								else:
												lowerCAmelCase  =				s
				return new_scores





def 							lowercase					(snake_case__   : Dict      ,       snake_case__   : Tuple      ,       snake_case__   : Union[str, Any]=None )     ->  Union[str, Any]:


				'''simple docstring'''






				if not qid_list:
								lowerCAmelCase  =				len(__lowerCAmelCase )
								return collections.OrderedDict(
								    [
								        ("""exact""", 100.0 * sum(exact_scores.values() ) / total),
								        ("""f1""", 100.0 * sum(fa_scores.values() ) / total),
								        ("""total""", total),
								    ] )
				else:
								lowerCAmelCase  =				len(__lowerCAmelCase )
								return collections.OrderedDict(
								    [
								        ("""exact""", 100.0 * sum(exact_scores[k] for k in qid_list ) / total),
								        ("""f1""", 100.0 * sum(fa_scores[k] for k in qid_list ) / total),
								        ("""total""", total),
								    ] )





def 							lowercase					(snake_case__   : List[Any]      ,       snake_case__   : int      ,       snake_case__   : List[Any] )     ->  Dict:


				'''simple docstring'''






				for k in new_eval:
								lowerCAmelCase  =				new_eval[k]





def 							lowercase					(snake_case__   : List[str]      ,       snake_case__   : int      ,       snake_case__   : Tuple      ,       snake_case__   : Tuple )     ->  Dict:


				'''simple docstring'''






				plt.step(__lowerCAmelCase      ,       __lowerCAmelCase      ,       color="""b"""      ,       alpha=0.2      ,       where="""post""" )
				plt.fill_between(__lowerCAmelCase      ,       __lowerCAmelCase      ,       step="""post"""      ,       alpha=0.2      ,       color="""b""" )
				plt.xlabel("""Recall""" )
				plt.ylabel("""Precision""" )
				plt.xlim([0.0, 1.05] )
				plt.ylim([0.0, 1.05] )
				plt.title(__lowerCAmelCase )
				plt.savefig(__lowerCAmelCase )
				plt.clf()





def 							lowercase					(snake_case__   : Optional[int]      ,       snake_case__   : Union[str, Any]      ,       snake_case__   : List[str]      ,       snake_case__   : Tuple      ,       snake_case__   : List[Any]=None      ,       snake_case__   : Any=None )     ->  str:


				'''simple docstring'''






				lowerCAmelCase  =				sorted(__lowerCAmelCase      ,       key=lambda snake_case__ : na_probs[k] )
				lowerCAmelCase  =				0.0
				lowerCAmelCase  =				1.0
				lowerCAmelCase  =				0.0
				lowerCAmelCase  =				[1.0]
				lowerCAmelCase  =				[0.0]
				lowerCAmelCase  =				0.0
				for i, qid in enumerate(__lowerCAmelCase ):
								if qid_to_has_ans[qid]:
												true_pos += scores[qid]
								lowerCAmelCase  =				true_pos / float(i + 1 )
								lowerCAmelCase  =				true_pos / float(__lowerCAmelCase )
								if i == len(__lowerCAmelCase ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
												# i.e., if we can put a threshold after this point
												avg_prec += cur_p * (cur_r - recalls[-1])
												precisions.append(__lowerCAmelCase )
												recalls.append(__lowerCAmelCase )
				if out_image:
								plot_pr_curve(__lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase )
				return {"ap": 100.0 * avg_prec}





def 							lowercase					(snake_case__   : Optional[int]      ,       snake_case__   : Tuple      ,       snake_case__   : Any      ,       snake_case__   : Tuple      ,       snake_case__   : Any      ,       snake_case__   : List[Any] )     ->  Optional[Any]:


				'''simple docstring'''






				if out_image_dir and not os.path.exists(__lowerCAmelCase ):
								os.makedirs(__lowerCAmelCase )
				lowerCAmelCase  =				sum(1 for v in qid_to_has_ans.values() if v )
				if num_true_pos == 0:
								return
				lowerCAmelCase  =				make_precision_recall_eval(
				    __lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       out_image=os.path.join(__lowerCAmelCase      ,       """pr_exact.png""" )      ,       title="""Precision-Recall curve for Exact Match score"""      ,       )
				lowerCAmelCase  =				make_precision_recall_eval(
				    __lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       out_image=os.path.join(__lowerCAmelCase      ,       """pr_f1.png""" )      ,       title="""Precision-Recall curve for F1 score"""      ,       )
				lowerCAmelCase  =				{k: float(__lowerCAmelCase ) for k, v in qid_to_has_ans.items()}
				lowerCAmelCase  =				make_precision_recall_eval(
				    __lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       out_image=os.path.join(__lowerCAmelCase      ,       """pr_oracle.png""" )      ,       title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""      ,       )
				merge_eval(__lowerCAmelCase      ,       __lowerCAmelCase      ,       """pr_exact""" )
				merge_eval(__lowerCAmelCase      ,       __lowerCAmelCase      ,       """pr_f1""" )
				merge_eval(__lowerCAmelCase      ,       __lowerCAmelCase      ,       """pr_oracle""" )





def 							lowercase					(snake_case__   : Any      ,       snake_case__   : str      ,       snake_case__   : Optional[Any]      ,       snake_case__   : Union[str, Any] )     ->  List[str]:


				'''simple docstring'''






				if not qid_list:
								return
				lowerCAmelCase  =				[na_probs[k] for k in qid_list]
				lowerCAmelCase  =				np.ones_like(__lowerCAmelCase ) / float(len(__lowerCAmelCase ) )
				plt.hist(__lowerCAmelCase      ,       weights=__lowerCAmelCase      ,       bins=20      ,       range=(0.0, 1.0) )
				plt.xlabel("""Model probability of no-answer""" )
				plt.ylabel("""Proportion of dataset""" )
				plt.title(f'''Histogram of no-answer probability: {name}''' )
				plt.savefig(os.path.join(__lowerCAmelCase      ,       f'''na_prob_hist_{name}.png''' ) )
				plt.clf()





def 							lowercase					(snake_case__   : Tuple      ,       snake_case__   : int      ,       snake_case__   : Tuple      ,       snake_case__   : Optional[Any] )     ->  str:


				'''simple docstring'''






				lowerCAmelCase  =				sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k] )
				lowerCAmelCase  =				num_no_ans
				lowerCAmelCase  =				cur_score
				lowerCAmelCase  =				0.0
				lowerCAmelCase  =				sorted(__lowerCAmelCase      ,       key=lambda snake_case__ : na_probs[k] )
				for i, qid in enumerate(__lowerCAmelCase ):
								if qid not in scores:
												continue
								if qid_to_has_ans[qid]:
												lowerCAmelCase  =				scores[qid]
								else:
												if preds[qid]:
																lowerCAmelCase  =				-1
												else:
																lowerCAmelCase  =				0
								cur_score += diff
								if cur_score > best_score:
												lowerCAmelCase  =				cur_score
												lowerCAmelCase  =				na_probs[qid]
				return 100.0 * best_score / len(__lowerCAmelCase ), best_thresh





def 							lowercase					(snake_case__   : List[str]      ,       snake_case__   : int      ,       snake_case__   : List[Any]      ,       snake_case__   : Dict      ,       snake_case__   : str      ,       snake_case__   : Optional[Any] )     ->  int:


				'''simple docstring'''






				lowerCAmelCase  =				find_best_thresh(__lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase )
				lowerCAmelCase  =				find_best_thresh(__lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase )
				lowerCAmelCase  =				best_exact
				lowerCAmelCase  =				exact_thresh
				lowerCAmelCase  =				best_fa
				lowerCAmelCase  =				fa_thresh





def 							lowercase					()     ->  List[Any]:


				'''simple docstring'''






				with open(OPTS.data_file ) as f:
								lowerCAmelCase  =				json.load(__lowerCAmelCase )
								lowerCAmelCase  =				dataset_json["""data"""]
				with open(OPTS.pred_file ) as f:
								lowerCAmelCase  =				json.load(__lowerCAmelCase )
				if OPTS.na_prob_file:
								with open(OPTS.na_prob_file ) as f:
												lowerCAmelCase  =				json.load(__lowerCAmelCase )
				else:
								lowerCAmelCase  =				{k: 0.0 for k in preds}
				lowerCAmelCase  =				make_qid_to_has_ans(__lowerCAmelCase )  # maps qid to True/False
				lowerCAmelCase  =				[k for k, v in qid_to_has_ans.items() if v]
				lowerCAmelCase  =				[k for k, v in qid_to_has_ans.items() if not v]
				lowerCAmelCase  =				get_raw_scores(__lowerCAmelCase      ,       __lowerCAmelCase )
				lowerCAmelCase  =				apply_no_ans_threshold(__lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       OPTS.na_prob_thresh )
				lowerCAmelCase  =				apply_no_ans_threshold(__lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       OPTS.na_prob_thresh )
				lowerCAmelCase  =				make_eval_dict(__lowerCAmelCase      ,       __lowerCAmelCase )
				if has_ans_qids:
								lowerCAmelCase  =				make_eval_dict(__lowerCAmelCase      ,       __lowerCAmelCase      ,       qid_list=__lowerCAmelCase )
								merge_eval(__lowerCAmelCase      ,       __lowerCAmelCase      ,       """HasAns""" )
				if no_ans_qids:
								lowerCAmelCase  =				make_eval_dict(__lowerCAmelCase      ,       __lowerCAmelCase      ,       qid_list=__lowerCAmelCase )
								merge_eval(__lowerCAmelCase      ,       __lowerCAmelCase      ,       """NoAns""" )
				if OPTS.na_prob_file:
								find_all_best_thresh(__lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase )
				if OPTS.na_prob_file and OPTS.out_image_dir:
								run_precision_recall_analysis(__lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       __lowerCAmelCase      ,       OPTS.out_image_dir )
								histogram_na_prob(__lowerCAmelCase      ,       __lowerCAmelCase      ,       OPTS.out_image_dir      ,       """hasAns""" )
								histogram_na_prob(__lowerCAmelCase      ,       __lowerCAmelCase      ,       OPTS.out_image_dir      ,       """noAns""" )
				if OPTS.out_file:
								with open(OPTS.out_file      ,       """w""" ) as f:
												json.dump(__lowerCAmelCase      ,       __lowerCAmelCase )
				else:
								print(json.dumps(__lowerCAmelCase      ,       indent=2 ) )


if __name__ == "__main__":
		a      =							parse_args()
		if OPTS.out_image_dir:
				import matplotlib

				matplotlib.use('Agg')
				import matplotlib.pyplot as plt
		main()











"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


a      =							{
    'configuration_biogpt': ['BIOGPT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'BioGptConfig'],
    'tokenization_biogpt': ['BioGptTokenizer'],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		a      =							[
		    'BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'BioGptForCausalLM',
		    'BioGptForTokenClassification',
		    'BioGptForSequenceClassification',
		    'BioGptModel',
		    'BioGptPreTrainedModel',
		]


if TYPE_CHECKING:
		from .configuration_biogpt import BIOGPT_PRETRAINED_CONFIG_ARCHIVE_MAP, BioGptConfig
		from .tokenization_biogpt import BioGptTokenizer

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_biogpt import (
				    BIOGPT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    BioGptForCausalLM,
				    BioGptForSequenceClassification,
				    BioGptForTokenClassification,
				    BioGptModel,
				    BioGptPreTrainedModel,
				)


else:
		import sys

		a      =							_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





