






'''simple docstring'''


import unittest

from transformers import EsmConfig, is_torch_available
from transformers.testing_utils import TestCasePlus, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import EsmForMaskedLM, EsmForSequenceClassification, EsmForTokenClassification, EsmModel
					from transformers.models.esm.modeling_esm import (
					    ESM_PRETRAINED_MODEL_ARCHIVE_LIST,
					    EsmEmbeddings,
					    create_position_ids_from_input_ids,
					)
class    _SCREAMING_SNAKE_CASE							:


			'''simple docstring'''





			def __init__(   self							:						List[str] ,      lowercase							:						Union[str, Any] ,      lowercase							:						Optional[Any]=1_3 ,      lowercase							:						List[str]=7 ,      lowercase							:						int=False ,      lowercase							:						List[str]=True ,      lowercase							:						Optional[Any]=False ,      lowercase							:						Union[str, Any]=True ,      lowercase							:						List[Any]=3_3 ,      lowercase							:						Optional[int]=3_2 ,      lowercase							:						Dict=5 ,      lowercase							:						Any=4 ,      lowercase							:						List[str]=3_7 ,      lowercase							:						Any="gelu" ,      lowercase							:						Union[str, Any]=0.1 ,      lowercase							:						Any=0.1 ,      lowercase							:						List[Any]=5_1_2 ,      lowercase							:						Union[str, Any]=1_6 ,      lowercase							:						Optional[Any]=2 ,      lowercase							:						str=0.0_2 ,      lowercase							:						List[Any]=3 ,      lowercase							:						Union[str, Any]=4 ,      lowercase							:						Union[str, Any]=None ,      )							->    Any:

									'''simple docstring'''







									UpperCamelCase__         = parent
									UpperCamelCase__         = batch_size
									UpperCamelCase__         = seq_length
									UpperCamelCase__         = is_training
									UpperCamelCase__         = use_input_mask
									UpperCamelCase__         = use_token_type_ids
									UpperCamelCase__         = use_labels
									UpperCamelCase__         = vocab_size
									UpperCamelCase__         = hidden_size
									UpperCamelCase__         = num_hidden_layers
									UpperCamelCase__         = num_attention_heads
									UpperCamelCase__         = intermediate_size
									UpperCamelCase__         = hidden_act
									UpperCamelCase__         = hidden_dropout_prob
									UpperCamelCase__         = attention_probs_dropout_prob
									UpperCamelCase__         = max_position_embeddings
									UpperCamelCase__         = type_vocab_size
									UpperCamelCase__         = type_sequence_label_size
									UpperCamelCase__         = initializer_range
									UpperCamelCase__         = num_labels
									UpperCamelCase__         = num_choices
									UpperCamelCase__         = scope






			def 				A      (   self							:						Dict			)							->    List[str]:

									'''simple docstring'''







									UpperCamelCase__         = ids_tensor([self.batch_size, self.seq_length] ,      self.vocab_size			)

									UpperCamelCase__         = None
									if self.use_input_mask:
															UpperCamelCase__         = random_attention_mask([self.batch_size, self.seq_length]			)

									UpperCamelCase__         = None
									UpperCamelCase__         = None
									UpperCamelCase__         = None
									if self.use_labels:
															UpperCamelCase__         = ids_tensor([self.batch_size] ,      self.type_sequence_label_size			)
															UpperCamelCase__         = ids_tensor([self.batch_size, self.seq_length] ,      self.num_labels			)
															UpperCamelCase__         = ids_tensor([self.batch_size] ,      self.num_choices			)

									UpperCamelCase__         = self.get_config()

									return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels






			def 				A      (   self							:						Optional[Any]			)							->    Optional[Any]:

									'''simple docstring'''







									return EsmConfig(
									    vocab_size=self.vocab_size ,      hidden_size=self.hidden_size ,      pad_token_id=1 ,      num_hidden_layers=self.num_hidden_layers ,      num_attention_heads=self.num_attention_heads ,      intermediate_size=self.intermediate_size ,      hidden_act=self.hidden_act ,      hidden_dropout_prob=self.hidden_dropout_prob ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob ,      max_position_embeddings=self.max_position_embeddings ,      type_vocab_size=self.type_vocab_size ,      initializer_range=self.initializer_range ,      )






			def 				A      (   self							:						Optional[int] ,      lowercase							:						Optional[Any] ,      lowercase							:						str ,      lowercase							:						int ,      lowercase							:						Optional[int] ,      lowercase							:						str ,      lowercase							:						List[Any]			)							->    str:

									'''simple docstring'''







									UpperCamelCase__         = EsmModel(config=lowercase			)
									model.to(lowercase			)
									model.eval()
									UpperCamelCase__         = model(lowercase ,      attention_mask=lowercase			)
									UpperCamelCase__         = model(lowercase			)
									UpperCamelCase__         = model(lowercase			)

									self.parent.assertEqual(result.last_hidden_state.shape ,      (self.batch_size, self.seq_length, self.hidden_size)			)
									self.parent.assertEqual(result.pooler_output.shape ,      (self.batch_size, self.hidden_size)			)






			def 				A      (   self							:						Dict ,      lowercase							:						Dict ,      lowercase							:						str ,      lowercase							:						Optional[Any] ,      lowercase							:						Dict ,      lowercase							:						str ,      lowercase							:						Optional[Any]			)							->    Tuple:

									'''simple docstring'''







									UpperCamelCase__         = EsmForMaskedLM(config=lowercase			)
									model.to(lowercase			)
									model.eval()
									UpperCamelCase__         = model(lowercase ,      attention_mask=lowercase ,      labels=lowercase			)
									self.parent.assertEqual(result.logits.shape ,      (self.batch_size, self.seq_length, self.vocab_size)			)






			def 				A      (   self							:						str ,      lowercase							:						Dict ,      lowercase							:						int ,      lowercase							:						Any ,      lowercase							:						Optional[Any] ,      lowercase							:						List[Any] ,      lowercase							:						Dict			)							->    Union[str, Any]:

									'''simple docstring'''







									UpperCamelCase__         = self.num_labels
									UpperCamelCase__         = EsmForTokenClassification(config=lowercase			)
									model.to(lowercase			)
									model.eval()
									UpperCamelCase__         = model(lowercase ,      attention_mask=lowercase ,      labels=lowercase			)
									self.parent.assertEqual(result.logits.shape ,      (self.batch_size, self.seq_length, self.num_labels)			)







			def 				A      (   self							:						int			)							->    Optional[Any]:

									'''simple docstring'''







									UpperCamelCase__         = self.prepare_config_and_inputs()
									(
									    (
									    UpperCamelCase__   
									)    ,   (
									    UpperCamelCase__   
									)    ,   (
									    UpperCamelCase__   
									)    ,   (
									    UpperCamelCase__   
									)    ,   (
									    UpperCamelCase__   
									)    ,   (
									    UpperCamelCase__   
									)    ,      
									)         = config_and_inputs
									UpperCamelCase__         = {"""input_ids""": input_ids, """attention_mask""": input_mask}
									return config, inputs_dict
@require_torch
class    _SCREAMING_SNAKE_CASE							(		SCREAMING_SNAKE_CASE       ,SCREAMING_SNAKE_CASE       ,unittest.TestCase	):


			'''simple docstring'''





			__a							:							Optional[Any]					     =					False

			__a							:							Optional[Any]					     =					(
			    (
			        EsmForMaskedLM,
			        EsmModel,
			        EsmForSequenceClassification,
			        EsmForTokenClassification,
			    )
			    if is_torch_available()
			    else ()
			)
			__a							:							Optional[Any]					     =					()
			__a							:							Dict					     =					(
			    {
			        "feature-extraction": EsmModel,
			        "fill-mask": EsmForMaskedLM,
			        "text-classification": EsmForSequenceClassification,
			        "token-classification": EsmForTokenClassification,
			        "zero-shot": EsmForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			__a							:							int					     =					True






			def 				A      (   self							:						Tuple			)							->    str:

									'''simple docstring'''







									UpperCamelCase__         = EsmModelTester(self			)
									UpperCamelCase__         = ConfigTester(self ,      config_class=lowercase ,      hidden_size=3_7			)






			def 				A      (   self							:						Union[str, Any]			)							->    Optional[int]:

									'''simple docstring'''







									self.config_tester.run_common_tests()






			def 				A      (   self							:						Union[str, Any]			)							->    Union[str, Any]:

									'''simple docstring'''







									UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*lowercase			)






			def 				A      (   self							:						Tuple			)							->    List[str]:

									'''simple docstring'''







									UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
									for type in ["absolute", "relative_key", "relative_key_query"]:
															UpperCamelCase__         = type
															self.model_tester.create_and_check_model(*lowercase			)






			def 				A      (   self							:						List[Any]			)							->    int:

									'''simple docstring'''







									UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_masked_lm(*lowercase			)






			def 				A      (   self							:						List[Any]			)							->    Tuple:

									'''simple docstring'''







									UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_token_classification(*lowercase			)






			@slow
			def 				A      (   self							:						Optional[int]			)							->    Optional[int]:

									'''simple docstring'''







									for model_name in ESM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															UpperCamelCase__         = EsmModel.from_pretrained(lowercase			)
															self.assertIsNotNone(lowercase			)






			def 				A      (   self							:						Any			)							->    List[str]:

									'''simple docstring'''







									UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()[0]
									UpperCamelCase__         = EsmEmbeddings(config=lowercase			)

									UpperCamelCase__         = torch.as_tensor([[1_2, 3_1, 1_3, model.padding_idx]]			)
									UpperCamelCase__         = torch.as_tensor(
									    [
									        [
									            0 + model.padding_idx + 1,
									            1 + model.padding_idx + 1,
									            2 + model.padding_idx + 1,
									            model.padding_idx,
									        ]
									    ]			)
									UpperCamelCase__         = create_position_ids_from_input_ids(lowercase ,      model.padding_idx			)
									self.assertEqual(position_ids.shape ,      expected_positions.shape			)
									self.assertTrue(torch.all(torch.eq(lowercase ,      lowercase			)			)			)






			def 				A      (   self							:						str			)							->    Union[str, Any]:

									'''simple docstring'''







									UpperCamelCase__         = self.model_tester.prepare_config_and_inputs()[0]
									UpperCamelCase__         = EsmEmbeddings(config=lowercase			)

									UpperCamelCase__         = torch.empty(2 ,      4 ,      3_0			)
									UpperCamelCase__         = [
									    0 + embeddings.padding_idx + 1,
									    1 + embeddings.padding_idx + 1,
									    2 + embeddings.padding_idx + 1,
									    3 + embeddings.padding_idx + 1,
									]
									UpperCamelCase__         = torch.as_tensor([expected_single_positions, expected_single_positions]			)
									UpperCamelCase__         = embeddings.create_position_ids_from_inputs_embeds(lowercase			)
									self.assertEqual(position_ids.shape ,      expected_positions.shape			)
									self.assertTrue(torch.all(torch.eq(lowercase ,      lowercase			)			)			)






			@unittest.skip("""Esm does not support embedding resizing"""			)
			def 				A      (   self							:						List[Any]			)							->    Optional[Any]:

									'''simple docstring'''







									pass






			@unittest.skip("""Esm does not support embedding resizing"""			)
			def 				A      (   self							:						Optional[Any]			)							->    int:

									'''simple docstring'''







									pass







			@unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests."""			)
			def 				A      (   self							:						int			)							->    Tuple:

									'''simple docstring'''







									pass
@require_torch
class    _SCREAMING_SNAKE_CASE							(		SCREAMING_SNAKE_CASE	):


			'''simple docstring'''





			@slow
			def 				A      (   self							:						Tuple			)							->    List[str]:

									'''simple docstring'''







									with torch.no_grad():
															UpperCamelCase__         = EsmForMaskedLM.from_pretrained("""facebook/esm2_t6_8M_UR50D"""			)
															model.eval()
															UpperCamelCase__         = torch.tensor([[0, 1, 2, 3, 4, 5]]			)
															UpperCamelCase__         = model(lowercase			)[0]

															UpperCamelCase__         = 3_3

															UpperCamelCase__         = torch.Size((1, 6, vocab_size)			)
															self.assertEqual(output.shape ,      lowercase			)

															UpperCamelCase__         = torch.tensor(
															    [[[8.9_2_1_5, -1_0.5_8_9_8, -6.4_6_7_1], [-6.3_9_6_7, -1_3.9_1_1_4, -1.1_2_1_2], [-7.7_8_1_2, -1_3.9_5_1_6, -3.7_4_0_6]]]			)
															self.assertTrue(torch.allclose(output[:, :3, :3] ,      lowercase ,      atol=1e-4			)			)







			@slow
			def 				A      (   self							:						Optional[int]			)							->    Optional[Any]:

									'''simple docstring'''







									with torch.no_grad():
															UpperCamelCase__         = EsmModel.from_pretrained("""facebook/esm2_t6_8M_UR50D"""			)
															model.eval()

															UpperCamelCase__         = torch.tensor([[0, 6, 4, 1_3, 5, 4, 1_6, 1_2, 1_1, 7, 2]]			)
															UpperCamelCase__         = model(lowercase			)[0]
															# compare the actual values for a slice.
															UpperCamelCase__         = torch.tensor(
															    [[[0.1_4_4_4, 0.5_4_1_3, 0.3_2_4_8], [0.3_0_3_4, 0.0_0_5_3, 0.3_1_0_8], [0.3_2_2_8, -0.2_4_9_9, 0.3_4_1_5]]]			)
															self.assertTrue(torch.allclose(output[:, :3, :3] ,      lowercase ,      atol=1e-4			)			)














'''simple docstring'''


import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCamelCase_				: Union[str, Any]    	=							re.compile(r'''\b(a|an|the)\b''', re.UNICODE)

lowerCamelCase_				: int    	=							None







def 	__magic_name__( ):
						'''simple docstring'''





						UpperCamelCase__         = argparse.ArgumentParser("""Official evaluation script for SQuAD version 2.0."""					)
						parser.add_argument("""data_file"""						,      metavar="""data.json"""						,      help="""Input data JSON file."""					)
						parser.add_argument("""pred_file"""						,      metavar="""pred.json"""						,      help="""Model predictions."""					)
						parser.add_argument(
						    """--out-file"""						,      """-o"""						,      metavar="""eval.json"""						,      help="""Write accuracy metrics to file (default is stdout)."""					)
						parser.add_argument(
						    """--na-prob-file"""						,      """-n"""						,      metavar="""na_prob.json"""						,      help="""Model estimates of probability of no answer."""					)
						parser.add_argument(
						    """--na-prob-thresh"""						,      """-t"""						,      type=_A						,      default=1.0						,      help="""Predict \"\" if no-answer probability exceeds this (default = 1.0)."""						,      )
						parser.add_argument(
						    """--out-image-dir"""						,      """-p"""						,      metavar="""out_images"""						,      default=_A						,      help="""Save precision-recall curves to directory."""					)
						parser.add_argument("""--verbose"""						,      """-v"""						,      action="""store_true"""					)
						if len(sys.argv					) == 1:
												parser.print_help()
												sys.exit(1					)
						return parser.parse_args()







def 	__magic_name__( _A					):
						'''simple docstring'''





						UpperCamelCase__         = {}
						for article in dataset:
												for p in article["paragraphs"]:
																		for qa in p["qas"]:
																								UpperCamelCase__         = bool(qa["""answers"""]["""text"""]					)
						return qid_to_has_ans







def 	__magic_name__( _A					):
						'''simple docstring'''





						def remove_articles(_A					):
												return ARTICLES_REGEX.sub(""" """						,      _A					)

						def white_space_fix(_A					):
												return " ".join(text.split()					)

						def remove_punc(_A					):
												UpperCamelCase__         = set(string.punctuation					)
												return "".join(ch for ch in text if ch not in exclude					)

						def lower(_A					):
												return text.lower()

						return white_space_fix(remove_articles(remove_punc(lower(_A					)					)					)					)







def 	__magic_name__( _A					):
						'''simple docstring'''





						if not s:
												return []
						return normalize_answer(_A					).split()







def 	__magic_name__( _A						,      _A					):
						'''simple docstring'''





						return int(normalize_answer(_A					) == normalize_answer(_A					)					)







def 	__magic_name__( _A						,      _A					):
						'''simple docstring'''





						UpperCamelCase__         = get_tokens(_A					)
						UpperCamelCase__         = get_tokens(_A					)
						UpperCamelCase__         = collections.Counter(_A					) & collections.Counter(_A					)
						UpperCamelCase__         = sum(common.values()					)
						if len(_A					) == 0 or len(_A					) == 0:
												# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
												return int(gold_toks == pred_toks					)
						if num_same == 0:
												return 0
						UpperCamelCase__         = 1.0 * num_same / len(_A					)
						UpperCamelCase__         = 1.0 * num_same / len(_A					)
						UpperCamelCase__         = (2 * precision * recall) / (precision + recall)
						return fa







def 	__magic_name__( _A						,      _A					):
						'''simple docstring'''





						UpperCamelCase__         = {}
						UpperCamelCase__         = {}
						for article in dataset:
												for p in article["paragraphs"]:
																		for qa in p["qas"]:
																								UpperCamelCase__         = qa["""id"""]
																								UpperCamelCase__         = [t for t in qa["""answers"""]["""text"""] if normalize_answer(_A					)]
																								if not gold_answers:
																														# For unanswerable questions, only correct answer is empty string
																														UpperCamelCase__         = [""""""]
																								if qid not in preds:
																														print(f"Missing prediction for {qid}"					)
																														continue
																								UpperCamelCase__         = preds[qid]
																								# Take max over all gold answers
																								UpperCamelCase__         = max(compute_exact(_A						,      _A					) for a in gold_answers					)
																								UpperCamelCase__         = max(compute_fa(_A						,      _A					) for a in gold_answers					)
						return exact_scores, fa_scores







def 	__magic_name__( _A						,      _A						,      _A						,      _A					):
						'''simple docstring'''





						UpperCamelCase__         = {}
						for qid, s in scores.items():
												UpperCamelCase__         = na_probs[qid] > na_prob_thresh
												if pred_na:
																		UpperCamelCase__         = float(not qid_to_has_ans[qid]					)
												else:
																		UpperCamelCase__         = s
						return new_scores







def 	__magic_name__( _A						,      _A						,      _A=None					):
						'''simple docstring'''





						if not qid_list:
												UpperCamelCase__         = len(_A					)
												return collections.OrderedDict(
												    [
												        ("""exact""", 1_0_0.0 * sum(exact_scores.values()					) / total),
												        ("""f1""", 1_0_0.0 * sum(fa_scores.values()					) / total),
												        ("""total""", total),
												    ]					)
						else:
												UpperCamelCase__         = len(_A					)
												return collections.OrderedDict(
												    [
												        ("""exact""", 1_0_0.0 * sum(exact_scores[k] for k in qid_list					) / total),
												        ("""f1""", 1_0_0.0 * sum(fa_scores[k] for k in qid_list					) / total),
												        ("""total""", total),
												    ]					)







def 	__magic_name__( _A						,      _A						,      _A					):
						'''simple docstring'''





						for k in new_eval:
												UpperCamelCase__         = new_eval[k]







def 	__magic_name__( _A						,      _A						,      _A						,      _A					):
						'''simple docstring'''





						plt.step(_A						,      _A						,      color="""b"""						,      alpha=0.2						,      where="""post"""					)
						plt.fill_between(_A						,      _A						,      step="""post"""						,      alpha=0.2						,      color="""b"""					)
						plt.xlabel("""Recall"""					)
						plt.ylabel("""Precision"""					)
						plt.xlim([0.0, 1.0_5]					)
						plt.ylim([0.0, 1.0_5]					)
						plt.title(_A					)
						plt.savefig(_A					)
						plt.clf()







def 	__magic_name__( _A						,      _A						,      _A						,      _A						,      _A=None						,      _A=None					):
						'''simple docstring'''





						UpperCamelCase__         = sorted(_A						,      key=lambda _A					: na_probs[k]					)
						UpperCamelCase__         = 0.0
						UpperCamelCase__         = 1.0
						UpperCamelCase__         = 0.0
						UpperCamelCase__         = [1.0]
						UpperCamelCase__         = [0.0]
						UpperCamelCase__         = 0.0
						for i, qid in enumerate(_A					):
												if qid_to_has_ans[qid]:
																		true_pos += scores[qid]
												UpperCamelCase__         = true_pos / float(i + 1					)
												UpperCamelCase__         = true_pos / float(_A					)
												if i == len(_A					) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
																		# i.e., if we can put a threshold after this point
																		avg_prec += cur_p * (cur_r - recalls[-1])
																		precisions.append(_A					)
																		recalls.append(_A					)
						if out_image:
												plot_pr_curve(_A						,      _A						,      _A						,      _A					)
						return {"ap": 1_0_0.0 * avg_prec}







def 	__magic_name__( _A						,      _A						,      _A						,      _A						,      _A						,      _A					):
						'''simple docstring'''





						if out_image_dir and not os.path.exists(_A					):
												os.makedirs(_A					)
						UpperCamelCase__         = sum(1 for v in qid_to_has_ans.values() if v					)
						if num_true_pos == 0:
												return
						UpperCamelCase__         = make_precision_recall_eval(
						    _A						,      _A						,      _A						,      _A						,      out_image=os.path.join(_A						,      """pr_exact.png"""					)						,      title="""Precision-Recall curve for Exact Match score"""						,      )
						UpperCamelCase__         = make_precision_recall_eval(
						    _A						,      _A						,      _A						,      _A						,      out_image=os.path.join(_A						,      """pr_f1.png"""					)						,      title="""Precision-Recall curve for F1 score"""						,      )
						UpperCamelCase__         = {k: float(_A					) for k, v in qid_to_has_ans.items()}
						UpperCamelCase__         = make_precision_recall_eval(
						    _A						,      _A						,      _A						,      _A						,      out_image=os.path.join(_A						,      """pr_oracle.png"""					)						,      title="""Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"""						,      )
						merge_eval(_A						,      _A						,      """pr_exact"""					)
						merge_eval(_A						,      _A						,      """pr_f1"""					)
						merge_eval(_A						,      _A						,      """pr_oracle"""					)







def 	__magic_name__( _A						,      _A						,      _A						,      _A					):
						'''simple docstring'''





						if not qid_list:
												return
						UpperCamelCase__         = [na_probs[k] for k in qid_list]
						UpperCamelCase__         = np.ones_like(_A					) / float(len(_A					)					)
						plt.hist(_A						,      weights=_A						,      bins=20						,      range=(0.0, 1.0)					)
						plt.xlabel("""Model probability of no-answer"""					)
						plt.ylabel("""Proportion of dataset"""					)
						plt.title(f"Histogram of no-answer probability: {name}"					)
						plt.savefig(os.path.join(_A						,      f"na_prob_hist_{name}.png"					)					)
						plt.clf()







def 	__magic_name__( _A						,      _A						,      _A						,      _A					):
						'''simple docstring'''





						UpperCamelCase__         = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]					)
						UpperCamelCase__         = num_no_ans
						UpperCamelCase__         = cur_score
						UpperCamelCase__         = 0.0
						UpperCamelCase__         = sorted(_A						,      key=lambda _A					: na_probs[k]					)
						for i, qid in enumerate(_A					):
												if qid not in scores:
																		continue
												if qid_to_has_ans[qid]:
																		UpperCamelCase__         = scores[qid]
												else:
																		if preds[qid]:
																								UpperCamelCase__         = -1
																		else:
																								UpperCamelCase__         = 0
												cur_score += diff
												if cur_score > best_score:
																		UpperCamelCase__         = cur_score
																		UpperCamelCase__         = na_probs[qid]
						return 1_0_0.0 * best_score / len(_A					), best_thresh







def 	__magic_name__( _A						,      _A						,      _A						,      _A						,      _A						,      _A					):
						'''simple docstring'''





						UpperCamelCase__    ,   UpperCamelCase__         = find_best_thresh(_A						,      _A						,      _A						,      _A					)
						UpperCamelCase__    ,   UpperCamelCase__         = find_best_thresh(_A						,      _A						,      _A						,      _A					)
						UpperCamelCase__         = best_exact
						UpperCamelCase__         = exact_thresh
						UpperCamelCase__         = best_fa
						UpperCamelCase__         = fa_thresh







def 	__magic_name__( ):
						'''simple docstring'''





						with open(OPTS.data_file					) as f:
												UpperCamelCase__         = json.load(_A					)
												UpperCamelCase__         = dataset_json["""data"""]
						with open(OPTS.pred_file					) as f:
												UpperCamelCase__         = json.load(_A					)
						if OPTS.na_prob_file:
												with open(OPTS.na_prob_file					) as f:
																		UpperCamelCase__         = json.load(_A					)
						else:
												UpperCamelCase__         = {k: 0.0 for k in preds}
						UpperCamelCase__         = make_qid_to_has_ans(_A					)  # maps qid to True/False
						UpperCamelCase__         = [k for k, v in qid_to_has_ans.items() if v]
						UpperCamelCase__         = [k for k, v in qid_to_has_ans.items() if not v]
						UpperCamelCase__    ,   UpperCamelCase__         = get_raw_scores(_A						,      _A					)
						UpperCamelCase__         = apply_no_ans_threshold(_A						,      _A						,      _A						,      OPTS.na_prob_thresh					)
						UpperCamelCase__         = apply_no_ans_threshold(_A						,      _A						,      _A						,      OPTS.na_prob_thresh					)
						UpperCamelCase__         = make_eval_dict(_A						,      _A					)
						if has_ans_qids:
												UpperCamelCase__         = make_eval_dict(_A						,      _A						,      qid_list=_A					)
												merge_eval(_A						,      _A						,      """HasAns"""					)
						if no_ans_qids:
												UpperCamelCase__         = make_eval_dict(_A						,      _A						,      qid_list=_A					)
												merge_eval(_A						,      _A						,      """NoAns"""					)
						if OPTS.na_prob_file:
												find_all_best_thresh(_A						,      _A						,      _A						,      _A						,      _A						,      _A					)
						if OPTS.na_prob_file and OPTS.out_image_dir:
												run_precision_recall_analysis(_A						,      _A						,      _A						,      _A						,      _A						,      OPTS.out_image_dir					)
												histogram_na_prob(_A						,      _A						,      OPTS.out_image_dir						,      """hasAns"""					)
												histogram_na_prob(_A						,      _A						,      OPTS.out_image_dir						,      """noAns"""					)
						if OPTS.out_file:
												with open(OPTS.out_file						,      """w"""					) as f:
																		json.dump(_A						,      _A					)
						else:
												print(json.dumps(_A						,      indent=2					)					)


if __name__ == "__main__":
					lowerCamelCase_				: str    	=							parse_args()
					if OPTS.out_image_dir:
										import matplotlib

										matplotlib.use('''Agg''')
										import matplotlib.pyplot as plt
					main()
