





import unittest

from transformers import EsmConfig, is_torch_available
from transformers.testing_utils import TestCasePlus, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers.models.esm.modeling_esmfold import EsmForProteinFolding







class 	__lowerCAmelCase      :





   def __init__(			self       ,      snake_case       ,      snake_case=13       ,      snake_case=7       ,      snake_case=False       ,      snake_case=True       ,      snake_case=False       ,      snake_case=False       ,      snake_case=19       ,      snake_case=32       ,      snake_case=5       ,      snake_case=4       ,      snake_case=37       ,      snake_case="gelu"       ,      snake_case=0.1       ,      snake_case=0.1       ,      snake_case=512       ,      snake_case=16       ,      snake_case=2       ,      snake_case=0.02       ,      snake_case=3       ,      snake_case=4       ,      snake_case=None       ,      )					->	int:
      """simple docstring"""





      a__						:    List[str]   		=	parent
      a__						:    int   		=	batch_size
      a__						:    Union[str, Any]   		=	seq_length
      a__						:    List[str]   		=	is_training
      a__						:    Optional[int]   		=	use_input_mask
      a__						:    Dict   		=	use_token_type_ids
      a__						:    Dict   		=	use_labels
      a__						:    Union[str, Any]   		=	vocab_size
      a__						:    List[str]   		=	hidden_size
      a__						:    Optional[Any]   		=	num_hidden_layers
      a__						:    List[Any]   		=	num_attention_heads
      a__						:    Any   		=	intermediate_size
      a__						:    List[str]   		=	hidden_act
      a__						:    Optional[int]   		=	hidden_dropout_prob
      a__						:    Dict   		=	attention_probs_dropout_prob
      a__						:    Tuple   		=	max_position_embeddings
      a__						:    List[str]   		=	type_vocab_size
      a__						:    Optional[Any]   		=	type_sequence_label_size
      a__						:    Optional[int]   		=	initializer_range
      a__						:    List[Any]   		=	num_labels
      a__						:    List[Any]   		=	num_choices
      a__						:    Tuple   		=	scope





   def 						_snake_case			(			self   )					->	str:
      """simple docstring"""





      a__						:    int   		=	ids_tensor([self.batch_size, self.seq_length]       ,      self.vocab_size   )

      a__						:    List[str]   		=	None
      if self.use_input_mask:
         a__						:    str   		=	random_attention_mask([self.batch_size, self.seq_length]   )

      a__						:    List[str]   		=	None
      a__						:    str   		=	None
      a__						:    Dict   		=	None
      if self.use_labels:
         a__						:    Any   		=	ids_tensor([self.batch_size]       ,      self.type_sequence_label_size   )
         a__						:    Optional[Any]   		=	ids_tensor([self.batch_size, self.seq_length]       ,      self.num_labels   )
         a__						:    Tuple   		=	ids_tensor([self.batch_size]       ,      self.num_choices   )

      a__						:    Tuple   		=	self.get_config()

      return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels





   def 						_snake_case			(			self   )					->	List[Any]:
      """simple docstring"""





      a__						:    Dict   		=	EsmConfig(
          vocab_size=33       ,      hidden_size=self.hidden_size       ,      pad_token_id=1       ,      num_hidden_layers=self.num_hidden_layers       ,      num_attention_heads=self.num_attention_heads       ,      intermediate_size=self.intermediate_size       ,      hidden_act=self.hidden_act       ,      hidden_dropout_prob=self.hidden_dropout_prob       ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,      max_position_embeddings=self.max_position_embeddings       ,      type_vocab_size=self.type_vocab_size       ,      initializer_range=self.initializer_range       ,      is_folding_model=_A       ,      esmfold_config={"trunk": {"num_blocks": 2}, "fp16_esm": False}       ,      )
      return config





   def 						_snake_case			(			self       ,      snake_case       ,      snake_case       ,      snake_case       ,      snake_case       ,      snake_case       ,      snake_case   )					->	Union[str, Any]:
      """simple docstring"""





      a__						:    Any   		=	EsmForProteinFolding(config=_A   ).float()
      model.to(_A   )
      model.eval()
      a__						:    List[Any]   		=	model(_A       ,      attention_mask=_A   )
      a__						:    Dict   		=	model(_A   )
      a__						:    str   		=	model(_A   )

      self.parent.assertEqual(result.positions.shape       ,      (8, self.batch_size, self.seq_length, 14, 3)   )
      self.parent.assertEqual(result.angles.shape       ,      (8, self.batch_size, self.seq_length, 7, 2)   )





   def 						_snake_case			(			self   )					->	Optional[Any]:
      """simple docstring"""





      a__						:    List[str]   		=	self.prepare_config_and_inputs()
      (
          a__      
      )						:    List[str]   		=	config_and_inputs
      a__						:    Dict   		=	{'''input_ids''': input_ids, '''attention_mask''': input_mask}
      return config, inputs_dict







@require_torch
class 	__lowerCAmelCase      (       _UpperCamelCase		,_UpperCamelCase		,unittest.TestCase	):
   _UpperCamelCase			:    Any  		=					False

   _UpperCamelCase			:    int  		=					(EsmForProteinFolding,) if is_torch_available() else ()
   _UpperCamelCase			:    int  		=					()
   _UpperCamelCase			:    List[str]  		=					{} if is_torch_available() else {}
   _UpperCamelCase			:    List[Any]  		=					False





   def 						_snake_case			(			self   )					->	Tuple:
      """simple docstring"""





      a__						:    Dict   		=	EsmFoldModelTester(self   )
      a__						:    List[Any]   		=	ConfigTester(self       ,      config_class=_A       ,      hidden_size=37   )





   def 						_snake_case			(			self   )					->	Any:
      """simple docstring"""





      self.config_tester.run_common_tests()





   def 						_snake_case			(			self   )					->	List[Any]:
      """simple docstring"""





      a__						:    Tuple   		=	self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*_A   )





   @unittest.skip("Does not support attention outputs"   )
   def 						_snake_case			(			self   )					->	int:
      """simple docstring"""





      pass





   @unittest.skip
   def 						_snake_case			(			self   )					->	Tuple:
      """simple docstring"""





      pass





   @unittest.skip("Esm does not support embedding resizing"   )
   def 						_snake_case			(			self   )					->	Dict:
      """simple docstring"""





      pass





   @unittest.skip("Esm does not support embedding resizing"   )
   def 						_snake_case			(			self   )					->	Any:
      """simple docstring"""





      pass





   @unittest.skip("ESMFold does not support passing input embeds!"   )
   def 						_snake_case			(			self   )					->	int:
      """simple docstring"""





      pass





   @unittest.skip("ESMFold does not support head pruning."   )
   def 						_snake_case			(			self   )					->	List[str]:
      """simple docstring"""





      pass





   @unittest.skip("ESMFold does not support head pruning."   )
   def 						_snake_case			(			self   )					->	List[Any]:
      """simple docstring"""





      pass





   @unittest.skip("ESMFold does not support head pruning."   )
   def 						_snake_case			(			self   )					->	Any:
      """simple docstring"""





      pass





   @unittest.skip("ESMFold does not support head pruning."   )
   def 						_snake_case			(			self   )					->	Tuple:
      """simple docstring"""





      pass





   @unittest.skip("ESMFold does not support head pruning."   )
   def 						_snake_case			(			self   )					->	int:
      """simple docstring"""





      pass





   @unittest.skip("ESMFold does not output hidden states in the normal way."   )
   def 						_snake_case			(			self   )					->	List[str]:
      """simple docstring"""





      pass





   @unittest.skip("ESMfold does not output hidden states in the normal way."   )
   def 						_snake_case			(			self   )					->	List[str]:
      """simple docstring"""





      pass





   @unittest.skip("ESMFold only has one output format."   )
   def 						_snake_case			(			self   )					->	int:
      """simple docstring"""





      pass





   @unittest.skip("This test doesn\'t work for ESMFold and doesn\'t test core functionality"   )
   def 						_snake_case			(			self   )					->	Dict:
      """simple docstring"""





      pass





   @unittest.skip("ESMFold does not support input chunking."   )
   def 						_snake_case			(			self   )					->	Optional[Any]:
      """simple docstring"""





      pass





   @unittest.skip("ESMFold doesn\'t respect you and it certainly doesn\'t respect your initialization arguments."   )
   def 						_snake_case			(			self   )					->	Union[str, Any]:
      """simple docstring"""





      pass





   @unittest.skip("ESMFold doesn\'t support torchscript compilation."   )
   def 						_snake_case			(			self   )					->	Union[str, Any]:
      """simple docstring"""





      pass





   @unittest.skip("ESMFold doesn\'t support torchscript compilation."   )
   def 						_snake_case			(			self   )					->	Optional[Any]:
      """simple docstring"""





      pass





   @unittest.skip("ESMFold doesn\'t support torchscript compilation."   )
   def 						_snake_case			(			self   )					->	Any:
      """simple docstring"""





      pass





   @unittest.skip("ESMFold doesn\'t support data parallel."   )
   def 						_snake_case			(			self   )					->	List[Any]:
      """simple docstring"""





      pass





   @unittest.skip("Will be fixed soon by reducing the size of the model used for common tests."   )
   def 						_snake_case			(			self   )					->	List[Any]:
      """simple docstring"""





      pass







@require_torch
class 	__lowerCAmelCase      (       _UpperCamelCase	):





   @slow
   def 						_snake_case			(			self   )					->	int:
      """simple docstring"""





      a__						:    Any   		=	EsmForProteinFolding.from_pretrained("facebook/esmfold_v1"   ).float()
      model.eval()
      a__						:    Any   		=	torch.tensor([[0, 6, 4, 13, 5, 4, 16, 12, 11, 7, 2]]   )
      a__						:    Tuple   		=	model(_A   )['''positions''']
      a__						:    Optional[Any]   		=	torch.tensor([2.5_828, 0.7_993, -10.9_334]       ,      dtype=torch.floataa   )
      self.assertTrue(torch.allclose(position_outputs[0, 0, 0, 0]       ,      _A       ,      atol=1E-4   )   )









'''simple docstring'''





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_UpperCamelCase						:						List[str]  =					re.compile(R'\b(a|an|the)\b', re.UNICODE)

_UpperCamelCase						:						Optional[int]  =					None
def __UpperCAmelCase   (					)    ->   Optional[Any]:
			UpperCAmelCase_      :       List[Any]          =       argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''		)
			parser.add_argument('''data_file'''			,       metavar='''data.json'''			,       help='''Input data JSON file.'''		)
			parser.add_argument('''pred_file'''			,       metavar='''pred.json'''			,       help='''Model predictions.'''		)
			parser.add_argument(
			    '''--out-file'''			,       '''-o'''			,       metavar='''eval.json'''			,       help='''Write accuracy metrics to file (default is stdout).'''		)
			parser.add_argument(
			    '''--na-prob-file'''			,       '''-n'''			,       metavar='''na_prob.json'''			,       help='''Model estimates of probability of no answer.'''		)
			parser.add_argument(
			    '''--na-prob-thresh'''			,       '''-t'''			,       type=A			,       default=1.0			,       help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''			,       )
			parser.add_argument(
			    '''--out-image-dir'''			,       '''-p'''			,       metavar='''out_images'''			,       default=A			,       help='''Save precision-recall curves to directory.'''		)
			parser.add_argument('''--verbose'''			,       '''-v'''			,       action='''store_true'''		)
			if len(sys.argv		) == 1:
						parser.print_help()
						sys.exit(1		)
			return parser.parse_args()
def __UpperCAmelCase   (					A					:			int		)    ->   Optional[Any]:
			UpperCAmelCase_      :       Optional[Any]          =       {}
			for article in dataset:
						for p in article["paragraphs"]:
									for qa in p["qas"]:
												UpperCAmelCase_      :       Union[str, Any]          =       bool(qa['''answers''']['''text''']		)
			return qid_to_has_ans
def __UpperCAmelCase   (					A					:			List[Any]		)    ->   Dict:

			def remove_articles(A					:			Dict		):
						return ARTICLES_REGEX.sub(''' '''			,       A		)

			def white_space_fix(A					:			str		):
						return " ".join(text.split()		)

			def remove_punc(A					:			Tuple		):
						UpperCAmelCase_      :       Dict          =       set(string.punctuation		)
						return "".join(ch for ch in text if ch not in exclude		)

			def lower(A					:			Any		):
						return text.lower()

			return white_space_fix(remove_articles(remove_punc(lower(A		)		)		)		)
def __UpperCAmelCase   (					A					:			Optional[Any]		)    ->   Tuple:
			if not s:
						return []
			return normalize_answer(A		).split()
def __UpperCAmelCase   (					A					:			Optional[int]			,       A					:			Dict		)    ->   List[str]:
			return int(normalize_answer(A		) == normalize_answer(A		)		)
def __UpperCAmelCase   (					A					:			Optional[Any]			,       A					:			List[Any]		)    ->   str:
			UpperCAmelCase_      :       Tuple          =       get_tokens(A		)
			UpperCAmelCase_      :       Union[str, Any]          =       get_tokens(A		)
			UpperCAmelCase_      :       Dict          =       collections.Counter(A		) & collections.Counter(A		)
			UpperCAmelCase_      :       Optional[int]          =       sum(common.values()		)
			if len(A		) == 0 or len(A		) == 0:
						# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
						return int(gold_toks == pred_toks		)
			if num_same == 0:
						return 0
			UpperCAmelCase_      :       int          =       1.0 * num_same / len(A		)
			UpperCAmelCase_      :       Any          =       1.0 * num_same / len(A		)
			UpperCAmelCase_      :       Any          =       (2 * precision * recall) / (precision + recall)
			return fa
def __UpperCAmelCase   (					A					:			Union[str, Any]			,       A					:			str		)    ->   Union[str, Any]:
			UpperCAmelCase_      :       List[Any]          =       {}
			UpperCAmelCase_      :       str          =       {}
			for article in dataset:
						for p in article["paragraphs"]:
									for qa in p["qas"]:
												UpperCAmelCase_      :       Optional[Any]          =       qa['''id''']
												UpperCAmelCase_      :       List[str]          =       [t for t in qa['''answers''']['''text'''] if normalize_answer(A		)]
												if not gold_answers:
															# For unanswerable questions, only correct answer is empty string
															UpperCAmelCase_      :       Tuple          =       ['''''']
												if qid not in preds:
															print(F"Missing prediction for {qid}"		)
															continue
												UpperCAmelCase_      :       Tuple          =       preds[qid]
												# Take max over all gold answers
												UpperCAmelCase_      :       List[Any]          =       max(compute_exact(A			,       A		) for a in gold_answers		)
												UpperCAmelCase_      :       int          =       max(compute_fa(A			,       A		) for a in gold_answers		)
			return exact_scores, fa_scores
def __UpperCAmelCase   (					A					:			Optional[int]			,       A					:			Tuple			,       A					:			Optional[Any]			,       A					:			List[Any]		)    ->   Optional[int]:
			UpperCAmelCase_      :       Tuple          =       {}
			for qid, s in scores.items():
						UpperCAmelCase_      :       Union[str, Any]          =       na_probs[qid] > na_prob_thresh
						if pred_na:
									UpperCAmelCase_      :       List[str]          =       float(not qid_to_has_ans[qid]		)
						else:
									UpperCAmelCase_      :       Any          =       s
			return new_scores
def __UpperCAmelCase   (					A					:			Any			,       A					:			Any			,       A					:			str=None		)    ->   int:
			if not qid_list:
						UpperCAmelCase_      :       str          =       len(A		)
						return collections.OrderedDict(
						    [
						        ('''exact''', 100.0 * sum(exact_scores.values()		) / total),
						        ('''f1''', 100.0 * sum(fa_scores.values()		) / total),
						        ('''total''', total),
						    ]		)
			else:
						UpperCAmelCase_      :       Dict          =       len(A		)
						return collections.OrderedDict(
						    [
						        ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list		) / total),
						        ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list		) / total),
						        ('''total''', total),
						    ]		)
def __UpperCAmelCase   (					A					:			Union[str, Any]			,       A					:			Dict			,       A					:			Union[str, Any]		)    ->   Dict:
			for k in new_eval:
						UpperCAmelCase_      :       str          =       new_eval[k]
def __UpperCAmelCase   (					A					:			str			,       A					:			Tuple			,       A					:			Any			,       A					:			List[str]		)    ->   Optional[int]:
			plt.step(A			,       A			,       color='''b'''			,       alpha=0.2			,       where='''post'''		)
			plt.fill_between(A			,       A			,       step='''post'''			,       alpha=0.2			,       color='''b'''		)
			plt.xlabel('''Recall'''		)
			plt.ylabel('''Precision'''		)
			plt.xlim([0.0, 1.05]		)
			plt.ylim([0.0, 1.05]		)
			plt.title(A		)
			plt.savefig(A		)
			plt.clf()
def __UpperCAmelCase   (					A					:			Optional[Any]			,       A					:			Optional[Any]			,       A					:			Optional[int]			,       A					:			Optional[int]			,       A					:			str=None			,       A					:			Any=None		)    ->   Any:
			UpperCAmelCase_      :       Optional[Any]          =       sorted(A			,       key=lambda A		: na_probs[k]		)
			UpperCAmelCase_      :       int          =       0.0
			UpperCAmelCase_      :       List[Any]          =       1.0
			UpperCAmelCase_      :       Optional[int]          =       0.0
			UpperCAmelCase_      :       Optional[Any]          =       [1.0]
			UpperCAmelCase_      :       List[Any]          =       [0.0]
			UpperCAmelCase_      :       Dict          =       0.0
			for i, qid in enumerate(A		):
						if qid_to_has_ans[qid]:
									true_pos += scores[qid]
						UpperCAmelCase_      :       List[Any]          =       true_pos / float(i + 1		)
						UpperCAmelCase_      :       Union[str, Any]          =       true_pos / float(A		)
						if i == len(A		) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
									# i.e., if we can put a threshold after this point
									avg_prec += cur_p * (cur_r - recalls[-1])
									precisions.append(A		)
									recalls.append(A		)
			if out_image:
						plot_pr_curve(A			,       A			,       A			,       A		)
			return {"ap": 100.0 * avg_prec}
def __UpperCAmelCase   (					A					:			List[Any]			,       A					:			str			,       A					:			Any			,       A					:			Optional[int]			,       A					:			Any			,       A					:			List[str]		)    ->   List[Any]:
			if out_image_dir and not os.path.exists(A		):
						os.makedirs(A		)
			UpperCAmelCase_      :       Any          =       sum(1 for v in qid_to_has_ans.values() if v		)
			if num_true_pos == 0:
						return
			UpperCAmelCase_      :       Union[str, Any]          =       make_precision_recall_eval(
			    A			,       A			,       A			,       A			,       out_image=os.path.join(A			,       '''pr_exact.png'''		)			,       title='''Precision-Recall curve for Exact Match score'''			,       )
			UpperCAmelCase_      :       List[str]          =       make_precision_recall_eval(
			    A			,       A			,       A			,       A			,       out_image=os.path.join(A			,       '''pr_f1.png'''		)			,       title='''Precision-Recall curve for F1 score'''			,       )
			UpperCAmelCase_      :       Dict          =       {k: float(A		) for k, v in qid_to_has_ans.items()}
			UpperCAmelCase_      :       Optional[int]          =       make_precision_recall_eval(
			    A			,       A			,       A			,       A			,       out_image=os.path.join(A			,       '''pr_oracle.png'''		)			,       title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''			,       )
			merge_eval(A			,       A			,       '''pr_exact'''		)
			merge_eval(A			,       A			,       '''pr_f1'''		)
			merge_eval(A			,       A			,       '''pr_oracle'''		)
def __UpperCAmelCase   (					A					:			List[str]			,       A					:			Tuple			,       A					:			str			,       A					:			List[Any]		)    ->   Dict:
			if not qid_list:
						return
			UpperCAmelCase_      :       List[Any]          =       [na_probs[k] for k in qid_list]
			UpperCAmelCase_      :       Union[str, Any]          =       np.ones_like(A		) / float(len(A		)		)
			plt.hist(A			,       weights=A			,       bins=2_0			,       range=(0.0, 1.0)		)
			plt.xlabel('''Model probability of no-answer'''		)
			plt.ylabel('''Proportion of dataset'''		)
			plt.title(F"Histogram of no-answer probability: {name}"		)
			plt.savefig(os.path.join(A			,       F"na_prob_hist_{name}.png"		)		)
			plt.clf()
def __UpperCAmelCase   (					A					:			str			,       A					:			Dict			,       A					:			List[str]			,       A					:			int		)    ->   Optional[Any]:
			UpperCAmelCase_      :       Tuple          =       sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]		)
			UpperCAmelCase_      :       Optional[Any]          =       num_no_ans
			UpperCAmelCase_      :       int          =       cur_score
			UpperCAmelCase_      :       List[Any]          =       0.0
			UpperCAmelCase_      :       Dict          =       sorted(A			,       key=lambda A		: na_probs[k]		)
			for i, qid in enumerate(A		):
						if qid not in scores:
									continue
						if qid_to_has_ans[qid]:
									UpperCAmelCase_      :       Tuple          =       scores[qid]
						else:
									if preds[qid]:
												UpperCAmelCase_      :       Optional[int]          =       -1
									else:
												UpperCAmelCase_      :       int          =       0
						cur_score += diff
						if cur_score > best_score:
									UpperCAmelCase_      :       Dict          =       cur_score
									UpperCAmelCase_      :       Dict          =       na_probs[qid]
			return 100.0 * best_score / len(A		), best_thresh
def __UpperCAmelCase   (					A					:			int			,       A					:			List[Any]			,       A					:			Union[str, Any]			,       A					:			Tuple			,       A					:			str			,       A					:			Union[str, Any]		)    ->   Tuple:
			UpperCAmelCase_    ,      UpperCAmelCase_      :       List[Any]          =       find_best_thresh(A			,       A			,       A			,       A		)
			UpperCAmelCase_    ,      UpperCAmelCase_      :       Optional[int]          =       find_best_thresh(A			,       A			,       A			,       A		)
			UpperCAmelCase_      :       List[str]          =       best_exact
			UpperCAmelCase_      :       List[Any]          =       exact_thresh
			UpperCAmelCase_      :       Any          =       best_fa
			UpperCAmelCase_      :       str          =       fa_thresh
def __UpperCAmelCase   (					)    ->   Tuple:
			with open(OPTS.data_file		) as f:
						UpperCAmelCase_      :       Optional[int]          =       json.load(A		)
						UpperCAmelCase_      :       str          =       dataset_json['''data''']
			with open(OPTS.pred_file		) as f:
						UpperCAmelCase_      :       List[Any]          =       json.load(A		)
			if OPTS.na_prob_file:
						with open(OPTS.na_prob_file		) as f:
									UpperCAmelCase_      :       List[str]          =       json.load(A		)
			else:
						UpperCAmelCase_      :       List[str]          =       {k: 0.0 for k in preds}
			UpperCAmelCase_      :       Optional[Any]          =       make_qid_to_has_ans(A		)  # maps qid to True/False
			UpperCAmelCase_      :       Optional[int]          =       [k for k, v in qid_to_has_ans.items() if v]
			UpperCAmelCase_      :       Dict          =       [k for k, v in qid_to_has_ans.items() if not v]
			UpperCAmelCase_    ,      UpperCAmelCase_      :       List[Any]          =       get_raw_scores(A			,       A		)
			UpperCAmelCase_      :       Optional[int]          =       apply_no_ans_threshold(A			,       A			,       A			,       OPTS.na_prob_thresh		)
			UpperCAmelCase_      :       Tuple          =       apply_no_ans_threshold(A			,       A			,       A			,       OPTS.na_prob_thresh		)
			UpperCAmelCase_      :       int          =       make_eval_dict(A			,       A		)
			if has_ans_qids:
						UpperCAmelCase_      :       Optional[int]          =       make_eval_dict(A			,       A			,       qid_list=A		)
						merge_eval(A			,       A			,       '''HasAns'''		)
			if no_ans_qids:
						UpperCAmelCase_      :       Dict          =       make_eval_dict(A			,       A			,       qid_list=A		)
						merge_eval(A			,       A			,       '''NoAns'''		)
			if OPTS.na_prob_file:
						find_all_best_thresh(A			,       A			,       A			,       A			,       A			,       A		)
			if OPTS.na_prob_file and OPTS.out_image_dir:
						run_precision_recall_analysis(A			,       A			,       A			,       A			,       A			,       OPTS.out_image_dir		)
						histogram_na_prob(A			,       A			,       OPTS.out_image_dir			,       '''hasAns'''		)
						histogram_na_prob(A			,       A			,       OPTS.out_image_dir			,       '''noAns'''		)
			if OPTS.out_file:
						with open(OPTS.out_file			,       '''w'''		) as f:
									json.dump(A			,       A		)
			else:
						print(json.dumps(A			,       indent=2		)		)


if __name__ == "__main__":
						_UpperCamelCase						:						Union[str, Any]  =					parse_args()
						if OPTS.out_image_dir:
												import matplotlib

												matplotlib.use('Agg')
												import matplotlib.pyplot as plt
						main()


