





import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipaConfig,
    BlipaForConditionalGeneration,
    BlipaProcessor,
    BlipaVisionConfig,
    BlipImageProcessor,
    OPTConfig,
    TaConfig,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD






def    _A	(   ):
   a__						:    Union[str, Any]   		=	'''https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png'''
   a__						:    Tuple   		=	Image.open(requests.get(lowerCamelCase				,			stream=lowerCamelCase ).raw ).convert("RGB" )

   return image






def    _A	(   lowerCamelCase ):
   a__						:    str   		=	[]
   # fmt: off

   # vision encoder
   rename_keys.append(("visual_encoder.cls_token", "vision_model.embeddings.class_embedding") )
   rename_keys.append(("visual_encoder.pos_embed", "vision_model.embeddings.position_embedding") )
   rename_keys.append(("visual_encoder.patch_embed.proj.weight", "vision_model.embeddings.patch_embedding.weight") )
   rename_keys.append(("visual_encoder.patch_embed.proj.bias", "vision_model.embeddings.patch_embedding.bias") )
   rename_keys.append(("ln_vision.weight", "vision_model.post_layernorm.weight") )
   rename_keys.append(("ln_vision.bias", "vision_model.post_layernorm.bias") )

   for i in range(config.vision_config.num_hidden_layers ):
      rename_keys.append((F"""visual_encoder.blocks.{i}.norm1.weight""", F"""vision_model.encoder.layers.{i}.layer_norm1.weight""") )
      rename_keys.append((F"""visual_encoder.blocks.{i}.norm1.bias""", F"""vision_model.encoder.layers.{i}.layer_norm1.bias""") )
      rename_keys.append((F"""visual_encoder.blocks.{i}.norm2.weight""", F"""vision_model.encoder.layers.{i}.layer_norm2.weight""") )
      rename_keys.append((F"""visual_encoder.blocks.{i}.norm2.bias""", F"""vision_model.encoder.layers.{i}.layer_norm2.bias""") )
      rename_keys.append((F"""visual_encoder.blocks.{i}.attn.qkv.weight""", F"""vision_model.encoder.layers.{i}.self_attn.qkv.weight""") )
      rename_keys.append((F"""visual_encoder.blocks.{i}.attn.proj.weight""", F"""vision_model.encoder.layers.{i}.self_attn.projection.weight""",) )
      rename_keys.append((F"""visual_encoder.blocks.{i}.attn.proj.bias""", F"""vision_model.encoder.layers.{i}.self_attn.projection.bias""") )
      rename_keys.append((F"""visual_encoder.blocks.{i}.mlp.fc1.weight""", F"""vision_model.encoder.layers.{i}.mlp.fc1.weight""") )
      rename_keys.append((F"""visual_encoder.blocks.{i}.mlp.fc1.bias""", F"""vision_model.encoder.layers.{i}.mlp.fc1.bias""") )
      rename_keys.append((F"""visual_encoder.blocks.{i}.mlp.fc2.weight""", F"""vision_model.encoder.layers.{i}.mlp.fc2.weight""") )
      rename_keys.append((F"""visual_encoder.blocks.{i}.mlp.fc2.bias""", F"""vision_model.encoder.layers.{i}.mlp.fc2.bias""") )

   # QFormer
   rename_keys.append(("Qformer.bert.embeddings.LayerNorm.weight", "qformer.layernorm.weight") )
   rename_keys.append(("Qformer.bert.embeddings.LayerNorm.bias", "qformer.layernorm.bias") )

   # fmt: on
   return rename_keys






def    _A	(   lowerCamelCase				,			lowerCamelCase				,			lowerCamelCase ):
   a__						:    Union[str, Any]   		=	dct.pop(lowerCamelCase )
   a__						:    int   		=	val






def    _A	(   lowerCamelCase				,			lowerCamelCase ):
   for i in range(config.vision_config.num_hidden_layers ):
      # read in original q and v biases
      a__						:    Optional[Any]   		=	state_dict.pop(F"""visual_encoder.blocks.{i}.attn.q_bias""" )
      a__						:    Any   		=	state_dict.pop(F"""visual_encoder.blocks.{i}.attn.v_bias""" )

      # next, set bias in the state dict
      a__						:    Any   		=	torch.cat((q_bias, torch.zeros_like(lowerCamelCase				,			requires_grad=lowerCamelCase ), v_bias) )
      a__						:    List[Any]   		=	qkv_bias






def    _A	(   lowerCamelCase				,			lowerCamelCase ):
   a__						:    str   		=	364 if '''coco''' in model_name else 224
   a__						:    Dict   		=	BlipaVisionConfig(image_size=lowerCamelCase ).to_dict()

   # make sure the models have proper bos_token_id and eos_token_id set (important for generation)
   # seems like flan-T5 models don't have bos_token_id properly set?
   if "opt-2.7b" in model_name:
      a__						:    Any   		=	OPTConfig.from_pretrained("facebook/opt-2.7b"				,			eos_token_id=lowerCamelCase ).to_dict()
   elif "opt-6.7b" in model_name:
      a__						:    Dict   		=	OPTConfig.from_pretrained("facebook/opt-6.7b"				,			eos_token_id=lowerCamelCase ).to_dict()
   elif "t5-xl" in model_name:
      a__						:    Dict   		=	TaConfig.from_pretrained("google/flan-t5-xl"				,			dense_act_fn="gelu"				,			bos_token_id=1 ).to_dict()
   elif "t5-xxl" in model_name:
      a__						:    str   		=	TaConfig.from_pretrained("google/flan-t5-xxl"				,			dense_act_fn="gelu"				,			bos_token_id=1 ).to_dict()

   a__						:    Any   		=	BlipaConfig(vision_config=lowerCamelCase				,			text_config=lowerCamelCase )

   return config, image_size






@torch.no_grad()
def    _A	(   lowerCamelCase				,			lowerCamelCase=None				,			lowerCamelCase=False ):
   a__						:    Any   		=	(
       AutoTokenizer.from_pretrained("facebook/opt-2.7b" )
       if '''opt''' in model_name
       else AutoTokenizer.from_pretrained("google/flan-t5-xl" )
   )
   a__						:    Optional[int]   		=	tokenizer("\n"				,			add_special_tokens=lowerCamelCase ).input_ids[0]
   a__						:    int   		=	get_blipa_config(lowerCamelCase				,			eos_token_id=lowerCamelCase )

   a__						:    List[str]   		=	BlipaForConditionalGeneration(lowerCamelCase ).eval()

   a__						:    List[str]   		=	{
       '''blip2-opt-2.7b''': ('''blip2_opt''', '''pretrain_opt2.7b'''),
       '''blip2-opt-6.7b''': ('''blip2_opt''', '''pretrain_opt6.7b'''),
       '''blip2-opt-2.7b-coco''': ('''blip2_opt''', '''caption_coco_opt2.7b'''),
       '''blip2-opt-6.7b-coco''': ('''blip2_opt''', '''caption_coco_opt6.7b'''),
       '''blip2-flan-t5-xl''': ('''blip2_t5''', '''pretrain_flant5xl'''),
       '''blip2-flan-t5-xl-coco''': ('''blip2_t5''', '''caption_coco_flant5xl'''),
       '''blip2-flan-t5-xxl''': ('''blip2_t5''', '''pretrain_flant5xxl'''),
   }

   a__						:    int   		=	model_name_to_original[model_name]

   # load original model
   print("Loading original model..." )
   a__						:    Any   		=	'''cuda''' if torch.cuda.is_available() else '''cpu'''
   a__						:    int   		=	load_model_and_preprocess(
       name=lowerCamelCase				,			model_type=lowerCamelCase				,			is_eval=lowerCamelCase				,			device=lowerCamelCase )
   original_model.eval()
   print("Done!" )

   # update state dict keys
   a__						:    List[str]   		=	original_model.state_dict()
   a__						:    Any   		=	create_rename_keys(lowerCamelCase )
   for src, dest in rename_keys:
      rename_key(lowerCamelCase				,			lowerCamelCase				,			lowerCamelCase )

   # some keys can be renamed efficiently
   for key, val in state_dict.copy().items():
      a__						:    List[Any]   		=	state_dict.pop(lowerCamelCase )
      if key.startswith("Qformer.bert" ):
         a__						:    List[Any]   		=	key.replace("Qformer.bert"				,			"qformer" )
      if "attention.self" in key:
         a__						:    Tuple   		=	key.replace("self"				,			"attention" )
      if "opt_proj" in key:
         a__						:    List[str]   		=	key.replace("opt_proj"				,			"language_projection" )
      if "t5_proj" in key:
         a__						:    Dict   		=	key.replace("t5_proj"				,			"language_projection" )
      if key.startswith("opt" ):
         a__						:    Any   		=	key.replace("opt"				,			"language" )
      if key.startswith("t5" ):
         a__						:    Tuple   		=	key.replace("t5"				,			"language" )
      a__						:    Optional[int]   		=	val

   # read in qv biases
   read_in_q_v_bias(lowerCamelCase				,			lowerCamelCase )

   a__						:    int   		=	hf_model.load_state_dict(lowerCamelCase				,			strict=lowerCamelCase )
   assert len(lowerCamelCase ) == 0
   assert unexpected_keys == ["qformer.embeddings.position_ids"]

   a__						:    Any   		=	load_demo_image()
   a__						:    Dict   		=	vis_processors['''eval'''](lowerCamelCase ).unsqueeze(0 ).to(lowerCamelCase )
   a__						:    List[Any]   		=	tokenizer(["\n"]				,			return_tensors="pt" ).input_ids.to(lowerCamelCase )

   # create processor
   a__						:    List[Any]   		=	BlipImageProcessor(
       size={"height": image_size, "width": image_size}				,			image_mean=lowerCamelCase				,			image_std=lowerCamelCase )
   a__						:    Optional[Any]   		=	BlipaProcessor(image_processor=lowerCamelCase				,			tokenizer=lowerCamelCase )
   a__						:    Any   		=	processor(images=lowerCamelCase				,			return_tensors="pt" ).pixel_values.to(lowerCamelCase )

   # make sure processor creates exact same pixel values
   assert torch.allclose(lowerCamelCase				,			lowerCamelCase )

   original_model.to(lowerCamelCase )
   hf_model.to(lowerCamelCase )
   with torch.no_grad():
      if "opt" in model_name:
         a__						:    List[Any]   		=	original_model({"image": original_pixel_values, "text_input": [""]} ).logits
         a__						:    Optional[int]   		=	hf_model(lowerCamelCase				,			lowerCamelCase ).logits
      else:
         a__						:    Optional[Any]   		=	original_model(
             {"image": original_pixel_values, "text_input": ["\n"], "text_output": ["\n"]} ).logits
         a__						:    int   		=	input_ids.masked_fill(input_ids == tokenizer.pad_token_id				,			-100 )
         a__						:    Dict   		=	hf_model(lowerCamelCase				,			lowerCamelCase				,			labels=lowerCamelCase ).logits

   assert original_logits.shape == logits.shape
   print("First values of original logits:"				,			original_logits[0, :3, :3] )
   print("First values of HF logits:"				,			logits[0, :3, :3] )

   # assert values
   if model_name == "blip2-flan-t5-xl":
      a__						:    Any   		=	torch.tensor(
          [[-41.5850, -4.4440, -8.9922], [-47.4322, -5.9143, -1.7340]]				,			device=lowerCamelCase )
      assert torch.allclose(logits[0, :3, :3]				,			lowerCamelCase				,			atol=1E-4 )
   elif model_name == "blip2-flan-t5-xl-coco":
      a__						:    Union[str, Any]   		=	torch.tensor(
          [[-57.0109, -9.8967, -12.6280], [-68.6578, -12.7191, -10.5065]]				,			device=lowerCamelCase )
   else:
      # cast to same type
      a__						:    Dict   		=	logits.dtype
      assert torch.allclose(original_logits.to(lowerCamelCase )				,			lowerCamelCase				,			atol=1E-2 )
   print("Looks ok!" )

   print("Generating a caption..." )
   a__						:    Optional[int]   		=	''''''
   a__						:    Tuple   		=	tokenizer(lowerCamelCase				,			return_tensors="pt" ).input_ids.to(lowerCamelCase )

   a__						:    Optional[Any]   		=	original_model.generate({"image": original_pixel_values} )
   a__						:    str   		=	hf_model.generate(
       lowerCamelCase				,			lowerCamelCase				,			do_sample=lowerCamelCase				,			num_beams=5				,			max_length=30				,			min_length=1				,			top_p=0.9				,			repetition_penalty=1.0				,			length_penalty=1.0				,			temperature=1				,			)
   print("Original generation:"				,			lowerCamelCase )
   a__						:    List[str]   		=	input_ids.shape[1]
   a__						:    Dict   		=	processor.batch_decode(outputs[:, prompt_length:]				,			skip_special_tokens=lowerCamelCase )
   a__						:    Union[str, Any]   		=	[text.strip() for text in output_text]
   print("HF generation:"				,			lowerCamelCase )

   if pytorch_dump_folder_path is not None:
      processor.save_pretrained(lowerCamelCase )
      hf_model.save_pretrained(lowerCamelCase )

   if push_to_hub:
      processor.push_to_hub(F"""nielsr/{model_name}""" )
      hf_model.push_to_hub(F"""nielsr/{model_name}""" )


if __name__ == "__main__":
  SCREAMING_SNAKE_CASE__							:      List[Any]         =    argparse.ArgumentParser()
  SCREAMING_SNAKE_CASE__							:      Dict         =    [
      'blip2-opt-2.7b',
      'blip2-opt-6.7b',
      'blip2-opt-2.7b-coco',
      'blip2-opt-6.7b-coco',
      'blip2-flan-t5-xl',
      'blip2-flan-t5-xl-coco',
      'blip2-flan-t5-xxl',
  ]
  parser.add_argument(
      """--model_name""",
      default="""blip2-opt-2.7b""",
      choices=choices,
      type=str,
      help="""Path to hf config.json of model to convert""",
  )
  parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
  parser.add_argument(
      """--push_to_hub""",
      action="""store_true""",
      help="""Whether to push the model and processor to the hub after converting""",
  )

  SCREAMING_SNAKE_CASE__							:      int         =    parser.parse_args()

  convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)









'''simple docstring'''





import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_UpperCamelCase						:						List[str]  =					re.compile(R'\b(a|an|the)\b', re.UNICODE)

_UpperCamelCase						:						Optional[int]  =					None
def __UpperCAmelCase   (					)    ->   Optional[Any]:
			UpperCAmelCase_      :       List[Any]          =       argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''		)
			parser.add_argument('''data_file'''			,       metavar='''data.json'''			,       help='''Input data JSON file.'''		)
			parser.add_argument('''pred_file'''			,       metavar='''pred.json'''			,       help='''Model predictions.'''		)
			parser.add_argument(
			    '''--out-file'''			,       '''-o'''			,       metavar='''eval.json'''			,       help='''Write accuracy metrics to file (default is stdout).'''		)
			parser.add_argument(
			    '''--na-prob-file'''			,       '''-n'''			,       metavar='''na_prob.json'''			,       help='''Model estimates of probability of no answer.'''		)
			parser.add_argument(
			    '''--na-prob-thresh'''			,       '''-t'''			,       type=A			,       default=1.0			,       help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''			,       )
			parser.add_argument(
			    '''--out-image-dir'''			,       '''-p'''			,       metavar='''out_images'''			,       default=A			,       help='''Save precision-recall curves to directory.'''		)
			parser.add_argument('''--verbose'''			,       '''-v'''			,       action='''store_true'''		)
			if len(sys.argv		) == 1:
						parser.print_help()
						sys.exit(1		)
			return parser.parse_args()
def __UpperCAmelCase   (					A					:			int		)    ->   Optional[Any]:
			UpperCAmelCase_      :       Optional[Any]          =       {}
			for article in dataset:
						for p in article["paragraphs"]:
									for qa in p["qas"]:
												UpperCAmelCase_      :       Union[str, Any]          =       bool(qa['''answers''']['''text''']		)
			return qid_to_has_ans
def __UpperCAmelCase   (					A					:			List[Any]		)    ->   Dict:

			def remove_articles(A					:			Dict		):
						return ARTICLES_REGEX.sub(''' '''			,       A		)

			def white_space_fix(A					:			str		):
						return " ".join(text.split()		)

			def remove_punc(A					:			Tuple		):
						UpperCAmelCase_      :       Dict          =       set(string.punctuation		)
						return "".join(ch for ch in text if ch not in exclude		)

			def lower(A					:			Any		):
						return text.lower()

			return white_space_fix(remove_articles(remove_punc(lower(A		)		)		)		)
def __UpperCAmelCase   (					A					:			Optional[Any]		)    ->   Tuple:
			if not s:
						return []
			return normalize_answer(A		).split()
def __UpperCAmelCase   (					A					:			Optional[int]			,       A					:			Dict		)    ->   List[str]:
			return int(normalize_answer(A		) == normalize_answer(A		)		)
def __UpperCAmelCase   (					A					:			Optional[Any]			,       A					:			List[Any]		)    ->   str:
			UpperCAmelCase_      :       Tuple          =       get_tokens(A		)
			UpperCAmelCase_      :       Union[str, Any]          =       get_tokens(A		)
			UpperCAmelCase_      :       Dict          =       collections.Counter(A		) & collections.Counter(A		)
			UpperCAmelCase_      :       Optional[int]          =       sum(common.values()		)
			if len(A		) == 0 or len(A		) == 0:
						# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
						return int(gold_toks == pred_toks		)
			if num_same == 0:
						return 0
			UpperCAmelCase_      :       int          =       1.0 * num_same / len(A		)
			UpperCAmelCase_      :       Any          =       1.0 * num_same / len(A		)
			UpperCAmelCase_      :       Any          =       (2 * precision * recall) / (precision + recall)
			return fa
def __UpperCAmelCase   (					A					:			Union[str, Any]			,       A					:			str		)    ->   Union[str, Any]:
			UpperCAmelCase_      :       List[Any]          =       {}
			UpperCAmelCase_      :       str          =       {}
			for article in dataset:
						for p in article["paragraphs"]:
									for qa in p["qas"]:
												UpperCAmelCase_      :       Optional[Any]          =       qa['''id''']
												UpperCAmelCase_      :       List[str]          =       [t for t in qa['''answers''']['''text'''] if normalize_answer(A		)]
												if not gold_answers:
															# For unanswerable questions, only correct answer is empty string
															UpperCAmelCase_      :       Tuple          =       ['''''']
												if qid not in preds:
															print(F"Missing prediction for {qid}"		)
															continue
												UpperCAmelCase_      :       Tuple          =       preds[qid]
												# Take max over all gold answers
												UpperCAmelCase_      :       List[Any]          =       max(compute_exact(A			,       A		) for a in gold_answers		)
												UpperCAmelCase_      :       int          =       max(compute_fa(A			,       A		) for a in gold_answers		)
			return exact_scores, fa_scores
def __UpperCAmelCase   (					A					:			Optional[int]			,       A					:			Tuple			,       A					:			Optional[Any]			,       A					:			List[Any]		)    ->   Optional[int]:
			UpperCAmelCase_      :       Tuple          =       {}
			for qid, s in scores.items():
						UpperCAmelCase_      :       Union[str, Any]          =       na_probs[qid] > na_prob_thresh
						if pred_na:
									UpperCAmelCase_      :       List[str]          =       float(not qid_to_has_ans[qid]		)
						else:
									UpperCAmelCase_      :       Any          =       s
			return new_scores
def __UpperCAmelCase   (					A					:			Any			,       A					:			Any			,       A					:			str=None		)    ->   int:
			if not qid_list:
						UpperCAmelCase_      :       str          =       len(A		)
						return collections.OrderedDict(
						    [
						        ('''exact''', 100.0 * sum(exact_scores.values()		) / total),
						        ('''f1''', 100.0 * sum(fa_scores.values()		) / total),
						        ('''total''', total),
						    ]		)
			else:
						UpperCAmelCase_      :       Dict          =       len(A		)
						return collections.OrderedDict(
						    [
						        ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list		) / total),
						        ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list		) / total),
						        ('''total''', total),
						    ]		)
def __UpperCAmelCase   (					A					:			Union[str, Any]			,       A					:			Dict			,       A					:			Union[str, Any]		)    ->   Dict:
			for k in new_eval:
						UpperCAmelCase_      :       str          =       new_eval[k]
def __UpperCAmelCase   (					A					:			str			,       A					:			Tuple			,       A					:			Any			,       A					:			List[str]		)    ->   Optional[int]:
			plt.step(A			,       A			,       color='''b'''			,       alpha=0.2			,       where='''post'''		)
			plt.fill_between(A			,       A			,       step='''post'''			,       alpha=0.2			,       color='''b'''		)
			plt.xlabel('''Recall'''		)
			plt.ylabel('''Precision'''		)
			plt.xlim([0.0, 1.05]		)
			plt.ylim([0.0, 1.05]		)
			plt.title(A		)
			plt.savefig(A		)
			plt.clf()
def __UpperCAmelCase   (					A					:			Optional[Any]			,       A					:			Optional[Any]			,       A					:			Optional[int]			,       A					:			Optional[int]			,       A					:			str=None			,       A					:			Any=None		)    ->   Any:
			UpperCAmelCase_      :       Optional[Any]          =       sorted(A			,       key=lambda A		: na_probs[k]		)
			UpperCAmelCase_      :       int          =       0.0
			UpperCAmelCase_      :       List[Any]          =       1.0
			UpperCAmelCase_      :       Optional[int]          =       0.0
			UpperCAmelCase_      :       Optional[Any]          =       [1.0]
			UpperCAmelCase_      :       List[Any]          =       [0.0]
			UpperCAmelCase_      :       Dict          =       0.0
			for i, qid in enumerate(A		):
						if qid_to_has_ans[qid]:
									true_pos += scores[qid]
						UpperCAmelCase_      :       List[Any]          =       true_pos / float(i + 1		)
						UpperCAmelCase_      :       Union[str, Any]          =       true_pos / float(A		)
						if i == len(A		) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
									# i.e., if we can put a threshold after this point
									avg_prec += cur_p * (cur_r - recalls[-1])
									precisions.append(A		)
									recalls.append(A		)
			if out_image:
						plot_pr_curve(A			,       A			,       A			,       A		)
			return {"ap": 100.0 * avg_prec}
def __UpperCAmelCase   (					A					:			List[Any]			,       A					:			str			,       A					:			Any			,       A					:			Optional[int]			,       A					:			Any			,       A					:			List[str]		)    ->   List[Any]:
			if out_image_dir and not os.path.exists(A		):
						os.makedirs(A		)
			UpperCAmelCase_      :       Any          =       sum(1 for v in qid_to_has_ans.values() if v		)
			if num_true_pos == 0:
						return
			UpperCAmelCase_      :       Union[str, Any]          =       make_precision_recall_eval(
			    A			,       A			,       A			,       A			,       out_image=os.path.join(A			,       '''pr_exact.png'''		)			,       title='''Precision-Recall curve for Exact Match score'''			,       )
			UpperCAmelCase_      :       List[str]          =       make_precision_recall_eval(
			    A			,       A			,       A			,       A			,       out_image=os.path.join(A			,       '''pr_f1.png'''		)			,       title='''Precision-Recall curve for F1 score'''			,       )
			UpperCAmelCase_      :       Dict          =       {k: float(A		) for k, v in qid_to_has_ans.items()}
			UpperCAmelCase_      :       Optional[int]          =       make_precision_recall_eval(
			    A			,       A			,       A			,       A			,       out_image=os.path.join(A			,       '''pr_oracle.png'''		)			,       title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''			,       )
			merge_eval(A			,       A			,       '''pr_exact'''		)
			merge_eval(A			,       A			,       '''pr_f1'''		)
			merge_eval(A			,       A			,       '''pr_oracle'''		)
def __UpperCAmelCase   (					A					:			List[str]			,       A					:			Tuple			,       A					:			str			,       A					:			List[Any]		)    ->   Dict:
			if not qid_list:
						return
			UpperCAmelCase_      :       List[Any]          =       [na_probs[k] for k in qid_list]
			UpperCAmelCase_      :       Union[str, Any]          =       np.ones_like(A		) / float(len(A		)		)
			plt.hist(A			,       weights=A			,       bins=2_0			,       range=(0.0, 1.0)		)
			plt.xlabel('''Model probability of no-answer'''		)
			plt.ylabel('''Proportion of dataset'''		)
			plt.title(F"Histogram of no-answer probability: {name}"		)
			plt.savefig(os.path.join(A			,       F"na_prob_hist_{name}.png"		)		)
			plt.clf()
def __UpperCAmelCase   (					A					:			str			,       A					:			Dict			,       A					:			List[str]			,       A					:			int		)    ->   Optional[Any]:
			UpperCAmelCase_      :       Tuple          =       sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]		)
			UpperCAmelCase_      :       Optional[Any]          =       num_no_ans
			UpperCAmelCase_      :       int          =       cur_score
			UpperCAmelCase_      :       List[Any]          =       0.0
			UpperCAmelCase_      :       Dict          =       sorted(A			,       key=lambda A		: na_probs[k]		)
			for i, qid in enumerate(A		):
						if qid not in scores:
									continue
						if qid_to_has_ans[qid]:
									UpperCAmelCase_      :       Tuple          =       scores[qid]
						else:
									if preds[qid]:
												UpperCAmelCase_      :       Optional[int]          =       -1
									else:
												UpperCAmelCase_      :       int          =       0
						cur_score += diff
						if cur_score > best_score:
									UpperCAmelCase_      :       Dict          =       cur_score
									UpperCAmelCase_      :       Dict          =       na_probs[qid]
			return 100.0 * best_score / len(A		), best_thresh
def __UpperCAmelCase   (					A					:			int			,       A					:			List[Any]			,       A					:			Union[str, Any]			,       A					:			Tuple			,       A					:			str			,       A					:			Union[str, Any]		)    ->   Tuple:
			UpperCAmelCase_    ,      UpperCAmelCase_      :       List[Any]          =       find_best_thresh(A			,       A			,       A			,       A		)
			UpperCAmelCase_    ,      UpperCAmelCase_      :       Optional[int]          =       find_best_thresh(A			,       A			,       A			,       A		)
			UpperCAmelCase_      :       List[str]          =       best_exact
			UpperCAmelCase_      :       List[Any]          =       exact_thresh
			UpperCAmelCase_      :       Any          =       best_fa
			UpperCAmelCase_      :       str          =       fa_thresh
def __UpperCAmelCase   (					)    ->   Tuple:
			with open(OPTS.data_file		) as f:
						UpperCAmelCase_      :       Optional[int]          =       json.load(A		)
						UpperCAmelCase_      :       str          =       dataset_json['''data''']
			with open(OPTS.pred_file		) as f:
						UpperCAmelCase_      :       List[Any]          =       json.load(A		)
			if OPTS.na_prob_file:
						with open(OPTS.na_prob_file		) as f:
									UpperCAmelCase_      :       List[str]          =       json.load(A		)
			else:
						UpperCAmelCase_      :       List[str]          =       {k: 0.0 for k in preds}
			UpperCAmelCase_      :       Optional[Any]          =       make_qid_to_has_ans(A		)  # maps qid to True/False
			UpperCAmelCase_      :       Optional[int]          =       [k for k, v in qid_to_has_ans.items() if v]
			UpperCAmelCase_      :       Dict          =       [k for k, v in qid_to_has_ans.items() if not v]
			UpperCAmelCase_    ,      UpperCAmelCase_      :       List[Any]          =       get_raw_scores(A			,       A		)
			UpperCAmelCase_      :       Optional[int]          =       apply_no_ans_threshold(A			,       A			,       A			,       OPTS.na_prob_thresh		)
			UpperCAmelCase_      :       Tuple          =       apply_no_ans_threshold(A			,       A			,       A			,       OPTS.na_prob_thresh		)
			UpperCAmelCase_      :       int          =       make_eval_dict(A			,       A		)
			if has_ans_qids:
						UpperCAmelCase_      :       Optional[int]          =       make_eval_dict(A			,       A			,       qid_list=A		)
						merge_eval(A			,       A			,       '''HasAns'''		)
			if no_ans_qids:
						UpperCAmelCase_      :       Dict          =       make_eval_dict(A			,       A			,       qid_list=A		)
						merge_eval(A			,       A			,       '''NoAns'''		)
			if OPTS.na_prob_file:
						find_all_best_thresh(A			,       A			,       A			,       A			,       A			,       A		)
			if OPTS.na_prob_file and OPTS.out_image_dir:
						run_precision_recall_analysis(A			,       A			,       A			,       A			,       A			,       OPTS.out_image_dir		)
						histogram_na_prob(A			,       A			,       OPTS.out_image_dir			,       '''hasAns'''		)
						histogram_na_prob(A			,       A			,       OPTS.out_image_dir			,       '''noAns'''		)
			if OPTS.out_file:
						with open(OPTS.out_file			,       '''w'''		) as f:
									json.dump(A			,       A		)
			else:
						print(json.dumps(A			,       indent=2		)		)


if __name__ == "__main__":
						_UpperCamelCase						:						Union[str, Any]  =					parse_args()
						if OPTS.out_image_dir:
												import matplotlib

												matplotlib.use('Agg')
												import matplotlib.pyplot as plt
						main()


