import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


UpperCAmelCase_           =				re.compile(R'''\b(a|an|the)\b''', re.UNICODE)

UpperCAmelCase_           =				None




def    lowerCAmelCase_  (		)     ->					int:







 '''simple docstring'''
 _UpperCamelCase:  List[str]    						=			argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''		)
 parser.add_argument('''data_file'''	,       metavar='''data.json'''	,       help='''Input data JSON file.'''		)
 parser.add_argument('''pred_file'''	,       metavar='''pred.json'''	,       help='''Model predictions.'''		)
 parser.add_argument(
     '''--out-file'''	,       '''-o'''	,       metavar='''eval.json'''	,       help='''Write accuracy metrics to file (default is stdout).'''		)
 parser.add_argument(
     '''--na-prob-file'''	,       '''-n'''	,       metavar='''na_prob.json'''	,       help='''Model estimates of probability of no answer.'''		)
 parser.add_argument(
     '''--na-prob-thresh'''	,       '''-t'''	,       type=lowercase	,       default=1.0	,       help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''	,       )
 parser.add_argument(
     '''--out-image-dir'''	,       '''-p'''	,       metavar='''out_images'''	,       default=lowercase	,       help='''Save precision-recall curves to directory.'''		)
 parser.add_argument('''--verbose'''	,       '''-v'''	,       action='''store_true'''		)
 if len(sys.argv		) == 1:
  parser.print_help()
  sys.exit(1		)
 return parser.parse_args()




def    lowerCAmelCase_  (		lowercase:				Any		)     ->					Union[str, Any]:







 '''simple docstring'''
 _UpperCamelCase:  str    						=			{}
 for article in dataset:
  for p in article["paragraphs"]:
   for qa in p["qas"]:
    _UpperCamelCase:  int    						=			bool(qa['''answers''']['''text''']		)
 return qid_to_has_ans




def    lowerCAmelCase_  (		lowercase:				Union[str, Any]		)     ->					Union[str, Any]:







 '''simple docstring'''

 def remove_articles(lowercase:				str		):
  return ARTICLES_REGEX.sub(''' '''	,       lowercase		)

 def white_space_fix(lowercase:				Optional[Any]		):
  return " ".join(text.split()		)

 def remove_punc(lowercase:				Optional[int]		):
  _UpperCamelCase:  int    						=			set(string.punctuation		)
  return "".join(ch for ch in text if ch not in exclude		)

 def lower(lowercase:				Optional[int]		):
  return text.lower()

 return white_space_fix(remove_articles(remove_punc(lower(lowercase		)		)		)		)




def    lowerCAmelCase_  (		lowercase:				Union[str, Any]		)     ->					List[Any]:







 '''simple docstring'''
 if not s:
  return []
 return normalize_answer(lowercase		).split()




def    lowerCAmelCase_  (		lowercase:				Union[str, Any]	,       lowercase:				int		)     ->					Dict:







 '''simple docstring'''
 return int(normalize_answer(lowercase		) == normalize_answer(lowercase		)		)




def    lowerCAmelCase_  (		lowercase:				str	,       lowercase:				Dict		)     ->					List[str]:







 '''simple docstring'''
 _UpperCamelCase:  Any    						=			get_tokens(lowercase		)
 _UpperCamelCase:  Any    						=			get_tokens(lowercase		)
 _UpperCamelCase:  Any    						=			collections.Counter(lowercase		) & collections.Counter(lowercase		)
 _UpperCamelCase:  Dict    						=			sum(common.values()		)
 if len(lowercase		) == 0 or len(lowercase		) == 0:
  # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
  return int(gold_toks == pred_toks		)
 if num_same == 0:
  return 0
 _UpperCamelCase:  Optional[int]    						=			1.0 * num_same / len(lowercase		)
 _UpperCamelCase:  Dict    						=			1.0 * num_same / len(lowercase		)
 _UpperCamelCase:  List[str]    						=			(2 * precision * recall) / (precision + recall)
 return fa




def    lowerCAmelCase_  (		lowercase:				int	,       lowercase:				List[Any]		)     ->					int:







 '''simple docstring'''
 _UpperCamelCase:  List[Any]    						=			{}
 _UpperCamelCase:  Tuple    						=			{}
 for article in dataset:
  for p in article["paragraphs"]:
   for qa in p["qas"]:
    _UpperCamelCase:  str    						=			qa['''id''']
    _UpperCamelCase:  Union[str, Any]    						=			[t for t in qa['''answers''']['''text'''] if normalize_answer(lowercase		)]
    if not gold_answers:
     # For unanswerable questions, only correct answer is empty string
     _UpperCamelCase:  List[str]    						=			['''''']
    if qid not in preds:
     print(F"""Missing prediction for {qid}"""		)
     continue
    _UpperCamelCase:  Tuple    						=			preds[qid]
    # Take max over all gold answers
    _UpperCamelCase:  Optional[Any]    						=			max(compute_exact(lowercase	,       lowercase		) for a in gold_answers		)
    _UpperCamelCase:  str    						=			max(compute_fa(lowercase	,       lowercase		) for a in gold_answers		)
 return exact_scores, fa_scores




def    lowerCAmelCase_  (		lowercase:				Any	,       lowercase:				Optional[int]	,       lowercase:				Tuple	,       lowercase:				Optional[int]		)     ->					Dict:







 '''simple docstring'''
 _UpperCamelCase:  Union[str, Any]    						=			{}
 for qid, s in scores.items():
  _UpperCamelCase:  Tuple    						=			na_probs[qid] > na_prob_thresh
  if pred_na:
   _UpperCamelCase:  Any    						=			float(not qid_to_has_ans[qid]		)
  else:
   _UpperCamelCase:  Union[str, Any]    						=			s
 return new_scores




def    lowerCAmelCase_  (		lowercase:				Tuple	,       lowercase:				Optional[Any]	,       lowercase:				Union[str, Any]=None		)     ->					int:







 '''simple docstring'''
 if not qid_list:
  _UpperCamelCase:  Any    						=			len(lowercase		)
  return collections.OrderedDict(
      [
          ('''exact''', 100.0 * sum(exact_scores.values()		) / total),
          ('''f1''', 100.0 * sum(fa_scores.values()		) / total),
          ('''total''', total),
      ]		)
 else:
  _UpperCamelCase:  List[Any]    						=			len(lowercase		)
  return collections.OrderedDict(
      [
          ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list		) / total),
          ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list		) / total),
          ('''total''', total),
      ]		)




def    lowerCAmelCase_  (		lowercase:				Any	,       lowercase:				Dict	,       lowercase:				List[str]		)     ->					Dict:







 '''simple docstring'''
 for k in new_eval:
  _UpperCamelCase:  List[str]    						=			new_eval[k]




def    lowerCAmelCase_  (		lowercase:				Optional[int]	,       lowercase:				int	,       lowercase:				Any	,       lowercase:				Optional[int]		)     ->					List[str]:







 '''simple docstring'''
 plt.step(lowercase	,       lowercase	,       color='''b'''	,       alpha=0.2	,       where='''post'''		)
 plt.fill_between(lowercase	,       lowercase	,       step='''post'''	,       alpha=0.2	,       color='''b'''		)
 plt.xlabel('''Recall'''		)
 plt.ylabel('''Precision'''		)
 plt.xlim([0.0, 1.05]		)
 plt.ylim([0.0, 1.05]		)
 plt.title(lowercase		)
 plt.savefig(lowercase		)
 plt.clf()




def    lowerCAmelCase_  (		lowercase:				int	,       lowercase:				List[Any]	,       lowercase:				Dict	,       lowercase:				Union[str, Any]	,       lowercase:				List[Any]=None	,       lowercase:				Dict=None		)     ->					Optional[Any]:







 '''simple docstring'''
 _UpperCamelCase:  List[Any]    						=			sorted(lowercase	,       key=lambda lowercase		: na_probs[k]		)
 _UpperCamelCase:  str    						=			0.0
 _UpperCamelCase:  str    						=			1.0
 _UpperCamelCase:  Dict    						=			0.0
 _UpperCamelCase:  Union[str, Any]    						=			[1.0]
 _UpperCamelCase:  Dict    						=			[0.0]
 _UpperCamelCase:  Optional[Any]    						=			0.0
 for i, qid in enumerate(lowercase		):
  if qid_to_has_ans[qid]:
   true_pos += scores[qid]
  _UpperCamelCase:  Any    						=			true_pos / float(i + 1		)
  _UpperCamelCase:  Dict    						=			true_pos / float(lowercase		)
  if i == len(lowercase		) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
   # i.e., if we can put a threshold after this point
   avg_prec += cur_p * (cur_r - recalls[-1])
   precisions.append(lowercase		)
   recalls.append(lowercase		)
 if out_image:
  plot_pr_curve(lowercase	,       lowercase	,       lowercase	,       lowercase		)
 return {"ap": 100.0 * avg_prec}




def    lowerCAmelCase_  (		lowercase:				Optional[int]	,       lowercase:				Optional[int]	,       lowercase:				Optional[int]	,       lowercase:				List[str]	,       lowercase:				Union[str, Any]	,       lowercase:				Optional[Any]		)     ->					List[str]:







 '''simple docstring'''
 if out_image_dir and not os.path.exists(lowercase		):
  os.makedirs(lowercase		)
 _UpperCamelCase:  int    						=			sum(1 for v in qid_to_has_ans.values() if v		)
 if num_true_pos == 0:
  return
 _UpperCamelCase:  List[Any]    						=			make_precision_recall_eval(
     lowercase	,       lowercase	,       lowercase	,       lowercase	,       out_image=os.path.join(lowercase	,       '''pr_exact.png'''		)	,       title='''Precision-Recall curve for Exact Match score'''	,       )
 _UpperCamelCase:  Any    						=			make_precision_recall_eval(
     lowercase	,       lowercase	,       lowercase	,       lowercase	,       out_image=os.path.join(lowercase	,       '''pr_f1.png'''		)	,       title='''Precision-Recall curve for F1 score'''	,       )
 _UpperCamelCase:  List[Any]    						=			{k: float(lowercase		) for k, v in qid_to_has_ans.items()}
 _UpperCamelCase:  Optional[Any]    						=			make_precision_recall_eval(
     lowercase	,       lowercase	,       lowercase	,       lowercase	,       out_image=os.path.join(lowercase	,       '''pr_oracle.png'''		)	,       title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''	,       )
 merge_eval(lowercase	,       lowercase	,       '''pr_exact'''		)
 merge_eval(lowercase	,       lowercase	,       '''pr_f1'''		)
 merge_eval(lowercase	,       lowercase	,       '''pr_oracle'''		)




def    lowerCAmelCase_  (		lowercase:				int	,       lowercase:				List[Any]	,       lowercase:				Union[str, Any]	,       lowercase:				List[str]		)     ->					Dict:







 '''simple docstring'''
 if not qid_list:
  return
 _UpperCamelCase:  Tuple    						=			[na_probs[k] for k in qid_list]
 _UpperCamelCase:  int    						=			np.ones_like(lowercase		) / float(len(lowercase		)		)
 plt.hist(lowercase	,       weights=lowercase	,       bins=20	,       range=(0.0, 1.0)		)
 plt.xlabel('''Model probability of no-answer'''		)
 plt.ylabel('''Proportion of dataset'''		)
 plt.title(F"""Histogram of no-answer probability: {name}"""		)
 plt.savefig(os.path.join(lowercase	,       F"""na_prob_hist_{name}.png"""		)		)
 plt.clf()




def    lowerCAmelCase_  (		lowercase:				int	,       lowercase:				List[str]	,       lowercase:				List[Any]	,       lowercase:				Optional[int]		)     ->					List[Any]:







 '''simple docstring'''
 _UpperCamelCase:  int    						=			sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]		)
 _UpperCamelCase:  Optional[Any]    						=			num_no_ans
 _UpperCamelCase:  Optional[Any]    						=			cur_score
 _UpperCamelCase:  Union[str, Any]    						=			0.0
 _UpperCamelCase:  Dict    						=			sorted(lowercase	,       key=lambda lowercase		: na_probs[k]		)
 for i, qid in enumerate(lowercase		):
  if qid not in scores:
   continue
  if qid_to_has_ans[qid]:
   _UpperCamelCase:  Dict    						=			scores[qid]
  else:
   if preds[qid]:
    _UpperCamelCase:  List[Any]    						=			-1
   else:
    _UpperCamelCase:  Tuple    						=			0
  cur_score += diff
  if cur_score > best_score:
   _UpperCamelCase:  Union[str, Any]    						=			cur_score
   _UpperCamelCase:  Optional[Any]    						=			na_probs[qid]
 return 100.0 * best_score / len(lowercase		), best_thresh




def    lowerCAmelCase_  (		lowercase:				Optional[Any]	,       lowercase:				List[str]	,       lowercase:				List[Any]	,       lowercase:				List[str]	,       lowercase:				Dict	,       lowercase:				int		)     ->					Optional[Any]:







 '''simple docstring'''
 _UpperCamelCase						,				_UpperCamelCase:  Union[str, Any]    						=			find_best_thresh(lowercase	,       lowercase	,       lowercase	,       lowercase		)
 _UpperCamelCase						,				_UpperCamelCase:  Any    						=			find_best_thresh(lowercase	,       lowercase	,       lowercase	,       lowercase		)
 _UpperCamelCase:  Optional[int]    						=			best_exact
 _UpperCamelCase:  Tuple    						=			exact_thresh
 _UpperCamelCase:  Optional[Any]    						=			best_fa
 _UpperCamelCase:  Dict    						=			fa_thresh




def    lowerCAmelCase_  (		)     ->					Optional[Any]:







 '''simple docstring'''
 with open(OPTS.data_file		) as f:
  _UpperCamelCase:  Tuple    						=			json.load(lowercase		)
  _UpperCamelCase:  Optional[int]    						=			dataset_json['''data''']
 with open(OPTS.pred_file		) as f:
  _UpperCamelCase:  List[Any]    						=			json.load(lowercase		)
 if OPTS.na_prob_file:
  with open(OPTS.na_prob_file		) as f:
   _UpperCamelCase:  Any    						=			json.load(lowercase		)
 else:
  _UpperCamelCase:  str    						=			{k: 0.0 for k in preds}
 _UpperCamelCase:  List[str]    						=			make_qid_to_has_ans(lowercase		)  # maps qid to True/False
 _UpperCamelCase:  List[Any]    						=			[k for k, v in qid_to_has_ans.items() if v]
 _UpperCamelCase:  Dict    						=			[k for k, v in qid_to_has_ans.items() if not v]
 _UpperCamelCase						,				_UpperCamelCase:  Dict    						=			get_raw_scores(lowercase	,       lowercase		)
 _UpperCamelCase:  Optional[int]    						=			apply_no_ans_threshold(lowercase	,       lowercase	,       lowercase	,       OPTS.na_prob_thresh		)
 _UpperCamelCase:  str    						=			apply_no_ans_threshold(lowercase	,       lowercase	,       lowercase	,       OPTS.na_prob_thresh		)
 _UpperCamelCase:  Union[str, Any]    						=			make_eval_dict(lowercase	,       lowercase		)
 if has_ans_qids:
  _UpperCamelCase:  Dict    						=			make_eval_dict(lowercase	,       lowercase	,       qid_list=lowercase		)
  merge_eval(lowercase	,       lowercase	,       '''HasAns'''		)
 if no_ans_qids:
  _UpperCamelCase:  Optional[int]    						=			make_eval_dict(lowercase	,       lowercase	,       qid_list=lowercase		)
  merge_eval(lowercase	,       lowercase	,       '''NoAns'''		)
 if OPTS.na_prob_file:
  find_all_best_thresh(lowercase	,       lowercase	,       lowercase	,       lowercase	,       lowercase	,       lowercase		)
 if OPTS.na_prob_file and OPTS.out_image_dir:
  run_precision_recall_analysis(lowercase	,       lowercase	,       lowercase	,       lowercase	,       lowercase	,       OPTS.out_image_dir		)
  histogram_na_prob(lowercase	,       lowercase	,       OPTS.out_image_dir	,       '''hasAns'''		)
  histogram_na_prob(lowercase	,       lowercase	,       OPTS.out_image_dir	,       '''noAns'''		)
 if OPTS.out_file:
  with open(OPTS.out_file	,       '''w'''		) as f:
   json.dump(lowercase	,       lowercase		)
 else:
  print(json.dumps(lowercase	,       indent=2		)		)


if __name__ == "__main__":
  UpperCAmelCase_           =				parse_args()
  if OPTS.out_image_dir:
    import matplotlib

    matplotlib.use('''Agg''')
    import matplotlib.pyplot as plt
  main()
import unittest

import numpy as np

from transformers import AlbertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
  import jax.numpy as jnp

  from transformers.models.albert.modeling_flax_albert import (
      FlaxAlbertForMaskedLM,
      FlaxAlbertForMultipleChoice,
      FlaxAlbertForPreTraining,
      FlaxAlbertForQuestionAnswering,
      FlaxAlbertForSequenceClassification,
      FlaxAlbertForTokenClassification,
      FlaxAlbertModel,
  )




class 			__magic_name__	(		unittest.TestCase     ):





   """simple docstring"""




   def __init__(						self       :	Any		,	_lowercase       :	Optional[int]		,	_lowercase       :	Tuple=13		,	_lowercase       :	str=7		,	_lowercase       :	List[Any]=True		,	_lowercase       :	Optional[int]=True		,	_lowercase       :	str=True		,	_lowercase       :	Optional[int]=True		,	_lowercase       :	Dict=99		,	_lowercase       :	List[Any]=32		,	_lowercase       :	List[str]=5		,	_lowercase       :	str=4		,	_lowercase       :	int=37		,	_lowercase       :	List[str]="gelu"		,	_lowercase       :	Any=0.1		,	_lowercase       :	Optional[int]=0.1		,	_lowercase       :	Dict=512		,	_lowercase       :	int=16		,	_lowercase       :	Optional[Any]=2		,	_lowercase       :	Dict=0.02		,	_lowercase       :	List[Any]=4		,	):



    """simple docstring"""



    _UpperCamelCase:  List[str]    						=			parent
    _UpperCamelCase:  int    						=			batch_size
    _UpperCamelCase:  List[str]    						=			seq_length
    _UpperCamelCase:  Optional[int]    						=			is_training
    _UpperCamelCase:  Optional[Any]    						=			use_attention_mask
    _UpperCamelCase:  Any    						=			use_token_type_ids
    _UpperCamelCase:  List[str]    						=			use_labels
    _UpperCamelCase:  Optional[int]    						=			vocab_size
    _UpperCamelCase:  List[str]    						=			hidden_size
    _UpperCamelCase:  Union[str, Any]    						=			num_hidden_layers
    _UpperCamelCase:  Any    						=			num_attention_heads
    _UpperCamelCase:  List[str]    						=			intermediate_size
    _UpperCamelCase:  Union[str, Any]    						=			hidden_act
    _UpperCamelCase:  Dict    						=			hidden_dropout_prob
    _UpperCamelCase:  List[str]    						=			attention_probs_dropout_prob
    _UpperCamelCase:  str    						=			max_position_embeddings
    _UpperCamelCase:  Dict    						=			type_vocab_size
    _UpperCamelCase:  Tuple    						=			type_sequence_label_size
    _UpperCamelCase:  List[Any]    						=			initializer_range
    _UpperCamelCase:  str    						=			num_choices



   def 	lowerCAmelCase    (						self       :	str				):



    """simple docstring"""



    _UpperCamelCase:  str    						=			ids_tensor([self.batch_size, self.seq_length]		,	self.vocab_size				)

    _UpperCamelCase:  Any    						=			None
    if self.use_attention_mask:
     _UpperCamelCase:  List[str]    						=			random_attention_mask([self.batch_size, self.seq_length]				)

    _UpperCamelCase:  Optional[Any]    						=			None
    if self.use_token_type_ids:
     _UpperCamelCase:  Optional[Any]    						=			ids_tensor([self.batch_size, self.seq_length]		,	self.type_vocab_size				)

    _UpperCamelCase:  Tuple    						=			AlbertConfig(
        vocab_size=self.vocab_size		,	hidden_size=self.hidden_size		,	num_hidden_layers=self.num_hidden_layers		,	num_attention_heads=self.num_attention_heads		,	intermediate_size=self.intermediate_size		,	hidden_act=self.hidden_act		,	hidden_dropout_prob=self.hidden_dropout_prob		,	attention_probs_dropout_prob=self.attention_probs_dropout_prob		,	max_position_embeddings=self.max_position_embeddings		,	type_vocab_size=self.type_vocab_size		,	is_decoder=_lowercase		,	initializer_range=self.initializer_range		,	)

    return config, input_ids, token_type_ids, attention_mask



   def 	lowerCAmelCase    (						self       :	int				):



    """simple docstring"""



    _UpperCamelCase:  List[Any]    						=			self.prepare_config_and_inputs()
    _UpperCamelCase						,				_UpperCamelCase						,				_UpperCamelCase						,				_UpperCamelCase:  int    						=			config_and_inputs
    _UpperCamelCase:  int    						=			{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': attention_mask}
    return config, inputs_dict




@require_flax
class 			__magic_name__	(		__a   ,						unittest.TestCase     ):





   """simple docstring"""




   lowerCAmelCase   :     Any	      = (
       (
           FlaxAlbertModel,
           FlaxAlbertForPreTraining,
           FlaxAlbertForMaskedLM,
           FlaxAlbertForMultipleChoice,
           FlaxAlbertForQuestionAnswering,
           FlaxAlbertForSequenceClassification,
           FlaxAlbertForTokenClassification,
           FlaxAlbertForQuestionAnswering,
       )
       if is_flax_available()
       else ()
   )



   def 	lowerCAmelCase    (						self       :	Union[str, Any]				):



    """simple docstring"""



    _UpperCamelCase:  Optional[int]    						=			FlaxAlbertModelTester(self				)



   @slow
   def 	lowerCAmelCase    (						self       :	Union[str, Any]				):



    """simple docstring"""



    for model_class_name in self.all_model_classes:
     _UpperCamelCase:  int    						=			model_class_name.from_pretrained('''albert-base-v2'''				)
     _UpperCamelCase:  List[Any]    						=			model(np.ones((1, 1)				)				)
     self.assertIsNotNone(_lowercase				)




@require_flax
class 			__magic_name__	(		unittest.TestCase     ):





   """simple docstring"""




   @slow
   def 	lowerCAmelCase    (						self       :	int				):



    """simple docstring"""



    _UpperCamelCase:  Optional[int]    						=			FlaxAlbertModel.from_pretrained('''albert-base-v2'''				)
    _UpperCamelCase:  List[Any]    						=			np.array([[0, 345, 232, 328, 740, 140, 1_695, 69, 6_078, 1_588, 2]]				)
    _UpperCamelCase:  Optional[int]    						=			np.array([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]				)
    _UpperCamelCase:  Optional[int]    						=			model(_lowercase		,	attention_mask=_lowercase				)[0]
    _UpperCamelCase:  Tuple    						=			(1, 11, 768)
    self.assertEqual(output.shape		,	_lowercase				)
    _UpperCamelCase:  Tuple    						=			np.array(
        [[[-0.6513, 1.5035, -0.2766], [-0.6515, 1.5046, -0.2780], [-0.6512, 1.5049, -0.2784]]]				)

    self.assertTrue(jnp.allclose(output[:, 1:4, 1:4]		,	_lowercase		,	atol=1E-4				)				)