



'''simple docstring'''



import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCamelCase									=							re.compile(R"""\b(a|an|the)\b""", re.UNICODE)

lowerCamelCase									=							None



def 	_A							( ):



							"""simple docstring"""



							__lowercase													=argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'       )
							parser.add_argument('data_file'	, metavar='data.json'	, help='Input data JSON file.'       )
							parser.add_argument('pred_file'	, metavar='pred.json'	, help='Model predictions.'       )
							parser.add_argument(
							    '--out-file'	, '-o'	, metavar='eval.json'	, help='Write accuracy metrics to file (default is stdout).'       )
							parser.add_argument(
							    '--na-prob-file'	, '-n'	, metavar='na_prob.json'	, help='Model estimates of probability of no answer.'       )
							parser.add_argument(
							    '--na-prob-thresh'	, '-t'	, type=UpperCAmelCase_	, default=1.0	, help='Predict "" if no-answer probability exceeds this (default = 1.0).'	, )
							parser.add_argument(
							    '--out-image-dir'	, '-p'	, metavar='out_images'	, default=UpperCAmelCase_	, help='Save precision-recall curves to directory.'       )
							parser.add_argument('--verbose'	, '-v'	, action='store_true'       )
							if len(sys.argv       ) == 1:
														parser.print_help()
														sys.exit(1       )
							return parser.parse_args()



def 	_A							( _lowerCAmelCase       ):



							"""simple docstring"""



							__lowercase													={}
							for article in dataset:
														for p in article["paragraphs"]:
																					for qa in p["qas"]:
																												__lowercase													=bool(qa['answers']['text']       )
							return qid_to_has_ans



def 	_A							( _lowerCAmelCase       ):



							"""simple docstring"""



							def remove_articles(_lowerCAmelCase       ):
														return ARTICLES_REGEX.sub(' '	, UpperCAmelCase_       )

							def white_space_fix(_lowerCAmelCase       ):
														return " ".join(text.split()       )

							def remove_punc(_lowerCAmelCase       ):
														__lowercase													=set(string.punctuation       )
														return "".join(ch for ch in text if ch not in exclude       )

							def lower(_lowerCAmelCase       ):
														return text.lower()

							return white_space_fix(remove_articles(remove_punc(lower(UpperCAmelCase_       )       )       )       )



def 	_A							( _lowerCAmelCase       ):



							"""simple docstring"""



							if not s:
														return []
							return normalize_answer(UpperCAmelCase_       ).split()



def 	_A							( _lowerCAmelCase	, _lowerCAmelCase       ):



							"""simple docstring"""



							return int(normalize_answer(UpperCAmelCase_       ) == normalize_answer(UpperCAmelCase_       )       )



def 	_A							( _lowerCAmelCase	, _lowerCAmelCase       ):



							"""simple docstring"""



							__lowercase													=get_tokens(UpperCAmelCase_       )
							__lowercase													=get_tokens(UpperCAmelCase_       )
							__lowercase													=collections.Counter(UpperCAmelCase_       ) & collections.Counter(UpperCAmelCase_       )
							__lowercase													=sum(common.values()       )
							if len(UpperCAmelCase_       ) == 0 or len(UpperCAmelCase_       ) == 0:
														# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
														return int(gold_toks == pred_toks       )
							if num_same == 0:
														return 0
							__lowercase													=1.0 * num_same / len(UpperCAmelCase_       )
							__lowercase													=1.0 * num_same / len(UpperCAmelCase_       )
							__lowercase													=(2 * precision * recall) / (precision + recall)
							return fa



def 	_A							( _lowerCAmelCase	, _lowerCAmelCase       ):



							"""simple docstring"""



							__lowercase													={}
							__lowercase													={}
							for article in dataset:
														for p in article["paragraphs"]:
																					for qa in p["qas"]:
																												__lowercase													=qa['''id''']
																												__lowercase													=[t for t in qa['''answers''']['''text'''] if normalize_answer(UpperCAmelCase_       )]
																												if not gold_answers:
																																			# For unanswerable questions, only correct answer is empty string
																																			__lowercase													=['''''']
																												if qid not in preds:
																																			print(f"""Missing prediction for {qid}"""       )
																																			continue
																												__lowercase													=preds[qid]
																												# Take max over all gold answers
																												__lowercase													=max(compute_exact(UpperCAmelCase_	, UpperCAmelCase_       ) for a in gold_answers       )
																												__lowercase													=max(compute_fa(UpperCAmelCase_	, UpperCAmelCase_       ) for a in gold_answers       )
							return exact_scores, fa_scores



def 	_A							( _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase       ):



							"""simple docstring"""



							__lowercase													={}
							for qid, s in scores.items():
														__lowercase													=na_probs[qid] > na_prob_thresh
														if pred_na:
																					__lowercase													=float(not qid_to_has_ans[qid]       )
														else:
																					__lowercase													=s
							return new_scores



def 	_A							( _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase=None       ):



							"""simple docstring"""



							if not qid_list:
														__lowercase													=len(UpperCAmelCase_       )
														return collections.OrderedDict(
														    [
														        ('exact', 1_00.0 * sum(exact_scores.values()       ) / total),
														        ('f1', 1_00.0 * sum(fa_scores.values()       ) / total),
														        ('total', total),
														    ]       )
							else:
														__lowercase													=len(UpperCAmelCase_       )
														return collections.OrderedDict(
														    [
														        ('exact', 1_00.0 * sum(exact_scores[k] for k in qid_list       ) / total),
														        ('f1', 1_00.0 * sum(fa_scores[k] for k in qid_list       ) / total),
														        ('total', total),
														    ]       )



def 	_A							( _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase       ):



							"""simple docstring"""



							for k in new_eval:
														__lowercase													=new_eval[k]



def 	_A							( _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase       ):



							"""simple docstring"""



							plt.step(UpperCAmelCase_	, UpperCAmelCase_	, color='b'	, alpha=0.2	, where='post'       )
							plt.fill_between(UpperCAmelCase_	, UpperCAmelCase_	, step='post'	, alpha=0.2	, color='b'       )
							plt.xlabel('Recall'       )
							plt.ylabel('Precision'       )
							plt.xlim([0.0, 1.05]       )
							plt.ylim([0.0, 1.05]       )
							plt.title(UpperCAmelCase_       )
							plt.savefig(UpperCAmelCase_       )
							plt.clf()



def 	_A							( _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase=None	, _lowerCAmelCase=None       ):



							"""simple docstring"""



							__lowercase													=sorted(UpperCAmelCase_	, key=lambda _lowerCAmelCase       : na_probs[k]       )
							__lowercase													=0.0
							__lowercase													=1.0
							__lowercase													=0.0
							__lowercase													=[1.0]
							__lowercase													=[0.0]
							__lowercase													=0.0
							for i, qid in enumerate(UpperCAmelCase_       ):
														if qid_to_has_ans[qid]:
																					true_pos += scores[qid]
														__lowercase													=true_pos / float(i + 1       )
														__lowercase													=true_pos / float(UpperCAmelCase_       )
														if i == len(UpperCAmelCase_       ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
																					# i.e., if we can put a threshold after this point
																					avg_prec += cur_p * (cur_r - recalls[-1])
																					precisions.append(UpperCAmelCase_       )
																					recalls.append(UpperCAmelCase_       )
							if out_image:
														plot_pr_curve(UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_       )
							return {"ap": 1_00.0 * avg_prec}



def 	_A							( _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase       ):



							"""simple docstring"""



							if out_image_dir and not os.path.exists(UpperCAmelCase_       ):
														os.makedirs(UpperCAmelCase_       )
							__lowercase													=sum(1 for v in qid_to_has_ans.values() if v       )
							if num_true_pos == 0:
														return
							__lowercase													=make_precision_recall_eval(
							    UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, out_image=os.path.join(UpperCAmelCase_	, 'pr_exact.png'       )	, title='Precision-Recall curve for Exact Match score'	, )
							__lowercase													=make_precision_recall_eval(
							    UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, out_image=os.path.join(UpperCAmelCase_	, 'pr_f1.png'       )	, title='Precision-Recall curve for F1 score'	, )
							__lowercase													={k: float(UpperCAmelCase_       ) for k, v in qid_to_has_ans.items()}
							__lowercase													=make_precision_recall_eval(
							    UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, out_image=os.path.join(UpperCAmelCase_	, 'pr_oracle.png'       )	, title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'	, )
							merge_eval(UpperCAmelCase_	, UpperCAmelCase_	, 'pr_exact'       )
							merge_eval(UpperCAmelCase_	, UpperCAmelCase_	, 'pr_f1'       )
							merge_eval(UpperCAmelCase_	, UpperCAmelCase_	, 'pr_oracle'       )



def 	_A							( _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase       ):



							"""simple docstring"""



							if not qid_list:
														return
							__lowercase													=[na_probs[k] for k in qid_list]
							__lowercase													=np.ones_like(UpperCAmelCase_       ) / float(len(UpperCAmelCase_       )       )
							plt.hist(UpperCAmelCase_	, weights=UpperCAmelCase_	, bins=20	, range=(0.0, 1.0)       )
							plt.xlabel('Model probability of no-answer'       )
							plt.ylabel('Proportion of dataset'       )
							plt.title(f"""Histogram of no-answer probability: {name}"""       )
							plt.savefig(os.path.join(UpperCAmelCase_	, f"""na_prob_hist_{name}.png"""       )       )
							plt.clf()



def 	_A							( _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase       ):



							"""simple docstring"""



							__lowercase													=sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]       )
							__lowercase													=num_no_ans
							__lowercase													=cur_score
							__lowercase													=0.0
							__lowercase													=sorted(UpperCAmelCase_	, key=lambda _lowerCAmelCase       : na_probs[k]       )
							for i, qid in enumerate(UpperCAmelCase_       ):
														if qid not in scores:
																					continue
														if qid_to_has_ans[qid]:
																					__lowercase													=scores[qid]
														else:
																					if preds[qid]:
																												__lowercase													=-1
																					else:
																												__lowercase													=0
														cur_score += diff
														if cur_score > best_score:
																					__lowercase													=cur_score
																					__lowercase													=na_probs[qid]
							return 1_00.0 * best_score / len(UpperCAmelCase_       ), best_thresh



def 	_A							( _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase	, _lowerCAmelCase       ):



							"""simple docstring"""



							__lowercase													=find_best_thresh(UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_       )
							__lowercase													=find_best_thresh(UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_       )
							__lowercase													=best_exact
							__lowercase													=exact_thresh
							__lowercase													=best_fa
							__lowercase													=fa_thresh



def 	_A							( ):



							"""simple docstring"""



							with open(OPTS.data_file       ) as f:
														__lowercase													=json.load(UpperCAmelCase_       )
														__lowercase													=dataset_json['''data''']
							with open(OPTS.pred_file       ) as f:
														__lowercase													=json.load(UpperCAmelCase_       )
							if OPTS.na_prob_file:
														with open(OPTS.na_prob_file       ) as f:
																					__lowercase													=json.load(UpperCAmelCase_       )
							else:
														__lowercase													={k: 0.0 for k in preds}
							__lowercase													=make_qid_to_has_ans(UpperCAmelCase_       )  # maps qid to True/False
							__lowercase													=[k for k, v in qid_to_has_ans.items() if v]
							__lowercase													=[k for k, v in qid_to_has_ans.items() if not v]
							__lowercase													=get_raw_scores(UpperCAmelCase_	, UpperCAmelCase_       )
							__lowercase													=apply_no_ans_threshold(UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, OPTS.na_prob_thresh       )
							__lowercase													=apply_no_ans_threshold(UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, OPTS.na_prob_thresh       )
							__lowercase													=make_eval_dict(UpperCAmelCase_	, UpperCAmelCase_       )
							if has_ans_qids:
														__lowercase													=make_eval_dict(UpperCAmelCase_	, UpperCAmelCase_	, qid_list=UpperCAmelCase_       )
														merge_eval(UpperCAmelCase_	, UpperCAmelCase_	, 'HasAns'       )
							if no_ans_qids:
														__lowercase													=make_eval_dict(UpperCAmelCase_	, UpperCAmelCase_	, qid_list=UpperCAmelCase_       )
														merge_eval(UpperCAmelCase_	, UpperCAmelCase_	, 'NoAns'       )
							if OPTS.na_prob_file:
														find_all_best_thresh(UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_       )
							if OPTS.na_prob_file and OPTS.out_image_dir:
														run_precision_recall_analysis(UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, UpperCAmelCase_	, OPTS.out_image_dir       )
														histogram_na_prob(UpperCAmelCase_	, UpperCAmelCase_	, OPTS.out_image_dir	, 'hasAns'       )
														histogram_na_prob(UpperCAmelCase_	, UpperCAmelCase_	, OPTS.out_image_dir	, 'noAns'       )
							if OPTS.out_file:
														with open(OPTS.out_file	, 'w'       ) as f:
																					json.dump(UpperCAmelCase_	, UpperCAmelCase_       )
							else:
														print(json.dumps(UpperCAmelCase_	, indent=2       )       )


if __name__ == "__main__":
		lowerCamelCase									=							parse_args()
		if OPTS.out_image_dir:
				import matplotlib

				matplotlib.use("""Agg""")
				import matplotlib.pyplot as plt
		main()







import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case							:       Optional[int]										=	logging.get_logger(__name__)

snake_case							:       Optional[int]										=	{
    '''microsoft/unispeech-large-1500h-cv''': (
        '''https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json'''
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}







class 						_snake_case    ( _snake_case				):
 SCREAMING_SNAKE_CASE__ =			'unispeech'




 def __init__(						self   ,    _lowerCamelCase=32   ,    _lowerCamelCase=768   ,    _lowerCamelCase=12   ,    _lowerCamelCase=12   ,    _lowerCamelCase=3072   ,    _lowerCamelCase="gelu"   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.0   ,    _lowerCamelCase=0.0   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.02   ,    _lowerCamelCase=1e-5   ,    _lowerCamelCase="group"   ,    _lowerCamelCase="gelu"   ,    _lowerCamelCase=(512, 512, 512, 512, 512, 512, 512)   ,    _lowerCamelCase=(5, 2, 2, 2, 2, 2, 2)   ,    _lowerCamelCase=(10, 3, 3, 3, 3, 2, 2)   ,    _lowerCamelCase=False   ,    _lowerCamelCase=128   ,    _lowerCamelCase=16   ,    _lowerCamelCase=False   ,    _lowerCamelCase=True   ,    _lowerCamelCase=0.05   ,    _lowerCamelCase=10   ,    _lowerCamelCase=2   ,    _lowerCamelCase=0.0   ,    _lowerCamelCase=10   ,    _lowerCamelCase=0   ,    _lowerCamelCase=320   ,    _lowerCamelCase=2   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=100   ,    _lowerCamelCase=256   ,    _lowerCamelCase=256   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase="mean"   ,    _lowerCamelCase=False   ,    _lowerCamelCase=False   ,    _lowerCamelCase=256   ,    _lowerCamelCase=80   ,    _lowerCamelCase=0   ,    _lowerCamelCase=1   ,    _lowerCamelCase=2   ,    _lowerCamelCase=0.5   ,    **_lowerCamelCase   ,    ):
       super().__init__(**_lowerCamelCase   ,    pad_token_id=_lowerCamelCase   ,    bos_token_id=_lowerCamelCase   ,    eos_token_id=_lowerCamelCase       )
       a		:Any									=			hidden_size
       a		:str									=			feat_extract_norm
       a		:List[Any]									=			feat_extract_activation
       a		:Tuple									=			list(_lowerCamelCase       )
       a		:Any									=			list(_lowerCamelCase       )
       a		:List[Any]									=			list(_lowerCamelCase       )
       a		:Union[str, Any]									=			conv_bias
       a		:str									=			num_conv_pos_embeddings
       a		:str									=			num_conv_pos_embedding_groups
       a		:Tuple									=			len(self.conv_dim       )
       a		:int									=			num_hidden_layers
       a		:Any									=			intermediate_size
       a		:Optional[Any]									=			hidden_act
       a		:Tuple									=			num_attention_heads
       a		:Any									=			hidden_dropout
       a		:Any									=			attention_dropout
       a		:Optional[Any]									=			activation_dropout
       a		:Optional[Any]									=			feat_proj_dropout
       a		:Any									=			final_dropout
       a		:int									=			layerdrop
       a		:int									=			layer_norm_eps
       a		:Dict									=			initializer_range
       a		:Dict									=			num_ctc_classes
       a		:Optional[Any]									=			vocab_size
       a		:str									=			do_stable_layer_norm
       a		:Tuple									=			use_weighted_layer_sum
       a		:Any									=			classifier_proj_size

       if (
           (len(self.conv_stride       ) != self.num_feat_extract_layers)
           or (len(self.conv_kernel       ) != self.num_feat_extract_layers)
           or (len(self.conv_dim       ) != self.num_feat_extract_layers)
       ):
             raise ValueError(
                 '''Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='''
                 ''' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='''
                 F''' {len(self.conv_dim       )}`, `len(config.conv_stride) = {len(self.conv_stride       )}`,'''
                 F''' `len(config.conv_kernel) = {len(self.conv_kernel       )}`.'''       )

       # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
       a		:List[Any]									=			apply_spec_augment
       a		:Any									=			mask_time_prob
       a		:Union[str, Any]									=			mask_time_length
       a		:str									=			mask_time_min_masks
       a		:Tuple									=			mask_feature_prob
       a		:Dict									=			mask_feature_length
       a		:int									=			mask_feature_min_masks

       # parameters for pretraining with codevector quantized representations
       a		:Union[str, Any]									=			num_codevectors_per_group
       a		:Dict									=			num_codevector_groups
       a		:List[Any]									=			contrastive_logits_temperature
       a		:Union[str, Any]									=			feat_quantizer_dropout
       a		:Optional[Any]									=			num_negatives
       a		:Tuple									=			codevector_dim
       a		:Optional[Any]									=			proj_codevector_dim
       a		:Union[str, Any]									=			diversity_loss_weight

       # ctc loss
       a		:List[Any]									=			ctc_loss_reduction
       a		:Union[str, Any]									=			ctc_zero_infinity

       # pretraining loss
       a		:int									=			replace_prob




 @property
 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       return functools.reduce(operator.mul   ,    self.conv_stride   ,    1       )





