






import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


SCREAMING_SNAKE_CASE__				     =			re.compile(r'''\b(a|an|the)\b''', re.UNICODE)

SCREAMING_SNAKE_CASE__				     =			None

def        UpperCAmelCase__		(						):
			__a						:  Optional[Any]									=     argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'				)
			parser.add_argument('data_file'   ,  metavar='data.json'   ,  help='Input data JSON file.'				)
			parser.add_argument('pred_file'   ,  metavar='pred.json'   ,  help='Model predictions.'				)
			parser.add_argument(
			    '--out-file'   ,  '-o'   ,  metavar='eval.json'   ,  help='Write accuracy metrics to file (default is stdout).'				)
			parser.add_argument(
			    '--na-prob-file'   ,  '-n'   ,  metavar='na_prob.json'   ,  help='Model estimates of probability of no answer.'				)
			parser.add_argument(
			    '--na-prob-thresh'   ,  '-t'   ,  type=lowerCamelCase_   ,  default=1.0   ,  help='Predict "" if no-answer probability exceeds this (default = 1.0).'   ,  )
			parser.add_argument(
			    '--out-image-dir'   ,  '-p'   ,  metavar='out_images'   ,  default=lowerCamelCase_   ,  help='Save precision-recall curves to directory.'				)
			parser.add_argument('--verbose'   ,  '-v'   ,  action='store_true'				)
			if len(sys.argv				) == 1:
						parser.print_help()
						sys.exit(1				)
			return parser.parse_args()

def        UpperCAmelCase__		(						lowerCamelCase_       :       str				):
			__a						:  Union[str, Any]									=     {}
			for article in dataset:
						for p in article["paragraphs"]:
									for qa in p["qas"]:
												__a						:  int									=     bool(qa['answers']['text']				)
			return qid_to_has_ans

def        UpperCAmelCase__		(						lowerCamelCase_       :       Dict				):

			def remove_articles(lowerCamelCase_       :       List[Any]				):
						return ARTICLES_REGEX.sub(' '   ,  lowerCamelCase_				)

			def white_space_fix(lowerCamelCase_       :       Optional[Any]				):
						return " ".join(text.split()				)

			def remove_punc(lowerCamelCase_       :       Optional[Any]				):
						__a						:  Tuple									=     set(string.punctuation				)
						return "".join(ch for ch in text if ch not in exclude				)

			def lower(lowerCamelCase_       :       Any				):
						return text.lower()

			return white_space_fix(remove_articles(remove_punc(lower(lowerCamelCase_				)				)				)				)

def        UpperCAmelCase__		(						lowerCamelCase_       :       List[str]				):
			if not s:
						return []
			return normalize_answer(lowerCamelCase_				).split()

def        UpperCAmelCase__		(						lowerCamelCase_       :       str   ,  lowerCamelCase_       :       Optional[Any]				):
			return int(normalize_answer(lowerCamelCase_				) == normalize_answer(lowerCamelCase_				)				)

def        UpperCAmelCase__		(						lowerCamelCase_       :       int   ,  lowerCamelCase_       :       Any				):
			__a						:  int									=     get_tokens(lowerCamelCase_				)
			__a						:  Optional[int]									=     get_tokens(lowerCamelCase_				)
			__a						:  Any									=     collections.Counter(lowerCamelCase_				) & collections.Counter(lowerCamelCase_				)
			__a						:  List[str]									=     sum(common.values()				)
			if len(lowerCamelCase_				) == 0 or len(lowerCamelCase_				) == 0:
						# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
						return int(gold_toks == pred_toks				)
			if num_same == 0:
						return 0
			__a						:  List[str]									=     1.0 * num_same / len(lowerCamelCase_				)
			__a						:  Union[str, Any]									=     1.0 * num_same / len(lowerCamelCase_				)
			__a						:  int									=     (2 * precision * recall) / (precision + recall)
			return fa

def        UpperCAmelCase__		(						lowerCamelCase_       :       Union[str, Any]   ,  lowerCamelCase_       :       List[Any]				):
			__a						:  List[str]									=     {}
			__a						:  str									=     {}
			for article in dataset:
						for p in article["paragraphs"]:
									for qa in p["qas"]:
												__a						:  Optional[int]									=     qa['id']
												__a						:  Union[str, Any]									=     [t for t in qa['answers']['text'] if normalize_answer(lowerCamelCase_				)]
												if not gold_answers:
															# For unanswerable questions, only correct answer is empty string
															__a						:  List[Any]									=     ['']
												if qid not in preds:
															print(f'''Missing prediction for {qid}'''				)
															continue
												__a						:  List[str]									=     preds[qid]
												# Take max over all gold answers
												__a						:  List[str]									=     max(compute_exact(lowerCamelCase_   ,  lowerCamelCase_				) for a in gold_answers				)
												__a						:  Optional[Any]									=     max(compute_fa(lowerCamelCase_   ,  lowerCamelCase_				) for a in gold_answers				)
			return exact_scores, fa_scores

def        UpperCAmelCase__		(						lowerCamelCase_       :       str   ,  lowerCamelCase_       :       Dict   ,  lowerCamelCase_       :       Dict   ,  lowerCamelCase_       :       Dict				):
			__a						:  List[Any]									=     {}
			for qid, s in scores.items():
						__a						:  Any									=     na_probs[qid] > na_prob_thresh
						if pred_na:
									__a						:  Any									=     float(not qid_to_has_ans[qid]				)
						else:
									__a						:  List[str]									=     s
			return new_scores

def        UpperCAmelCase__		(						lowerCamelCase_       :       Dict   ,  lowerCamelCase_       :       List[Any]   ,  lowerCamelCase_       :       Union[str, Any]=None				):
			if not qid_list:
						__a						:  Optional[Any]									=     len(lowerCamelCase_				)
						return collections.OrderedDict(
						    [
						        ('exact', 100.0 * sum(exact_scores.values()				) / total),
						        ('f1', 100.0 * sum(fa_scores.values()				) / total),
						        ('total', total),
						    ]				)
			else:
						__a						:  Dict									=     len(lowerCamelCase_				)
						return collections.OrderedDict(
						    [
						        ('exact', 100.0 * sum(exact_scores[k] for k in qid_list				) / total),
						        ('f1', 100.0 * sum(fa_scores[k] for k in qid_list				) / total),
						        ('total', total),
						    ]				)

def        UpperCAmelCase__		(						lowerCamelCase_       :       str   ,  lowerCamelCase_       :       int   ,  lowerCamelCase_       :       Optional[int]				):
			for k in new_eval:
						__a						:  Tuple									=     new_eval[k]

def        UpperCAmelCase__		(						lowerCamelCase_       :       Optional[int]   ,  lowerCamelCase_       :       str   ,  lowerCamelCase_       :       Union[str, Any]   ,  lowerCamelCase_       :       Optional[int]				):
			plt.step(lowerCamelCase_   ,  lowerCamelCase_   ,  color='b'   ,  alpha=0.2   ,  where='post'				)
			plt.fill_between(lowerCamelCase_   ,  lowerCamelCase_   ,  step='post'   ,  alpha=0.2   ,  color='b'				)
			plt.xlabel('Recall'				)
			plt.ylabel('Precision'				)
			plt.xlim([0.0, 1.05]				)
			plt.ylim([0.0, 1.05]				)
			plt.title(lowerCamelCase_				)
			plt.savefig(lowerCamelCase_				)
			plt.clf()

def        UpperCAmelCase__		(						lowerCamelCase_       :       Optional[int]   ,  lowerCamelCase_       :       Optional[Any]   ,  lowerCamelCase_       :       List[Any]   ,  lowerCamelCase_       :       Any   ,  lowerCamelCase_       :       List[str]=None   ,  lowerCamelCase_       :       Optional[Any]=None				):
			__a						:  Tuple									=     sorted(lowerCamelCase_   ,  key=lambda lowerCamelCase_				: na_probs[k]				)
			__a						:  Any									=     0.0
			__a						:  List[str]									=     1.0
			__a						:  Any									=     0.0
			__a						:  Optional[int]									=     [1.0]
			__a						:  str									=     [0.0]
			__a						:  Union[str, Any]									=     0.0
			for i, qid in enumerate(lowerCamelCase_				):
						if qid_to_has_ans[qid]:
									true_pos += scores[qid]
						__a						:  Union[str, Any]									=     true_pos / float(i + 1				)
						__a						:  Tuple									=     true_pos / float(lowerCamelCase_				)
						if i == len(lowerCamelCase_				) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
									# i.e., if we can put a threshold after this point
									avg_prec += cur_p * (cur_r - recalls[-1])
									precisions.append(lowerCamelCase_				)
									recalls.append(lowerCamelCase_				)
			if out_image:
						plot_pr_curve(lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_				)
			return {"ap": 100.0 * avg_prec}

def        UpperCAmelCase__		(						lowerCamelCase_       :       int   ,  lowerCamelCase_       :       int   ,  lowerCamelCase_       :       Union[str, Any]   ,  lowerCamelCase_       :       int   ,  lowerCamelCase_       :       Optional[int]   ,  lowerCamelCase_       :       Union[str, Any]				):
			if out_image_dir and not os.path.exists(lowerCamelCase_				):
						os.makedirs(lowerCamelCase_				)
			__a						:  List[str]									=     sum(1 for v in qid_to_has_ans.values() if v				)
			if num_true_pos == 0:
						return
			__a						:  Any									=     make_precision_recall_eval(
			    lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  out_image=os.path.join(lowerCamelCase_   ,  'pr_exact.png'				)   ,  title='Precision-Recall curve for Exact Match score'   ,  )
			__a						:  Tuple									=     make_precision_recall_eval(
			    lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  out_image=os.path.join(lowerCamelCase_   ,  'pr_f1.png'				)   ,  title='Precision-Recall curve for F1 score'   ,  )
			__a						:  str									=     {k: float(lowerCamelCase_				) for k, v in qid_to_has_ans.items()}
			__a						:  List[str]									=     make_precision_recall_eval(
			    lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  out_image=os.path.join(lowerCamelCase_   ,  'pr_oracle.png'				)   ,  title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'   ,  )
			merge_eval(lowerCamelCase_   ,  lowerCamelCase_   ,  'pr_exact'				)
			merge_eval(lowerCamelCase_   ,  lowerCamelCase_   ,  'pr_f1'				)
			merge_eval(lowerCamelCase_   ,  lowerCamelCase_   ,  'pr_oracle'				)

def        UpperCAmelCase__		(						lowerCamelCase_       :       Any   ,  lowerCamelCase_       :       Optional[int]   ,  lowerCamelCase_       :       Optional[Any]   ,  lowerCamelCase_       :       Any				):
			if not qid_list:
						return
			__a						:  List[Any]									=     [na_probs[k] for k in qid_list]
			__a						:  Tuple									=     np.ones_like(lowerCamelCase_				) / float(len(lowerCamelCase_				)				)
			plt.hist(lowerCamelCase_   ,  weights=lowerCamelCase_   ,  bins=2_0   ,  range=(0.0, 1.0)				)
			plt.xlabel('Model probability of no-answer'				)
			plt.ylabel('Proportion of dataset'				)
			plt.title(f'''Histogram of no-answer probability: {name}'''				)
			plt.savefig(os.path.join(lowerCamelCase_   ,  f'''na_prob_hist_{name}.png'''				)				)
			plt.clf()

def        UpperCAmelCase__		(						lowerCamelCase_       :       Optional[Any]   ,  lowerCamelCase_       :       Optional[int]   ,  lowerCamelCase_       :       Optional[int]   ,  lowerCamelCase_       :       List[str]				):
			__a						:  List[Any]									=     sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]				)
			__a						:  Dict									=     num_no_ans
			__a						:  List[Any]									=     cur_score
			__a						:  List[Any]									=     0.0
			__a						:  Dict									=     sorted(lowerCamelCase_   ,  key=lambda lowerCamelCase_				: na_probs[k]				)
			for i, qid in enumerate(lowerCamelCase_				):
						if qid not in scores:
									continue
						if qid_to_has_ans[qid]:
									__a						:  Optional[Any]									=     scores[qid]
						else:
									if preds[qid]:
												__a						:  Any									=     -1
									else:
												__a						:  Any									=     0
						cur_score += diff
						if cur_score > best_score:
									__a						:  List[str]									=     cur_score
									__a						:  List[Any]									=     na_probs[qid]
			return 100.0 * best_score / len(lowerCamelCase_				), best_thresh

def        UpperCAmelCase__		(						lowerCamelCase_       :       int   ,  lowerCamelCase_       :       Any   ,  lowerCamelCase_       :       str   ,  lowerCamelCase_       :       int   ,  lowerCamelCase_       :       str   ,  lowerCamelCase_       :       List[Any]				):
			__a						,					__a						:  Optional[int]									=     find_best_thresh(lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_				)
			__a						,					__a						:  int									=     find_best_thresh(lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_				)
			__a						:  str									=     best_exact
			__a						:  Union[str, Any]									=     exact_thresh
			__a						:  Dict									=     best_fa
			__a						:  List[Any]									=     fa_thresh

def        UpperCAmelCase__		(						):
			with open(OPTS.data_file				) as f:
						__a						:  Dict									=     json.load(lowerCamelCase_				)
						__a						:  Union[str, Any]									=     dataset_json['data']
			with open(OPTS.pred_file				) as f:
						__a						:  Optional[int]									=     json.load(lowerCamelCase_				)
			if OPTS.na_prob_file:
						with open(OPTS.na_prob_file				) as f:
									__a						:  List[Any]									=     json.load(lowerCamelCase_				)
			else:
						__a						:  Optional[int]									=     {k: 0.0 for k in preds}
			__a						:  List[Any]									=     make_qid_to_has_ans(lowerCamelCase_				)  # maps qid to True/False
			__a						:  int									=     [k for k, v in qid_to_has_ans.items() if v]
			__a						:  int									=     [k for k, v in qid_to_has_ans.items() if not v]
			__a						,					__a						:  int									=     get_raw_scores(lowerCamelCase_   ,  lowerCamelCase_				)
			__a						:  List[str]									=     apply_no_ans_threshold(lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  OPTS.na_prob_thresh				)
			__a						:  int									=     apply_no_ans_threshold(lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  OPTS.na_prob_thresh				)
			__a						:  str									=     make_eval_dict(lowerCamelCase_   ,  lowerCamelCase_				)
			if has_ans_qids:
						__a						:  int									=     make_eval_dict(lowerCamelCase_   ,  lowerCamelCase_   ,  qid_list=lowerCamelCase_				)
						merge_eval(lowerCamelCase_   ,  lowerCamelCase_   ,  'HasAns'				)
			if no_ans_qids:
						__a						:  List[Any]									=     make_eval_dict(lowerCamelCase_   ,  lowerCamelCase_   ,  qid_list=lowerCamelCase_				)
						merge_eval(lowerCamelCase_   ,  lowerCamelCase_   ,  'NoAns'				)
			if OPTS.na_prob_file:
						find_all_best_thresh(lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_				)
			if OPTS.na_prob_file and OPTS.out_image_dir:
						run_precision_recall_analysis(lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  lowerCamelCase_   ,  OPTS.out_image_dir				)
						histogram_na_prob(lowerCamelCase_   ,  lowerCamelCase_   ,  OPTS.out_image_dir   ,  'hasAns'				)
						histogram_na_prob(lowerCamelCase_   ,  lowerCamelCase_   ,  OPTS.out_image_dir   ,  'noAns'				)
			if OPTS.out_file:
						with open(OPTS.out_file   ,  'w'				) as f:
									json.dump(lowerCamelCase_   ,  lowerCamelCase_				)
			else:
						print(json.dumps(lowerCamelCase_   ,  indent=2				)				)


if __name__ == "__main__":
							SCREAMING_SNAKE_CASE__				     =			parse_args()
							if OPTS.out_image_dir:
														import matplotlib

														matplotlib.use('''Agg''')
														import matplotlib.pyplot as plt
							main()










'''simple docstring'''

import faiss  # noqa: F401 # Here to have a nice missing dependency error message early on
import numpy  # noqa: F401 # Here to have a nice missing dependency error message early on
import requests  # noqa: F401 # Here to have a nice missing dependency error message early on
import sklearn  # noqa: F401 # Here to have a nice missing dependency error message early on
import tqdm  # noqa: F401 # Here to have a nice missing dependency error message early on
from mauve import compute_mauve  # From: mauve-text

import datasets


__lowerCAmelCase		     =  '''\
@inproceedings{pillutla-etal:mauve:neurips2021,
  title={MAUVE: Measuring the Gap Between Neural Text and Human Text using Divergence Frontiers},
  author={Pillutla, Krishna and Swayamdipta, Swabha and Zellers, Rowan and Thickstun, John and Welleck, Sean and Choi, Yejin and Harchaoui, Zaid},
  booktitle = {NeurIPS},
  year      = {2021}
}

'''

__lowerCAmelCase		     =  '''\
MAUVE is a library built on PyTorch and HuggingFace Transformers to measure the gap between neural text and human text with the eponymous MAUVE measure.

MAUVE summarizes both Type I and Type II errors measured softly using Kullback–Leibler (KL) divergences.

For details, see the MAUVE paper: https://arxiv.org/abs/2102.01454 (Neurips, 2021).

This metrics is a wrapper around the official implementation of MAUVE:
https://github.com/krishnap25/mauve
'''

__lowerCAmelCase		     =  '''
Calculates MAUVE scores between two lists of generated text and reference text.
Args:
    predictions: list of generated text to score. Each predictions
        should be a string with tokens separated by spaces.
    references: list of reference for each prediction. Each
        reference should be a string with tokens separated by spaces.
Optional Args:
    num_buckets: the size of the histogram to quantize P and Q. Options: \'auto\' (default) or an integer
    pca_max_data: the number data points to use for PCA dimensionality reduction prior to clustering. If -1, use all the data. Default -1
    kmeans_explained_var: amount of variance of the data to keep in dimensionality reduction by PCA. Default 0.9
    kmeans_num_redo: number of times to redo k-means clustering (the best objective is kept). Default 5
    kmeans_max_iter: maximum number of k-means iterations. Default 500
    featurize_model_name: name of the model from which features are obtained. Default \'gpt2-large\' Use one of [\'gpt2\', \'gpt2-medium\', \'gpt2-large\', \'gpt2-xl\'].
    device_id: Device for featurization. Supply a GPU id (e.g. 0 or 3) to use GPU. If no GPU with this id is found, use CPU
    max_text_length: maximum number of tokens to consider. Default 1024
    divergence_curve_discretization_size: Number of points to consider on the divergence curve. Default 25
    mauve_scaling_factor: "c" from the paper. Default 5.
    verbose: If True (default), print running time updates
    seed: random seed to initialize k-means cluster assignments.
Returns:
    mauve: MAUVE score, a number between 0 and 1. Larger values indicate that P and Q are closer,
    frontier_integral: Frontier Integral, a number between 0 and 1. Smaller values indicate that P and Q are closer,
    divergence_curve: a numpy.ndarray of shape (m, 2); plot it with matplotlib to view the divergence curve,
    p_hist: a discrete distribution, which is a quantized version of the text distribution p_text,
    q_hist: same as above, but with q_text.
Examples:

    >>> # faiss segfaults in doctest for some reason, so the .compute call is not tested with doctest
    >>> import datasets
    >>> mauve = datasets.load_metric(\'mauve\')
    >>> predictions = ["hello there", "general kenobi"]
    >>> references = ["hello there", "general kenobi"]
    >>> out = mauve.compute(predictions=predictions, references=references) # doctest: +SKIP
    >>> print(out.mauve) # doctest: +SKIP
    1.0
'''

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,		_KWARGS_DESCRIPTION  )
class       __magic_name__		(       datasets.Metric  ):







							def 				__lowercase     (				self   :			Optional[Any]						):
									return datasets.MetricInfo(
									    description=_DESCRIPTION      ,citation=_CITATION      ,homepage='https://github.com/krishnap25/mauve'      ,inputs_description=_KWARGS_DESCRIPTION      ,features=datasets.Features(
									        {
									            'predictions': datasets.Value('string'      ,id='sequence'						),
									            'references': datasets.Value('string'      ,id='sequence'						),
									        }						)      ,codebase_urls=['https://github.com/krishnap25/mauve']      ,reference_urls=[
									        'https://arxiv.org/abs/2102.01454',
									        'https://github.com/krishnap25/mauve',
									    ]      ,)







							def 				__lowercase     (				self   :			Tuple      ,_UpperCAmelCase   :			Optional[Any]      ,_UpperCAmelCase   :			Dict      ,_UpperCAmelCase   :			Any=None      ,_UpperCAmelCase   :			Tuple=None      ,_UpperCAmelCase   :			List[str]=None      ,_UpperCAmelCase   :			Optional[Any]=None      ,_UpperCAmelCase   :			Any="auto"      ,_UpperCAmelCase   :			Tuple=-1      ,_UpperCAmelCase   :			Union[str, Any]=0.9      ,_UpperCAmelCase   :			List[Any]=5      ,_UpperCAmelCase   :			Optional[int]=500      ,_UpperCAmelCase   :			int="gpt2-large"      ,_UpperCAmelCase   :			int=-1      ,_UpperCAmelCase   :			Tuple=1024      ,_UpperCAmelCase   :			Any=25      ,_UpperCAmelCase   :			int=5      ,_UpperCAmelCase   :			Optional[int]=True      ,_UpperCAmelCase   :			Optional[int]=25      ,):
									_a	: Any								=				compute_mauve(
									    p_text=_UpperCAmelCase      ,q_text=_UpperCAmelCase      ,p_features=_UpperCAmelCase      ,q_features=_UpperCAmelCase      ,p_tokens=_UpperCAmelCase      ,q_tokens=_UpperCAmelCase      ,num_buckets=_UpperCAmelCase      ,pca_max_data=_UpperCAmelCase      ,kmeans_explained_var=_UpperCAmelCase      ,kmeans_num_redo=_UpperCAmelCase      ,kmeans_max_iter=_UpperCAmelCase      ,featurize_model_name=_UpperCAmelCase      ,device_id=_UpperCAmelCase      ,max_text_length=_UpperCAmelCase      ,divergence_curve_discretization_size=_UpperCAmelCase      ,mauve_scaling_factor=_UpperCAmelCase      ,verbose=_UpperCAmelCase      ,seed=_UpperCAmelCase      ,)
									return out




