


import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


snake_case    :   Optional[Any]	    =					re.compile(R'''\b(a|an|the)\b''', re.UNICODE)

snake_case    :   int	    =					None
def     __lowerCamelCase						(			):




      """simple docstring"""





      a						:Optional[Any]										=	argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''						)
      parser.add_argument('''data_file'''			,  metavar='''data.json'''			,  help='''Input data JSON file.'''						)
      parser.add_argument('''pred_file'''			,  metavar='''pred.json'''			,  help='''Model predictions.'''						)
      parser.add_argument(
          '''--out-file'''			,  '''-o'''			,  metavar='''eval.json'''			,  help='''Write accuracy metrics to file (default is stdout).'''						)
      parser.add_argument(
          '''--na-prob-file'''			,  '''-n'''			,  metavar='''na_prob.json'''			,  help='''Model estimates of probability of no answer.'''						)
      parser.add_argument(
          '''--na-prob-thresh'''			,  '''-t'''			,  type=UpperCAmelCase_			,  default=1.0			,  help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''			,  )
      parser.add_argument(
          '''--out-image-dir'''			,  '''-p'''			,  metavar='''out_images'''			,  default=UpperCAmelCase_			,  help='''Save precision-recall curves to directory.'''						)
      parser.add_argument('''--verbose'''			,  '''-v'''			,  action='''store_true'''						)
      if len(sys.argv						) == 1:
            parser.print_help()
            sys.exit(1						)
      return parser.parse_args()
def     __lowerCamelCase						(			UpperCAmelCase_						:		Union[str, Any]						):




      """simple docstring"""





      a						:Union[str, Any]										=	{}
      for article in dataset:
            for p in article["paragraphs"]:
                  for qa in p["qas"]:
                        a						:str										=	bool(qa['''answers''']['''text''']						)
      return qid_to_has_ans
def     __lowerCamelCase						(			UpperCAmelCase_						:		Union[str, Any]						):




      """simple docstring"""





      def remove_articles(UpperCAmelCase_						:		Tuple						):
            return ARTICLES_REGEX.sub(''' '''			,  UpperCAmelCase_						)

      def white_space_fix(UpperCAmelCase_						:		int						):
            return " ".join(text.split()						)

      def remove_punc(UpperCAmelCase_						:		Dict						):
            a						:Optional[Any]										=	set(string.punctuation						)
            return "".join(ch for ch in text if ch not in exclude						)

      def lower(UpperCAmelCase_						:		Union[str, Any]						):
            return text.lower()

      return white_space_fix(remove_articles(remove_punc(lower(UpperCAmelCase_						)						)						)						)
def     __lowerCamelCase						(			UpperCAmelCase_						:		List[Any]						):




      """simple docstring"""





      if not s:
            return []
      return normalize_answer(UpperCAmelCase_						).split()
def     __lowerCamelCase						(			UpperCAmelCase_						:		Any			,  UpperCAmelCase_						:		Any						):




      """simple docstring"""





      return int(normalize_answer(UpperCAmelCase_						) == normalize_answer(UpperCAmelCase_						)						)
def     __lowerCamelCase						(			UpperCAmelCase_						:		Optional[Any]			,  UpperCAmelCase_						:		List[Any]						):




      """simple docstring"""





      a						:str										=	get_tokens(UpperCAmelCase_						)
      a						:List[Any]										=	get_tokens(UpperCAmelCase_						)
      a						:Dict										=	collections.Counter(UpperCAmelCase_						) & collections.Counter(UpperCAmelCase_						)
      a						:Any										=	sum(common.values()						)
      if len(UpperCAmelCase_						) == 0 or len(UpperCAmelCase_						) == 0:
            # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
            return int(gold_toks == pred_toks						)
      if num_same == 0:
            return 0
      a						:int										=	1.0 * num_same / len(UpperCAmelCase_						)
      a						:Tuple										=	1.0 * num_same / len(UpperCAmelCase_						)
      a						:Any										=	(2 * precision * recall) / (precision + recall)
      return fa
def     __lowerCamelCase						(			UpperCAmelCase_						:		int			,  UpperCAmelCase_						:		Union[str, Any]						):




      """simple docstring"""





      a						:Tuple										=	{}
      a						:Any										=	{}
      for article in dataset:
            for p in article["paragraphs"]:
                  for qa in p["qas"]:
                        a						:Tuple										=	qa['''id''']
                        a						:int										=	[t for t in qa['''answers''']['''text'''] if normalize_answer(UpperCAmelCase_						)]
                        if not gold_answers:
                              # For unanswerable questions, only correct answer is empty string
                              a						:List[Any]										=	['''''']
                        if qid not in preds:
                              print(F'''Missing prediction for {qid}'''						)
                              continue
                        a						:str										=	preds[qid]
                        # Take max over all gold answers
                        a						:Optional[int]										=	max(compute_exact(UpperCAmelCase_			,  UpperCAmelCase_						) for a in gold_answers						)
                        a						:List[str]										=	max(compute_fa(UpperCAmelCase_			,  UpperCAmelCase_						) for a in gold_answers						)
      return exact_scores, fa_scores
def     __lowerCamelCase						(			UpperCAmelCase_						:		List[Any]			,  UpperCAmelCase_						:		List[str]			,  UpperCAmelCase_						:		Optional[int]			,  UpperCAmelCase_						:		int						):




      """simple docstring"""





      a						:List[Any]										=	{}
      for qid, s in scores.items():
            a						:Optional[int]										=	na_probs[qid] > na_prob_thresh
            if pred_na:
                  a						:List[str]										=	float(not qid_to_has_ans[qid]						)
            else:
                  a						:List[str]										=	s
      return new_scores
def     __lowerCamelCase						(			UpperCAmelCase_						:		List[Any]			,  UpperCAmelCase_						:		List[str]			,  UpperCAmelCase_						:		Tuple=None						):




      """simple docstring"""





      if not qid_list:
            a						:List[Any]										=	len(UpperCAmelCase_						)
            return collections.OrderedDict(
                [
                    ('''exact''', 100.0 * sum(exact_scores.values()						) / total),
                    ('''f1''', 100.0 * sum(fa_scores.values()						) / total),
                    ('''total''', total),
                ]						)
      else:
            a						:Tuple										=	len(UpperCAmelCase_						)
            return collections.OrderedDict(
                [
                    ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list						) / total),
                    ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list						) / total),
                    ('''total''', total),
                ]						)
def     __lowerCamelCase						(			UpperCAmelCase_						:		List[str]			,  UpperCAmelCase_						:		Dict			,  UpperCAmelCase_						:		str						):




      """simple docstring"""





      for k in new_eval:
            a						:Any										=	new_eval[k]
def     __lowerCamelCase						(			UpperCAmelCase_						:		List[str]			,  UpperCAmelCase_						:		Union[str, Any]			,  UpperCAmelCase_						:		Dict			,  UpperCAmelCase_						:		Optional[Any]						):




      """simple docstring"""





      plt.step(UpperCAmelCase_			,  UpperCAmelCase_			,  color='''b'''			,  alpha=0.2			,  where='''post'''						)
      plt.fill_between(UpperCAmelCase_			,  UpperCAmelCase_			,  step='''post'''			,  alpha=0.2			,  color='''b'''						)
      plt.xlabel('''Recall'''						)
      plt.ylabel('''Precision'''						)
      plt.xlim([0.0, 1.05]						)
      plt.ylim([0.0, 1.05]						)
      plt.title(UpperCAmelCase_						)
      plt.savefig(UpperCAmelCase_						)
      plt.clf()
def     __lowerCamelCase						(			UpperCAmelCase_						:		int			,  UpperCAmelCase_						:		List[str]			,  UpperCAmelCase_						:		Optional[int]			,  UpperCAmelCase_						:		Tuple			,  UpperCAmelCase_						:		Union[str, Any]=None			,  UpperCAmelCase_						:		int=None						):




      """simple docstring"""





      a						:Any										=	sorted(UpperCAmelCase_			,  key=lambda UpperCAmelCase_						: na_probs[k]						)
      a						:Optional[int]										=	0.0
      a						:Optional[Any]										=	1.0
      a						:Union[str, Any]										=	0.0
      a						:Optional[int]										=	[1.0]
      a						:Optional[int]										=	[0.0]
      a						:Union[str, Any]										=	0.0
      for i, qid in enumerate(UpperCAmelCase_						):
            if qid_to_has_ans[qid]:
                  true_pos += scores[qid]
            a						:List[str]										=	true_pos / float(i + 1						)
            a						:Any										=	true_pos / float(UpperCAmelCase_						)
            if i == len(UpperCAmelCase_						) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
                  # i.e., if we can put a threshold after this point
                  avg_prec += cur_p * (cur_r - recalls[-1])
                  precisions.append(UpperCAmelCase_						)
                  recalls.append(UpperCAmelCase_						)
      if out_image:
            plot_pr_curve(UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_						)
      return {"ap": 100.0 * avg_prec}
def     __lowerCamelCase						(			UpperCAmelCase_						:		int			,  UpperCAmelCase_						:		int			,  UpperCAmelCase_						:		List[str]			,  UpperCAmelCase_						:		Any			,  UpperCAmelCase_						:		List[Any]			,  UpperCAmelCase_						:		Tuple						):




      """simple docstring"""





      if out_image_dir and not os.path.exists(UpperCAmelCase_						):
            os.makedirs(UpperCAmelCase_						)
      a						:str										=	sum(1 for v in qid_to_has_ans.values() if v						)
      if num_true_pos == 0:
            return
      a						:List[str]										=	make_precision_recall_eval(
          UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  out_image=os.path.join(UpperCAmelCase_			,  '''pr_exact.png'''						)			,  title='''Precision-Recall curve for Exact Match score'''			,  )
      a						:Any										=	make_precision_recall_eval(
          UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  out_image=os.path.join(UpperCAmelCase_			,  '''pr_f1.png'''						)			,  title='''Precision-Recall curve for F1 score'''			,  )
      a						:List[Any]										=	{k: float(UpperCAmelCase_						) for k, v in qid_to_has_ans.items()}
      a						:Optional[Any]										=	make_precision_recall_eval(
          UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  out_image=os.path.join(UpperCAmelCase_			,  '''pr_oracle.png'''						)			,  title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''			,  )
      merge_eval(UpperCAmelCase_			,  UpperCAmelCase_			,  '''pr_exact'''						)
      merge_eval(UpperCAmelCase_			,  UpperCAmelCase_			,  '''pr_f1'''						)
      merge_eval(UpperCAmelCase_			,  UpperCAmelCase_			,  '''pr_oracle'''						)
def     __lowerCamelCase						(			UpperCAmelCase_						:		Optional[Any]			,  UpperCAmelCase_						:		int			,  UpperCAmelCase_						:		Union[str, Any]			,  UpperCAmelCase_						:		Tuple						):




      """simple docstring"""





      if not qid_list:
            return
      a						:Tuple										=	[na_probs[k] for k in qid_list]
      a						:Dict										=	np.ones_like(UpperCAmelCase_						) / float(len(UpperCAmelCase_						)						)
      plt.hist(UpperCAmelCase_			,  weights=UpperCAmelCase_			,  bins=20			,  range=(0.0, 1.0)						)
      plt.xlabel('''Model probability of no-answer'''						)
      plt.ylabel('''Proportion of dataset'''						)
      plt.title(F'''Histogram of no-answer probability: {name}'''						)
      plt.savefig(os.path.join(UpperCAmelCase_			,  F'''na_prob_hist_{name}.png'''						)						)
      plt.clf()
def     __lowerCamelCase						(			UpperCAmelCase_						:		Union[str, Any]			,  UpperCAmelCase_						:		Optional[int]			,  UpperCAmelCase_						:		str			,  UpperCAmelCase_						:		Optional[int]						):




      """simple docstring"""





      a						:int										=	sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]						)
      a						:List[str]										=	num_no_ans
      a						:List[str]										=	cur_score
      a						:Dict										=	0.0
      a						:Union[str, Any]										=	sorted(UpperCAmelCase_			,  key=lambda UpperCAmelCase_						: na_probs[k]						)
      for i, qid in enumerate(UpperCAmelCase_						):
            if qid not in scores:
                  continue
            if qid_to_has_ans[qid]:
                  a						:Tuple										=	scores[qid]
            else:
                  if preds[qid]:
                        a						:List[Any]										=	-1
                  else:
                        a						:Union[str, Any]										=	0
            cur_score += diff
            if cur_score > best_score:
                  a						:Any										=	cur_score
                  a						:Dict										=	na_probs[qid]
      return 100.0 * best_score / len(UpperCAmelCase_						), best_thresh
def     __lowerCamelCase						(			UpperCAmelCase_						:		Optional[Any]			,  UpperCAmelCase_						:		List[Any]			,  UpperCAmelCase_						:		Dict			,  UpperCAmelCase_						:		Dict			,  UpperCAmelCase_						:		Optional[int]			,  UpperCAmelCase_						:		Union[str, Any]						):




      """simple docstring"""





      a						:str										=	find_best_thresh(UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_						)
      a						:Optional[Any]										=	find_best_thresh(UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_						)
      a						:Dict										=	best_exact
      a						:Optional[int]										=	exact_thresh
      a						:List[Any]										=	best_fa
      a						:Optional[int]										=	fa_thresh
def     __lowerCamelCase						(			):




      """simple docstring"""





      with open(OPTS.data_file						) as f:
            a						:Dict										=	json.load(UpperCAmelCase_						)
            a						:Optional[Any]										=	dataset_json['''data''']
      with open(OPTS.pred_file						) as f:
            a						:List[str]										=	json.load(UpperCAmelCase_						)
      if OPTS.na_prob_file:
            with open(OPTS.na_prob_file						) as f:
                  a						:Any										=	json.load(UpperCAmelCase_						)
      else:
            a						:Optional[Any]										=	{k: 0.0 for k in preds}
      a						:Tuple										=	make_qid_to_has_ans(UpperCAmelCase_						)  # maps qid to True/False
      a						:Dict										=	[k for k, v in qid_to_has_ans.items() if v]
      a						:Optional[Any]										=	[k for k, v in qid_to_has_ans.items() if not v]
      a						:Union[str, Any]										=	get_raw_scores(UpperCAmelCase_			,  UpperCAmelCase_						)
      a						:Tuple										=	apply_no_ans_threshold(UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  OPTS.na_prob_thresh						)
      a						:Optional[Any]										=	apply_no_ans_threshold(UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  OPTS.na_prob_thresh						)
      a						:Any										=	make_eval_dict(UpperCAmelCase_			,  UpperCAmelCase_						)
      if has_ans_qids:
            a						:Optional[int]										=	make_eval_dict(UpperCAmelCase_			,  UpperCAmelCase_			,  qid_list=UpperCAmelCase_						)
            merge_eval(UpperCAmelCase_			,  UpperCAmelCase_			,  '''HasAns'''						)
      if no_ans_qids:
            a						:Dict										=	make_eval_dict(UpperCAmelCase_			,  UpperCAmelCase_			,  qid_list=UpperCAmelCase_						)
            merge_eval(UpperCAmelCase_			,  UpperCAmelCase_			,  '''NoAns'''						)
      if OPTS.na_prob_file:
            find_all_best_thresh(UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_						)
      if OPTS.na_prob_file and OPTS.out_image_dir:
            run_precision_recall_analysis(UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  UpperCAmelCase_			,  OPTS.out_image_dir						)
            histogram_na_prob(UpperCAmelCase_			,  UpperCAmelCase_			,  OPTS.out_image_dir			,  '''hasAns'''						)
            histogram_na_prob(UpperCAmelCase_			,  UpperCAmelCase_			,  OPTS.out_image_dir			,  '''noAns'''						)
      if OPTS.out_file:
            with open(OPTS.out_file			,  '''w'''						) as f:
                  json.dump(UpperCAmelCase_			,  UpperCAmelCase_						)
      else:
            print(json.dumps(UpperCAmelCase_			,  indent=2						)						)


if __name__ == "__main__":
     snake_case    :   Any	    =					parse_args()
     if OPTS.out_image_dir:
          import matplotlib

          matplotlib.use('''Agg''')
          import matplotlib.pyplot as plt
     main()









# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import torch

from ..models.speechta import SpeechTaForTextToSpeech, SpeechTaHifiGan, SpeechTaProcessor
from ..utils import is_datasets_available
from .base import PipelineTool


if is_datasets_available():
     from datasets import load_dataset







class 		_snake_case  (				_snake_case							):
 SCREAMING_SNAKE_CASE__				=	'microsoft/speecht5_tts'
 SCREAMING_SNAKE_CASE__				=	(
     'This is a tool that reads an English text out loud. It takes an input named `text` which should contain the '
     'text to read (in English) and returns a waveform object containing the sound.'
 )
 SCREAMING_SNAKE_CASE__				=	'text_reader'
 SCREAMING_SNAKE_CASE__				=	SpeechTaProcessor
 SCREAMING_SNAKE_CASE__				=	SpeechTaForTextToSpeech
 SCREAMING_SNAKE_CASE__				=	SpeechTaHifiGan

 SCREAMING_SNAKE_CASE__				=	['text']
 SCREAMING_SNAKE_CASE__				=	['audio']




 def 	SCREAMING_SNAKE_CASE__    (		self   ):
       if self.post_processor is None:
             a						:List[Any]										=	'''microsoft/speecht5_hifigan'''
       super().setup()




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase ,  _lowerCamelCase=None   ):
       a						:Tuple										=	self.pre_processor(text=_lowerCamelCase ,  return_tensors='''pt''' ,  truncation=_lowerCamelCase   )

       if speaker_embeddings is None:
             if not is_datasets_available():
                   raise ImportError('''Datasets needs to be installed if not passing speaker embeddings.'''   )

             a						:List[Any]										=	load_dataset('''Matthijs/cmu-arctic-xvectors''' ,  split='''validation'''   )
             a						:int										=	torch.tensor(embeddings_dataset[7305]['''xvector''']   ).unsqueeze(0   )

       return {"input_ids": inputs["input_ids"], "speaker_embeddings": speaker_embeddings}




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       with torch.no_grad():
             return self.model.generate_speech(**_lowerCamelCase   )




 def 	SCREAMING_SNAKE_CASE__    (		self ,  _lowerCamelCase   ):
       with torch.no_grad():
             return self.post_processor(_lowerCamelCase   ).cpu().detach()





