

from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import fa_score, matthews_corrcoef

import datasets


lowercase_   							=  '\\n@inproceedings{wang2019glue,\n  title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding},\n  author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.},\n  note={In the Proceedings of ICLR.},\n  year={2019}\n}\n'

lowercase_   							=  '\\nGLUE, the General Language Understanding Evaluation benchmark\n(https://gluebenchmark.com/) is a collection of resources for training,\nevaluating, and analyzing natural language understanding systems.\n'

lowercase_   							=  '\nCompute GLUE evaluation metric associated to each GLUE dataset.\nArgs:\n    predictions: list of predictions to score.\n        Each translation should be tokenized into a list of tokens.\n    references: list of lists of references for each translation.\n        Each reference should be tokenized into a list of tokens.\nReturns: depending on the GLUE subset, one or several of:\n    "accuracy": Accuracy\n    "f1": F1 score\n    "pearson": Pearson Correlation\n    "spearmanr": Spearman Correlation\n    "matthews_correlation": Matthew Correlation\nExamples:\n\n    >>> glue_metric = datasets.load_metric(\'glue\', \'sst2\')  # \'sst2\' or any of ["mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> glue_metric = datasets.load_metric(\'glue\', \'mrpc\')  # \'mrpc\' or \'qqp\'\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> glue_metric = datasets.load_metric(\'glue\', \'stsb\')\n    >>> references = [0., 1., 2., 3., 4., 5.]\n    >>> predictions = [0., 1., 2., 3., 4., 5.]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print({"pearson": round(results["pearson"], 2), "spearmanr": round(results["spearmanr"], 2)})\n    {\'pearson\': 1.0, \'spearmanr\': 1.0}\n\n    >>> glue_metric = datasets.load_metric(\'glue\', \'cola\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'matthews_correlation\': 1.0}\n'
def 	a (     A__      :  Optional[int] ,  A__      :  Dict	)       -> Dict:





      """simple docstring"""



      return float((preds == labels).mean()	)
def 	a (     A__      :  Any ,  A__      :  int	)       -> Any:





      """simple docstring"""



      _lowercase      			=simple_accuracy(A__ ,  A__	)
      _lowercase      			=float(fa_score(y_true=A__ ,  y_pred=A__	)	)
      return {
          "accuracy": acc,
          "f1": fa,
      }
def 	a (     A__      :  Any ,  A__      :  List[Any]	)       -> int:





      """simple docstring"""



      _lowercase      			=float(pearsonr(A__ ,  A__	)[0]	)
      _lowercase      			=float(spearmanr(A__ ,  A__	)[0]	)
      return {
          "pearson": pearson_corr,
          "spearmanr": spearman_corr,
      }







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,     _KWARGS_DESCRIPTION				)
class 					__lowerCAmelCase	(   datasets.Metric				):




  def     A__	(							self   )    ->					Union[str, Any]:
        '''simple docstring'''






        if self.config_name not in [
            "sst2",
            "mnli",
            "mnli_mismatched",
            "mnli_matched",
            "cola",
            "stsb",
            "mrpc",
            "qqp",
            "qnli",
            "rte",
            "wnli",
            "hans",
        ]:
              raise KeyError(
                  'You should supply a configuration name selected in '
                  '["sst2", "mnli", "mnli_mismatched", "mnli_matched", '
                  '"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'   )
        return datasets.MetricInfo(
            description=_DESCRIPTION				,  citation=_CITATION				,  inputs_description=_KWARGS_DESCRIPTION				,  features=datasets.Features(
                {
                    'predictions': datasets.Value('int64' if self.config_name != 'stsb' else 'float32'   ),
                    'references': datasets.Value('int64' if self.config_name != 'stsb' else 'float32'   ),
                }   )				,  codebase_urls=[]				,  reference_urls=[]				,  format='numpy'				,  )




  def     A__	(							self				,  lowerCAmelCase				,  lowerCAmelCase   )    ->					Dict:
        '''simple docstring'''






        if self.config_name == "cola":
              return {"matthews_correlation": matthews_corrcoef(lowerCAmelCase				,  lowerCAmelCase   )}
        elif self.config_name == "stsb":
              return pearson_and_spearman(lowerCAmelCase				,  lowerCAmelCase   )
        elif self.config_name in ["mrpc", "qqp"]:
              return acc_and_fa(lowerCAmelCase				,  lowerCAmelCase   )
        elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]:
              return {"accuracy": simple_accuracy(lowerCAmelCase				,  lowerCAmelCase   )}
        else:
              raise KeyError(
                  'You should supply a configuration name selected in '
                  '["sst2", "mnli", "mnli_mismatched", "mnli_matched", '
                  '"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'   )





import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin


lowercase_   							=  '\nHugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf originally as a company that developed a chatbot app targeted at teenagers.[2] After open-sourcing the model behind the chatbot, the company pivoted to focus on being a platform for machine learning.\n\nIn March 2021, Hugging Face raised $40 million in a Series B funding round.[3]\n\nOn April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model.[4] In 2022, the workshop concluded with the announcement of BLOOM, a multilingual large language model with 176 billion parameters.[5]\n'





class 					__lowerCAmelCase	(   unittest.TestCase     ,     SCREAMING_SNAKE_CASE				):




  def     A__	(							self   )    ->					Union[str, Any]:
        '''simple docstring'''






        _lowercase      			=load_tool('text-question-answering'   )
        self.tool.setup()
        _lowercase      			=load_tool('text-question-answering'				,  remote=lowerCAmelCase   )




  def     A__	(							self   )    ->					Dict:
        '''simple docstring'''






        _lowercase      			=self.tool(lowerCAmelCase				,  'What did Hugging Face do in April 2021?'   )
        self.assertEqual(lowerCAmelCase				,  'launched the BigScience Research Workshop'   )




  def     A__	(							self   )    ->					Optional[Any]:
        '''simple docstring'''






        _lowercase      			=self.remote_tool(lowerCAmelCase				,  'What did Hugging Face do in April 2021?'   )
        self.assertEqual(lowerCAmelCase				,  'launched the BigScience Research Workshop'   )




  def     A__	(							self   )    ->					Any:
        '''simple docstring'''






        _lowercase      			=self.tool(text=lowerCAmelCase				,  question='What did Hugging Face do in April 2021?'   )
        self.assertEqual(lowerCAmelCase				,  'launched the BigScience Research Workshop'   )




  def     A__	(							self   )    ->					Optional[int]:
        '''simple docstring'''






        _lowercase      			=self.remote_tool(text=lowerCAmelCase				,  question='What did Hugging Face do in April 2021?'   )
        self.assertEqual(lowerCAmelCase				,  'launched the BigScience Research Workshop'   )


