



'''simple docstring'''





from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


A__    :   List[str]           =					'''\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
'''

A__    :   str           =					'''\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
'''

A__    :   Optional[Any]           =					'''
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for \'record\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'prediction_text\': the predicted answer text
        - for \'multirc\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question-answer pair as specified by the dataset
            - \'prediction\': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for \'record\': list of question-answers dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'answers\': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for \'record\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1\': F1 score
    - for \'multirc\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1_m\': Per-question macro-F1 score
        - \'f1_a\': Average F1 score over all answers
    - for \'axb\':
        \'matthews_correlation\': Matthew Correlation
    - for \'cb\':
        - \'accuracy\': Accuracy
        - \'f1\': F1 score
    - for all others:
        - \'accuracy\': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')
    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]
    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')
    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''





def   a_   (		_UpperCAmelCase    :      int ,_UpperCAmelCase    :      str				)    ->  Any:
      return float((preds == labels).mean()				)





def   a_   (		_UpperCAmelCase    :      Optional[int] ,_UpperCAmelCase    :      Any ,_UpperCAmelCase    :      str="binary"				)    ->  Dict:
      __snake_case :   Union[str, Any]													=    simple_accuracy(_UpperCAmelCase ,_UpperCAmelCase				)
      __snake_case :   List[Any]													=    float(fa_score(y_true=_UpperCAmelCase ,y_pred=_UpperCAmelCase ,average=_UpperCAmelCase				)				)
      return {
          "accuracy": acc,
          "f1": fa,
      }





def   a_   (		_UpperCAmelCase    :      int ,_UpperCAmelCase    :      int				)    ->  Tuple:
      __snake_case :   Any													=    {}
      for id_pred, label in zip(_UpperCAmelCase ,_UpperCAmelCase				):
            __snake_case :   str													=    f'''{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'''
            __snake_case :   Union[str, Any]													=    id_pred['prediction']
            if question_id in question_map:
                  question_map[question_id].append((pred, label)				)
            else:
                  __snake_case :   List[str]													=    [(pred, label)]
      __snake_case							,     __snake_case :   List[Any]													=    [], []
      for question, preds_labels in question_map.items():
            __snake_case							,     __snake_case :   Any													=    zip(*_UpperCAmelCase				)
            __snake_case :   int													=    fa_score(y_true=_UpperCAmelCase ,y_pred=_UpperCAmelCase ,average='macro'				)
            fas.append(_UpperCAmelCase				)
            __snake_case :   List[str]													=    int(sum(pred == label for pred, label in preds_labels				) == len(_UpperCAmelCase				)				)
            ems.append(_UpperCAmelCase				)
      __snake_case :   int													=    float(sum(_UpperCAmelCase				) / len(_UpperCAmelCase				)				)
      __snake_case :   Optional[int]													=    sum(_UpperCAmelCase				) / len(_UpperCAmelCase				)
      __snake_case :   int													=    float(fa_score(y_true=_UpperCAmelCase ,y_pred=[id_pred['prediction'] for id_pred in ids_preds]				)				)
      return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,     _KWARGS_DESCRIPTION				)
class  snake_case__						(    datasets.Metric				):
 def A_   (   self						:							Any  )      ->   List[str]:







       '''simple docstring'''
       if self.config_name not in [
           "boolq",
           "cb",
           "copa",
           "multirc",
           "record",
           "rte",
           "wic",
           "wsc",
           "wsc.fixed",
           "axb",
           "axg",
       ]:
             raise KeyError(
                 'You should supply a configuration name selected in '
                 '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'  )
       return datasets.MetricInfo(
           description=_DESCRIPTION      ,     citation=_CITATION      ,     inputs_description=_KWARGS_DESCRIPTION      ,     features=datasets.Features(self._get_feature_types()  )      ,     codebase_urls=[]      ,     reference_urls=[]      ,     format='numpy' if not self.config_name == 'record' and not self.config_name == 'multirc' else None      ,     )
 def A_   (   self						:							List[Any]  )      ->   Any:







       '''simple docstring'''
       if self.config_name == "record":
             return {
                 "predictions": {
                     "idx": {
                         "passage": datasets.Value('int64'  ),
                         "query": datasets.Value('int64'  ),
                     },
                     "prediction_text": datasets.Value('string'  ),
                 },
                 "references": {
                     "idx": {
                         "passage": datasets.Value('int64'  ),
                         "query": datasets.Value('int64'  ),
                     },
                     "answers": datasets.Sequence(datasets.Value('string'  )  ),
                 },
             }
       elif self.config_name == "multirc":
             return {
                 "predictions": {
                     "idx": {
                         "answer": datasets.Value('int64'  ),
                         "paragraph": datasets.Value('int64'  ),
                         "question": datasets.Value('int64'  ),
                     },
                     "prediction": datasets.Value('int64'  ),
                 },
                 "references": datasets.Value('int64'  ),
             }
       else:
             return {
                 "predictions": datasets.Value('int64'  ),
                 "references": datasets.Value('int64'  ),
             }





 def A_   (   self						:							str      ,     __a						:							Union[str, Any]      ,     __a						:							Dict  )      ->   Dict:







       '''simple docstring'''
       if self.config_name == "axb":
             return {"matthews_correlation": matthews_corrcoef(__a      ,     __a  )}
       elif self.config_name == "cb":
             return acc_and_fa(__a      ,     __a      ,     fa_avg='macro'  )
       elif self.config_name == "record":
             __snake_case :   Any													=    [
                 {
                     'qas': [
                         {'id': ref['idx']['query'], 'answers': [{'text': ans} for ans in ref['answers']]}
                         for ref in references
                     ]
                 }
             ]
             __snake_case :   Dict													=    {pred['idx']['query']: pred['prediction_text'] for pred in predictions}
             return evaluate_record(__a      ,     __a  )[0]
       elif self.config_name == "multirc":
             return evaluate_multirc(__a      ,     __a  )
       elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
             return {"accuracy": simple_accuracy(__a      ,     __a  )}
       else:
             raise KeyError(
                 'You should supply a configuration name selected in '
                 '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'  )








'''simple docstring'''





from ..utils import DummyObject, requires_backends



class  snake_case__						(    metaclass=SCREAMING_SNAKE_CASE_				):
 A__				= ['''note_seq''']
 def __init__(   self						:							Tuple      ,     *__a						:							int      ,     **__a						:							List[str]  )      ->   Any:







       '''simple docstring'''
       requires_backends(self      ,     ['note_seq']  )
 @classmethod
 def A_   (   cls						:							Optional[Any]      ,     *__a						:							Optional[Any]      ,     **__a						:							List[str]  )      ->   Union[str, Any]:







       '''simple docstring'''
       requires_backends(cls      ,     ['note_seq']  )





 @classmethod
 def A_   (   cls						:							List[Any]      ,     *__a						:							Union[str, Any]      ,     **__a						:							List[str]  )      ->   Any:







       '''simple docstring'''
       requires_backends(cls      ,     ['note_seq']  )



