



from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


UpperCAmelCase						:		Optional[Any]			= '\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n'

UpperCAmelCase						:		Tuple			= '\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n'

UpperCAmelCase						:		str			= '\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for \'record\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'prediction_text\': the predicted answer text\n        - for \'multirc\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question-answer pair as specified by the dataset\n            - \'prediction\': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for \'record\': list of question-answers dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'answers\': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for \'record\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1\': F1 score\n    - for \'multirc\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1_m\': Per-question macro-F1 score\n        - \'f1_a\': Average F1 score over all answers\n    - for \'axb\':\n        \'matthews_correlation\': Matthew Correlation\n    - for \'cb\':\n        - \'accuracy\': Accuracy\n        - \'f1\': F1 score\n    - for all others:\n        - \'accuracy\': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')\n    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]\n    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')\n    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'matthews_correlation\': 1.0}\n'


def 			_SCREAMING_SNAKE_CASE						(__lowerCAmelCase ,  __lowerCAmelCase							) ->						Dict:







  '''simple docstring'''


  return float((preds == labels).mean()							)


def 			_SCREAMING_SNAKE_CASE						(__lowerCAmelCase ,  __lowerCAmelCase ,  __lowerCAmelCase="binary"							) ->						List[Any]:







  '''simple docstring'''


  lowercase_								= simple_accuracy(_A ,  _A							)
  lowercase_								= float(fa_score(y_true=_A ,  y_pred=_A ,  average=_A							)							)
  return {
      "accuracy": acc,
      "f1": fa,
  }


def 			_SCREAMING_SNAKE_CASE						(__lowerCAmelCase ,  __lowerCAmelCase							) ->						int:







  '''simple docstring'''


  lowercase_								= {}
  for id_pred, label in zip(_A ,  _A							):
    lowercase_								= F'''{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'''
    lowercase_								= id_pred["""prediction"""]
    if question_id in question_map:
      question_map[question_id].append((pred, label)							)
    else:
      lowercase_								= [(pred, label)]
  lowercase_					,    lowercase_								= [], []
  for question, preds_labels in question_map.items():
    lowercase_					,    lowercase_								= zip(*_A							)
    lowercase_								= fa_score(y_true=_A ,  y_pred=_A ,  average="""macro"""							)
    fas.append(_A							)
    lowercase_								= int(sum(pred == label for pred, label in preds_labels							) == len(_A							)							)
    ems.append(_A							)
  lowercase_								= float(sum(_A							) / len(_A							)							)
  lowercase_								= sum(_A							) / len(_A							)
  lowercase_								= float(fa_score(y_true=_A ,  y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]							)							)
  return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION      ,  _KWARGS_DESCRIPTION   )
class    SCREAMING_SNAKE_CASE__ (							datasets.Metric   ):

    def 					_UpperCAmelCase	(				self			:				List[str]):







      """simple docstring"""







      if self.config_name not in [
          "boolq",
          "cb",
          "copa",
          "multirc",
          "record",
          "rte",
          "wic",
          "wsc",
          "wsc.fixed",
          "axb",
          "axg",
      ]:
        raise KeyError(
            """You should supply a configuration name selected in """
            """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]""")
      return datasets.MetricInfo(
          description=_DESCRIPTION ,  citation=_CITATION ,  inputs_description=_KWARGS_DESCRIPTION ,  features=datasets.Features(self._get_feature_types()) ,  codebase_urls=[] ,  reference_urls=[] ,  format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None ,  )

    def 					_UpperCAmelCase	(				self			:				Any):







      """simple docstring"""







      if self.config_name == "record":
        return {
            "predictions": {
                "idx": {
                    "passage": datasets.Value("""int64"""),
                    "query": datasets.Value("""int64"""),
                },
                "prediction_text": datasets.Value("""string"""),
            },
            "references": {
                "idx": {
                    "passage": datasets.Value("""int64"""),
                    "query": datasets.Value("""int64"""),
                },
                "answers": datasets.Sequence(datasets.Value("""string""")),
            },
        }
      elif self.config_name == "multirc":
        return {
            "predictions": {
                "idx": {
                    "answer": datasets.Value("""int64"""),
                    "paragraph": datasets.Value("""int64"""),
                    "question": datasets.Value("""int64"""),
                },
                "prediction": datasets.Value("""int64"""),
            },
            "references": datasets.Value("""int64"""),
        }
      else:
        return {
            "predictions": datasets.Value("""int64"""),
            "references": datasets.Value("""int64"""),
        }




    def 					_UpperCAmelCase	(				self			:				Optional[Any] ,  lowerCAmelCase_			:				Any ,  lowerCAmelCase_			:				str):







      """simple docstring"""







      if self.config_name == "axb":
        return {"matthews_correlation": matthews_corrcoef(_SCREAMING_SNAKE_CASE ,  _SCREAMING_SNAKE_CASE)}
      elif self.config_name == "cb":
        return acc_and_fa(_SCREAMING_SNAKE_CASE ,  _SCREAMING_SNAKE_CASE ,  fa_avg="""macro""")
      elif self.config_name == "record":
        lowercase_								= [
            {
                """qas""": [
                    {"""id""": ref["""idx"""]["""query"""], """answers""": [{"""text""": ans} for ans in ref["""answers"""]]}
                    for ref in references
                ]
            }
        ]
        lowercase_								= {pred["""idx"""]["""query"""]: pred["""prediction_text"""] for pred in predictions}
        return evaluate_record(_SCREAMING_SNAKE_CASE ,  _SCREAMING_SNAKE_CASE)[0]
      elif self.config_name == "multirc":
        return evaluate_multirc(_SCREAMING_SNAKE_CASE ,  _SCREAMING_SNAKE_CASE)
      elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
        return {"accuracy": simple_accuracy(_SCREAMING_SNAKE_CASE ,  _SCREAMING_SNAKE_CASE)}
      else:
        raise KeyError(
            """You should supply a configuration name selected in """
            """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]""")






"""simple docstring"""
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCAmelCase						:		Union[str, Any]			= {
    "configuration_longformer": [
        "LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "LongformerConfig",
        "LongformerOnnxConfig",
    ],
    "tokenization_longformer": ["LongformerTokenizer"],
}

try:
       if not is_tokenizers_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCAmelCase						:		List[Any]			= ["LongformerTokenizerFast"]

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCAmelCase						:		Tuple			= [
           "LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
           "LongformerForMaskedLM",
           "LongformerForMultipleChoice",
           "LongformerForQuestionAnswering",
           "LongformerForSequenceClassification",
           "LongformerForTokenClassification",
           "LongformerModel",
           "LongformerPreTrainedModel",
           "LongformerSelfAttention",
       ]

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       UpperCAmelCase						:		List[Any]			= [
           "TF_LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
           "TFLongformerForMaskedLM",
           "TFLongformerForMultipleChoice",
           "TFLongformerForQuestionAnswering",
           "TFLongformerForSequenceClassification",
           "TFLongformerForTokenClassification",
           "TFLongformerModel",
           "TFLongformerPreTrainedModel",
           "TFLongformerSelfAttention",
       ]


if TYPE_CHECKING:
       from .configuration_longformer import (
           LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
           LongformerConfig,
           LongformerOnnxConfig,
       )
       from .tokenization_longformer import LongformerTokenizer

       try:
              if not is_tokenizers_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_longformer_fast import LongformerTokenizerFast

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_longformer import (
                  LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
                  LongformerForMaskedLM,
                  LongformerForMultipleChoice,
                  LongformerForQuestionAnswering,
                  LongformerForSequenceClassification,
                  LongformerForTokenClassification,
                  LongformerModel,
                  LongformerPreTrainedModel,
                  LongformerSelfAttention,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_longformer import (
                  TF_LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
                  TFLongformerForMaskedLM,
                  TFLongformerForMultipleChoice,
                  TFLongformerForQuestionAnswering,
                  TFLongformerForSequenceClassification,
                  TFLongformerForTokenClassification,
                  TFLongformerModel,
                  TFLongformerPreTrainedModel,
                  TFLongformerSelfAttention,
              )

else:
       import sys

       UpperCAmelCase						:		Optional[Any]			= _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

