






"""simple docstring"""



from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


__A						:			Optional[int]    							=						'''\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
'''

__A						:			List[Any]    							=						'''\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
'''

__A						:			Optional[int]    							=						'''
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for \'record\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'prediction_text\': the predicted answer text
        - for \'multirc\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question-answer pair as specified by the dataset
            - \'prediction\': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for \'record\': list of question-answers dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'answers\': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for \'record\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1\': F1 score
    - for \'multirc\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1_m\': Per-question macro-F1 score
        - \'f1_a\': Average F1 score over all answers
    - for \'axb\':
        \'matthews_correlation\': Matthew Correlation
    - for \'cb\':
        - \'accuracy\': Accuracy
        - \'f1\': F1 score
    - for all others:
        - \'accuracy\': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')
    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]
    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')
    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''



def 		lowercase       (							__snake_case							:					Any						,   __snake_case							:					List[str]      ):
		return float((preds == labels).mean()      )



def 		lowercase       (							__snake_case							:					Dict						,   __snake_case							:					int						,   __snake_case							:					Any="binary"      ):
		lowercase_					:					Optional[int]							=  simple_accuracy(UpperCAmelCase_						,   UpperCAmelCase_      )
		lowercase_					:					Tuple							=  float(fa_score(y_true=UpperCAmelCase_						,   y_pred=UpperCAmelCase_						,   average=UpperCAmelCase_      )      )
		return {
		    "accuracy": acc,
		    "f1": fa,
		}



def 		lowercase       (							__snake_case							:					List[str]						,   __snake_case							:					Union[str, Any]      ):
		lowercase_					:					List[Any]							=  {}
		for id_pred, label in zip(UpperCAmelCase_						,   UpperCAmelCase_      ):
				lowercase_					:					Dict							=  F'''{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}'''
				lowercase_					:					List[str]							=  id_pred['''prediction''']
				if question_id in question_map:
						question_map[question_id].append((pred, label)      )
				else:
						lowercase_					:					Dict							=  [(pred, label)]
		lowercase_			,   lowercase_					:					Any							=  [], []
		for question, preds_labels in question_map.items():
				lowercase_			,   lowercase_					:					List[Any]							=  zip(*UpperCAmelCase_      )
				lowercase_					:					Optional[int]							=  fa_score(y_true=UpperCAmelCase_						,   y_pred=UpperCAmelCase_						,   average='''macro'''      )
				fas.append(UpperCAmelCase_      )
				lowercase_					:					Dict							=  int(sum(pred == label for pred, label in preds_labels      ) == len(UpperCAmelCase_      )      )
				ems.append(UpperCAmelCase_      )
		lowercase_					:					Dict							=  float(sum(UpperCAmelCase_      ) / len(UpperCAmelCase_      )      )
		lowercase_					:					Any							=  sum(UpperCAmelCase_      ) / len(UpperCAmelCase_      )
		lowercase_					:					List[str]							=  float(fa_score(y_true=UpperCAmelCase_						,   y_pred=[id_pred['''prediction'''] for id_pred in ids_preds]      )      )
		return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,		_KWARGS_DESCRIPTION				)
class    _UpperCAmelCase			(					datasets.Metric				):




					def     A    (							self	:		Optional[int]  )    ->    Optional[Any]:
							if self.config_name not in [
							    "boolq",
							    "cb",
							    "copa",
							    "multirc",
							    "record",
							    "rte",
							    "wic",
							    "wsc",
							    "wsc.fixed",
							    "axb",
							    "axg",
							]:
									raise KeyError(
									    '''You should supply a configuration name selected in '''
									    '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''  )
							return datasets.MetricInfo(
							    description=_DESCRIPTION       ,		citation=_CITATION       ,		inputs_description=_KWARGS_DESCRIPTION       ,		features=datasets.Features(self._get_feature_types()  )       ,		codebase_urls=[]       ,		reference_urls=[]       ,		format='''numpy''' if not self.config_name == '''record''' and not self.config_name == '''multirc''' else None       ,		)




					def     A    (							self	:		Union[str, Any]  )    ->    str:
							if self.config_name == "record":
									return {
									    "predictions": {
									        "idx": {
									            "passage": datasets.Value('''int64'''  ),
									            "query": datasets.Value('''int64'''  ),
									        },
									        "prediction_text": datasets.Value('''string'''  ),
									    },
									    "references": {
									        "idx": {
									            "passage": datasets.Value('''int64'''  ),
									            "query": datasets.Value('''int64'''  ),
									        },
									        "answers": datasets.Sequence(datasets.Value('''string'''  )  ),
									    },
									}
							elif self.config_name == "multirc":
									return {
									    "predictions": {
									        "idx": {
									            "answer": datasets.Value('''int64'''  ),
									            "paragraph": datasets.Value('''int64'''  ),
									            "question": datasets.Value('''int64'''  ),
									        },
									        "prediction": datasets.Value('''int64'''  ),
									    },
									    "references": datasets.Value('''int64'''  ),
									}
							else:
									return {
									    "predictions": datasets.Value('''int64'''  ),
									    "references": datasets.Value('''int64'''  ),
									}






					def     A    (							self	:		Tuple       ,		A	:		List[str]       ,		A	:		Tuple  )    ->    Optional[int]:
							if self.config_name == "axb":
									return {"matthews_correlation": matthews_corrcoef(UpperCamelCase_       ,		UpperCamelCase_  )}
							elif self.config_name == "cb":
									return acc_and_fa(UpperCamelCase_       ,		UpperCamelCase_       ,		fa_avg='''macro'''  )
							elif self.config_name == "record":
									lowercase_					:					str							=  [
									    {
									        '''qas''': [
									            {'''id''': ref['''idx''']['''query'''], '''answers''': [{'''text''': ans} for ans in ref['''answers''']]}
									            for ref in references
									        ]
									    }
									]
									lowercase_					:					str							=  {pred['''idx''']['''query''']: pred['''prediction_text'''] for pred in predictions}
									return evaluate_record(UpperCamelCase_       ,		UpperCamelCase_  )[0]
							elif self.config_name == "multirc":
									return evaluate_multirc(UpperCamelCase_       ,		UpperCamelCase_  )
							elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
									return {"accuracy": simple_accuracy(UpperCamelCase_       ,		UpperCamelCase_  )}
							else:
									raise KeyError(
									    '''You should supply a configuration name selected in '''
									    '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''  )










'''simple docstring'''




import warnings

from ...utils import logging
from .image_processing_perceiver import PerceiverImageProcessor


a__					:			int          =     logging.get_logger(__name__)






class 				__snake_case     (					__magic_name__							):





	def __init__(		self      ,       *UpperCamelCase_      ,       **UpperCamelCase_		)      ->		None:
		warnings.warn(
		    'The class PerceiverFeatureExtractor is deprecated and will be removed in version 5 of Transformers.'
		    ' Please use PerceiverImageProcessor instead.'      ,       UpperCamelCase_      ,       )
		super().__init__(*UpperCamelCase_      ,       **UpperCamelCase_		)



