





'''simple docstring'''

from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


lowercase__		:     Optional[Any]			=							'\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n'

lowercase__		:     Optional[int]			=							'\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n'

lowercase__		:     Dict			=							'\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for \'record\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'prediction_text\': the predicted answer text\n        - for \'multirc\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question-answer pair as specified by the dataset\n            - \'prediction\': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for \'record\': list of question-answers dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'answers\': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for \'record\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1\': F1 score\n    - for \'multirc\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1_m\': Per-question macro-F1 score\n        - \'f1_a\': Average F1 score over all answers\n    - for \'axb\':\n        \'matthews_correlation\': Matthew Correlation\n    - for \'cb\':\n        - \'accuracy\': Accuracy\n        - \'f1\': F1 score\n    - for all others:\n        - \'accuracy\': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')\n    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]\n    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')\n    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'matthews_correlation\': 1.0}\n'







def    a__							(							lowercase   :	int,   lowercase   :	Any )						->						Tuple:





     """simple docstring"""

     return float((preds == labels).mean() )







def    a__							(							lowercase   :	List[str],   lowercase   :	int,   lowercase   :	str="binary" )						->						List[Any]:





     """simple docstring"""

     _UpperCamelCase							       =			simple_accuracy(lowercase,   lowercase )
     _UpperCamelCase							       =			float(fa_score(y_true=lowercase,   y_pred=lowercase,   average=lowercase ) )
     return {
         "accuracy": acc,
         "f1": fa,
     }







def    a__							(							lowercase   :	str,   lowercase   :	List[str] )						->						List[str]:





     """simple docstring"""

     _UpperCamelCase							       =			{}
     for id_pred, label in zip(lowercase,   lowercase ):
          _UpperCamelCase							       =			F"""{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"""
          _UpperCamelCase							       =			id_pred['''prediction''']
          if question_id in question_map:
               question_map[question_id].append((pred, label) )
          else:
               _UpperCamelCase							       =			[(pred, label)]
     _UpperCamelCase       ,							_UpperCamelCase							       =			[], []
     for question, preds_labels in question_map.items():
          _UpperCamelCase       ,							_UpperCamelCase							       =			zip(*lowercase )
          _UpperCamelCase							       =			fa_score(y_true=lowercase,   y_pred=lowercase,   average='''macro''' )
          fas.append(lowercase )
          _UpperCamelCase							       =			int(sum(pred == label for pred, label in preds_labels ) == len(lowercase ) )
          ems.append(lowercase )
     _UpperCamelCase							       =			float(sum(lowercase ) / len(lowercase ) )
     _UpperCamelCase							       =			sum(lowercase ) / len(lowercase )
     _UpperCamelCase							       =			float(fa_score(y_true=lowercase,   y_pred=[id_pred['''prediction'''] for id_pred in ids_preds] ) )
     return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION		,		_KWARGS_DESCRIPTION					)
class 							__lowerCAmelCase	(       datasets.Metric					):






       """simple docstring"""





       def snake_case__		(				self  :						Union[str, Any]	)       ->	Tuple:

            '''simple docstring'''




            if self.config_name not in [
                "boolq",
                "cb",
                "copa",
                "multirc",
                "record",
                "rte",
                "wic",
                "wsc",
                "wsc.fixed",
                "axb",
                "axg",
            ]:
                 raise KeyError(
                     '''You should supply a configuration name selected in '''
                     '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''	)
            return datasets.MetricInfo(
                description=_DESCRIPTION			,					citation=_CITATION			,					inputs_description=_KWARGS_DESCRIPTION			,					features=datasets.Features(self._get_feature_types()	)			,					codebase_urls=[]			,					reference_urls=[]			,					format='''numpy''' if not self.config_name == '''record''' and not self.config_name == '''multirc''' else None			,					)





       def snake_case__		(				self  :						Tuple	)       ->	Any:

            '''simple docstring'''




            if self.config_name == "record":
                 return {
                     "predictions": {
                         "idx": {
                             "passage": datasets.Value('''int64'''	),
                             "query": datasets.Value('''int64'''	),
                         },
                         "prediction_text": datasets.Value('''string'''	),
                     },
                     "references": {
                         "idx": {
                             "passage": datasets.Value('''int64'''	),
                             "query": datasets.Value('''int64'''	),
                         },
                         "answers": datasets.Sequence(datasets.Value('''string'''	)	),
                     },
                 }
            elif self.config_name == "multirc":
                 return {
                     "predictions": {
                         "idx": {
                             "answer": datasets.Value('''int64'''	),
                             "paragraph": datasets.Value('''int64'''	),
                             "question": datasets.Value('''int64'''	),
                         },
                         "prediction": datasets.Value('''int64'''	),
                     },
                     "references": datasets.Value('''int64'''	),
                 }
            else:
                 return {
                     "predictions": datasets.Value('''int64'''	),
                     "references": datasets.Value('''int64'''	),
                 }







       def snake_case__		(				self  :						List[Any]			,					lowerCAmelCase__  :						Optional[int]			,					lowerCAmelCase__  :						int	)       ->	Any:

            '''simple docstring'''




            if self.config_name == "axb":
                 return {"matthews_correlation": matthews_corrcoef(lowerCAmelCase__			,					lowerCAmelCase__	)}
            elif self.config_name == "cb":
                 return acc_and_fa(lowerCAmelCase__			,					lowerCAmelCase__			,					fa_avg='''macro'''	)
            elif self.config_name == "record":
                 _UpperCamelCase							       =			[
                     {
                         '''qas''': [
                             {'''id''': ref['''idx''']['''query'''], '''answers''': [{'''text''': ans} for ans in ref['''answers''']]}
                             for ref in references
                         ]
                     }
                 ]
                 _UpperCamelCase							       =			{pred['''idx''']['''query''']: pred['''prediction_text'''] for pred in predictions}
                 return evaluate_record(lowerCAmelCase__			,					lowerCAmelCase__	)[0]
            elif self.config_name == "multirc":
                 return evaluate_multirc(lowerCAmelCase__			,					lowerCAmelCase__	)
            elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
                 return {"accuracy": simple_accuracy(lowerCAmelCase__			,					lowerCAmelCase__	)}
            else:
                 raise KeyError(
                     '''You should supply a configuration name selected in '''
                     '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'''	)








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowercase_													=      {
    '''configuration_albert''': ['''ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''AlbertConfig''', '''AlbertOnnxConfig'''],
}

try:
				if not is_sentencepiece_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_													=      ['''AlbertTokenizer''']

try:
				if not is_tokenizers_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_													=      ['''AlbertTokenizerFast''']

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_													=      [
				    '''ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
				    '''AlbertForMaskedLM''',
				    '''AlbertForMultipleChoice''',
				    '''AlbertForPreTraining''',
				    '''AlbertForQuestionAnswering''',
				    '''AlbertForSequenceClassification''',
				    '''AlbertForTokenClassification''',
				    '''AlbertModel''',
				    '''AlbertPreTrainedModel''',
				    '''load_tf_weights_in_albert''',
				]

try:
				if not is_tf_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_													=      [
				    '''TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
				    '''TFAlbertForMaskedLM''',
				    '''TFAlbertForMultipleChoice''',
				    '''TFAlbertForPreTraining''',
				    '''TFAlbertForQuestionAnswering''',
				    '''TFAlbertForSequenceClassification''',
				    '''TFAlbertForTokenClassification''',
				    '''TFAlbertMainLayer''',
				    '''TFAlbertModel''',
				    '''TFAlbertPreTrainedModel''',
				]

try:
				if not is_flax_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_													=      [
				    '''FlaxAlbertForMaskedLM''',
				    '''FlaxAlbertForMultipleChoice''',
				    '''FlaxAlbertForPreTraining''',
				    '''FlaxAlbertForQuestionAnswering''',
				    '''FlaxAlbertForSequenceClassification''',
				    '''FlaxAlbertForTokenClassification''',
				    '''FlaxAlbertModel''',
				    '''FlaxAlbertPreTrainedModel''',
				]

if TYPE_CHECKING:
				from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig, AlbertOnnxConfig

				try:
								if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_albert import AlbertTokenizer

				try:
								if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_albert_fast import AlbertTokenizerFast

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_albert import (
								    ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    AlbertForMaskedLM,
								    AlbertForMultipleChoice,
								    AlbertForPreTraining,
								    AlbertForQuestionAnswering,
								    AlbertForSequenceClassification,
								    AlbertForTokenClassification,
								    AlbertModel,
								    AlbertPreTrainedModel,
								    load_tf_weights_in_albert,
								)

				try:
								if not is_tf_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_tf_albert import (
								    TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    TFAlbertForMaskedLM,
								    TFAlbertForMultipleChoice,
								    TFAlbertForPreTraining,
								    TFAlbertForQuestionAnswering,
								    TFAlbertForSequenceClassification,
								    TFAlbertForTokenClassification,
								    TFAlbertMainLayer,
								    TFAlbertModel,
								    TFAlbertPreTrainedModel,
								)

				try:
								if not is_flax_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_flax_albert import (
								    FlaxAlbertForMaskedLM,
								    FlaxAlbertForMultipleChoice,
								    FlaxAlbertForPreTraining,
								    FlaxAlbertForQuestionAnswering,
								    FlaxAlbertForSequenceClassification,
								    FlaxAlbertForTokenClassification,
								    FlaxAlbertModel,
								    FlaxAlbertPreTrainedModel,
								)
else:
				import sys

				lowercase_													=      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




