





from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


UpperCAmelCase__  :     Tuple   			='''\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
'''

UpperCAmelCase__  :     List[Any]   			='''\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
'''

UpperCAmelCase__  :     Optional[int]   			='''
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for \'record\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'prediction_text\': the predicted answer text
        - for \'multirc\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question-answer pair as specified by the dataset
            - \'prediction\': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for \'record\': list of question-answers dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'answers\': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for \'record\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1\': F1 score
    - for \'multirc\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1_m\': Per-question macro-F1 score
        - \'f1_a\': Average F1 score over all answers
    - for \'axb\':
        \'matthews_correlation\': Matthew Correlation
    - for \'cb\':
        - \'accuracy\': Accuracy
        - \'f1\': F1 score
    - for all others:
        - \'accuracy\': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')
    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]
    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')
    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''





def        _lowercase   (  _UpperCAmelCase     , _UpperCAmelCase			)  ->					Optional[int]:
				return float((preds == labels).mean()			)





def        _lowercase   (  _UpperCAmelCase     , _UpperCAmelCase     , _UpperCAmelCase="binary"			)  ->					Tuple:
				lowerCamelCase			=simple_accuracy(_UpperCAmelCase     , _UpperCAmelCase			)
				lowerCamelCase			=float(fa_score(y_true=_UpperCAmelCase     , y_pred=_UpperCAmelCase     , average=_UpperCAmelCase			)			)
				return {
				    "accuracy": acc,
				    "f1": fa,
				}





def        _lowercase   (  _UpperCAmelCase     , _UpperCAmelCase			)  ->					Dict:
				lowerCamelCase			={}
				for id_pred, label in zip(_UpperCAmelCase     , _UpperCAmelCase			):
								lowerCamelCase			=F"""{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}"""
								lowerCamelCase			=id_pred["""prediction"""]
								if question_id in question_map:
												question_map[question_id].append((pred, label)			)
								else:
												lowerCamelCase			=[(pred, label)]
				lowerCamelCase	,					lowerCamelCase			=[], []
				for question, preds_labels in question_map.items():
								lowerCamelCase	,					lowerCamelCase			=zip(*_UpperCAmelCase			)
								lowerCamelCase			=fa_score(y_true=_UpperCAmelCase     , y_pred=_UpperCAmelCase     , average="""macro"""			)
								fas.append(_UpperCAmelCase			)
								lowerCamelCase			=int(sum(pred == label for pred, label in preds_labels			) == len(_UpperCAmelCase			)			)
								ems.append(_UpperCAmelCase			)
				lowerCamelCase			=float(sum(_UpperCAmelCase			) / len(_UpperCAmelCase			)			)
				lowerCamelCase			=sum(_UpperCAmelCase			) / len(_UpperCAmelCase			)
				lowerCamelCase			=float(fa_score(y_true=_UpperCAmelCase     , y_pred=[id_pred["""prediction"""] for id_pred in ids_preds]			)			)
				return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION      ,   _KWARGS_DESCRIPTION      )
class  __A       (  datasets.Metric      ):


				def _snake_case    (						self							):
								if self.config_name not in [
								    "boolq",
								    "cb",
								    "copa",
								    "multirc",
								    "record",
								    "rte",
								    "wic",
								    "wsc",
								    "wsc.fixed",
								    "axb",
								    "axg",
								]:
												raise KeyError(
												    """You should supply a configuration name selected in """
												    """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""							)
								return datasets.MetricInfo(
								    description=_DESCRIPTION				,	citation=_CITATION				,	inputs_description=_KWARGS_DESCRIPTION				,	features=datasets.Features(self._get_feature_types()							)				,	codebase_urls=[]				,	reference_urls=[]				,	format="""numpy""" if not self.config_name == """record""" and not self.config_name == """multirc""" else None				,	)


				def _snake_case    (						self							):
								if self.config_name == "record":
												return {
												    "predictions": {
												        "idx": {
												            "passage": datasets.Value("""int64"""							),
												            "query": datasets.Value("""int64"""							),
												        },
												        "prediction_text": datasets.Value("""string"""							),
												    },
												    "references": {
												        "idx": {
												            "passage": datasets.Value("""int64"""							),
												            "query": datasets.Value("""int64"""							),
												        },
												        "answers": datasets.Sequence(datasets.Value("""string"""							)							),
												    },
												}
								elif self.config_name == "multirc":
												return {
												    "predictions": {
												        "idx": {
												            "answer": datasets.Value("""int64"""							),
												            "paragraph": datasets.Value("""int64"""							),
												            "question": datasets.Value("""int64"""							),
												        },
												        "prediction": datasets.Value("""int64"""							),
												    },
												    "references": datasets.Value("""int64"""							),
												}
								else:
												return {
												    "predictions": datasets.Value("""int64"""							),
												    "references": datasets.Value("""int64"""							),
												}


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_							):
								if self.config_name == "axb":
												return {"matthews_correlation": matthews_corrcoef(UpperCAmelCase_				,	UpperCAmelCase_							)}
								elif self.config_name == "cb":
												return acc_and_fa(UpperCAmelCase_				,	UpperCAmelCase_				,	fa_avg="""macro"""							)
								elif self.config_name == "record":
												lowerCamelCase			=[
												    {
												        """qas""": [
												            {"""id""": ref["""idx"""]["""query"""], """answers""": [{"""text""": ans} for ans in ref["""answers"""]]}
												            for ref in references
												        ]
												    }
												]
												lowerCamelCase			={pred["""idx"""]["""query"""]: pred["""prediction_text"""] for pred in predictions}
												return evaluate_record(UpperCAmelCase_				,	UpperCAmelCase_							)[0]
								elif self.config_name == "multirc":
												return evaluate_multirc(UpperCAmelCase_				,	UpperCAmelCase_							)
								elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
												return {"accuracy": simple_accuracy(UpperCAmelCase_				,	UpperCAmelCase_							)}
								else:
												raise KeyError(
												    """You should supply a configuration name selected in """
												    """[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"""							)












import json
import os
import unittest

from transformers.models.biogpt.tokenization_biogpt import VOCAB_FILES_NAMES, BioGptTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin

class  __A       (  a      ,   unittest.TestCase      ):
				__A								=			BioGptTokenizer
				__A								=			False


				def _snake_case    (						self							):
								super().setUp()

								# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
								lowerCamelCase			=[
								    """l""",
								    """o""",
								    """w""",
								    """e""",
								    """r""",
								    """s""",
								    """t""",
								    """i""",
								    """d""",
								    """n""",
								    """w</w>""",
								    """r</w>""",
								    """t</w>""",
								    """lo""",
								    """low""",
								    """er</w>""",
								    """low</w>""",
								    """lowest</w>""",
								    """newer</w>""",
								    """wider</w>""",
								    """<unk>""",
								]
								lowerCamelCase			=dict(zip(UpperCAmelCase_				,	range(len(UpperCAmelCase_							)							)							)							)
								lowerCamelCase			=["""l o 123""", """lo w 1456""", """e r</w> 1789""", """"""]

								lowerCamelCase			=os.path.join(self.tmpdirname				,	VOCAB_FILES_NAMES["""vocab_file"""]							)
								lowerCamelCase			=os.path.join(self.tmpdirname				,	VOCAB_FILES_NAMES["""merges_file"""]							)
								with open(self.vocab_file				,	"""w"""							) as fp:
												fp.write(json.dumps(UpperCAmelCase_							)							)
								with open(self.merges_file				,	"""w"""							) as fp:
												fp.write("""\n""".join(UpperCAmelCase_							)							)


				def _snake_case    (						self				,	UpperCAmelCase_							):
								lowerCamelCase			="""lower newer"""
								lowerCamelCase			="""lower newer"""
								return input_text, output_text


				def _snake_case    (						self							):
								lowerCamelCase			=BioGptTokenizer(self.vocab_file				,	self.merges_file							)

								lowerCamelCase			="""lower"""
								lowerCamelCase			=["""low""", """er</w>"""]
								lowerCamelCase			=tokenizer.tokenize(UpperCAmelCase_							)
								self.assertListEqual(UpperCAmelCase_				,	UpperCAmelCase_							)

								lowerCamelCase			=tokens + ["""<unk>"""]
								lowerCamelCase			=[14, 15, 20]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_							)				,	UpperCAmelCase_							)


				@slow
				def _snake_case    (						self							):
								lowerCamelCase			=BioGptTokenizer.from_pretrained("""microsoft/biogpt"""							)

								lowerCamelCase			=tokenizer.encode("""sequence builders"""				,	add_special_tokens=UpperCAmelCase_							)
								lowerCamelCase			=tokenizer.encode("""multi-sequence build"""				,	add_special_tokens=UpperCAmelCase_							)

								lowerCamelCase			=tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_							)
								lowerCamelCase			=tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_				,	UpperCAmelCase_							)

								self.assertTrue(encoded_sentence == [2] + text							)
								self.assertTrue(encoded_pair == [2] + text + [2] + text_a							)





