


import math
import os
import re
import sys
import unittest
from pathlib import Path
from typing import Tuple
from unittest.mock import patch

from parameterized import parameterized

from transformers.testing_utils import (
    CaptureStderr,
    ExtendSysPath,
    TestCasePlus,
    execute_subprocess_async,
    get_gpu_count,
    get_torch_dist_unique_port,
    require_apex,
    require_bitsandbytes,
    require_fairscale,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    require_torch_non_multi_gpu,
    slow,
)
from transformers.trainer_callback import TrainerState
from transformers.trainer_utils import set_seed


_snake_case  :	Dict			      =       os.path.abspath(os.path.dirname(__file__))
with ExtendSysPath(f'''{bindir}/../../examples/pytorch/translation'''):
			from run_translation import main  # noqa


set_seed(42)
_snake_case  :	Union[str, Any]			      =       "sshleifer/student_marian_en_ro_6_1"
_snake_case  :	Tuple			      =       "sshleifer/tiny-mbart"

@require_torch
class        a	(_lowerCAmelCase   ):




							"""simple docstring"""


							def 					__snake_case      (							self :						str					,			lowerCamelCase :						Any=False					,			lowerCamelCase :						str=None					,			lowerCamelCase :						Union[str, Any]=True					,			lowerCamelCase :						Union[str, Any]=True					,			lowerCamelCase :						Optional[int]=True					,			lowerCamelCase :						List[Any]=True					,			)			->					str:
														__snake_case	:    Union[str, Any]											=     self.run_trainer(
														    eval_steps=1					,			max_len=12					,			model_name=lowerCamelCase					,			num_train_epochs=1					,			distributed=lowerCamelCase					,			extra_args_str=lowerCamelCase					,			predict_with_generate=lowerCamelCase					,			do_train=lowerCamelCase					,			do_eval=lowerCamelCase					,			do_predict=lowerCamelCase					,			)
														__snake_case	:    Any											=     TrainerState.load_from_json(os.path.join(lowerCamelCase					,			"trainer_state.json"       )       ).log_history

														if not do_eval:
																					return

														__snake_case	:    Dict											=     [log for log in logs if "eval_loss" in log.keys()]

														__snake_case	:    List[str]											=     eval_metrics[0]
														if predict_with_generate:
																					assert "eval_bleu" in first_step_stats

																					__snake_case	:    List[Any]											=     eval_metrics[-1]
																					assert isinstance(last_step_stats["eval_bleu"]					,			lowerCamelCase       )
																					assert not math.isnan(float(last_step_stats["eval_loss"]       )       ), "eval_loss must not be `nan`"

							@require_torch_non_multi_gpu
							def 					__snake_case      (							self :						List[Any]       )			->					int:
														self.run_seqaseq_quick()

							@require_torch_multi_gpu
							def 					__snake_case      (							self :						int       )			->					int:
														self.run_seqaseq_quick(distributed=lowerCamelCase       )

							@require_torch_multi_gpu
							def 					__snake_case      (							self :						Any       )			->					Union[str, Any]:
														self.run_seqaseq_quick(distributed=lowerCamelCase       )

							@unittest.skip("Requires an update of the env running those tests"       )
							@require_torch_multi_gpu
							@require_fairscale
							def 					__snake_case      (							self :						Dict       )			->					Any:
														self.run_seqaseq_quick(distributed=lowerCamelCase					,			extra_args_str="--sharded_ddp simple"       )

							@unittest.skip("Requires an update of the env running those tests"       )
							@require_torch_multi_gpu
							@require_fairscale
							def 					__snake_case      (							self :						int       )			->					List[Any]:
														self.run_seqaseq_quick(distributed=lowerCamelCase					,			extra_args_str="--sharded_ddp simple --fp16"       )

							@unittest.skip("Requires an update of the env running those tests"       )
							@require_torch_multi_gpu
							@require_fairscale
							def 					__snake_case      (							self :						Tuple       )			->					str:
														self.run_seqaseq_quick(distributed=lowerCamelCase					,			extra_args_str="--sharded_ddp zero_dp_2"					,			predict_with_generate=lowerCamelCase       )

							@unittest.skip("Requires an update of the env running those tests"       )
							@require_torch_multi_gpu
							@require_fairscale
							def 					__snake_case      (							self :						Any       )			->					Any:
														self.run_seqaseq_quick(
														    distributed=lowerCamelCase					,			extra_args_str="--sharded_ddp zero_dp_2 --fp16"					,			predict_with_generate=lowerCamelCase       )

							@require_apex
							@require_torch_gpu
							def 					__snake_case      (							self :						Tuple       )			->					int:
														# XXX: apex breaks the trainer if it's run twice e.g. run_seq2seq.main() from the same
														# program and it breaks other tests that run from the same pytest worker, therefore until this is
														# sorted out it must be run only in an external program, that is distributed=True in this
														# test and only under one or more gpus - if we want cpu will need to make a special test
														#
														# specifically to the problem traced it to self.optimizer.step() - if it's run 2nd time via
														# 2nd main() call it botches the future eval.
														#
														self.run_seqaseq_quick(distributed=lowerCamelCase					,			extra_args_str="--fp16 --fp16_backend=apex"       )
														# test 2nd time - was getting eval_loss': nan'
														# to reproduce the problem set distributed=False
														self.run_seqaseq_quick(distributed=lowerCamelCase					,			extra_args_str="--fp16 --fp16_backend=apex"       )

							@parameterized.expand(["base", "low", "high", "mixed"]       )
							@require_torch_multi_gpu
							def 					__snake_case      (							self :						List[Any]					,			lowerCamelCase :						Union[str, Any]       )			->					Any:
														# as each sub-test is slow-ish split into multiple sub-tests to avoid CI timeout
														__snake_case	:    Optional[int]											=     {
														    # test with the default log_level - should be info and thus log info once
														    "base": {"extra_args_str": "", "n_matches": 1},
														    # test with low log_level and log_level_replica - should be noisy on all processes
														    # now the info string should appear twice on 2 processes
														    "low": {"extra_args_str": "--log_level debug --log_level_replica debug", "n_matches": 2},
														    # test with high log_level and low log_level_replica
														    # now the info string should appear once only on the replica
														    "high": {"extra_args_str": "--log_level error --log_level_replica debug", "n_matches": 1},
														    # test with high log_level and log_level_replica - should be quiet on all processes
														    "mixed": {"extra_args_str": "--log_level error --log_level_replica error", "n_matches": 0},
														}

														__snake_case	:    Optional[Any]											=     experiments[experiment_id]
														__snake_case	:    Optional[Any]											=     {"distributed": True, "predict_with_generate": False, "do_eval": False, "do_predict": False}
														__snake_case	:    Tuple											=     "Running training"
														with CaptureStderr() as cl:
																					self.run_seqaseq_quick(**lowerCamelCase					,			extra_args_str=data["extra_args_str"]       )
														__snake_case	:    str											=     len(re.findall(lowerCamelCase					,			cl.err       )       )
														self.assertEqual(lowerCamelCase					,			data["n_matches"]       )

							@slow
							def 					__snake_case      (							self :						Tuple       )			->					Union[str, Any]:
														__snake_case	:    Any											=     self.run_trainer(
														    eval_steps=2					,			max_len=128					,			model_name=lowerCamelCase					,			learning_rate=3E-4					,			num_train_epochs=10					,			distributed=lowerCamelCase					,			)

														# Check metrics
														__snake_case	:    Any											=     TrainerState.load_from_json(os.path.join(lowerCamelCase					,			"trainer_state.json"       )       ).log_history
														__snake_case	:    int											=     [log for log in logs if "eval_loss" in log.keys()]
														__snake_case	:    Tuple											=     eval_metrics[0]
														__snake_case	:    Any											=     eval_metrics[-1]

														assert first_step_stats["eval_loss"] > last_step_stats["eval_loss"], "model learned nothing"
														assert isinstance(last_step_stats["eval_bleu"]					,			lowerCamelCase       )

														# test if do_predict saves generations and metrics
														__snake_case	:    Union[str, Any]											=     os.listdir(lowerCamelCase       )
														__snake_case	:    List[str]											=     {os.path.basename(lowerCamelCase       ) for p in contents}
														assert "generated_predictions.txt" in contents
														assert "predict_results.json" in contents

							@slow
							@require_bitsandbytes
							def 					__snake_case      (							self :						Optional[int]       )			->					Any:
														from transformers.training_args import OptimizerNames

														def train_and_return_metrics(lowerCamelCase :						str       ) -> Tuple[int, float]:
																					__snake_case	:    Any											=     "--skip_memory_metrics 0"

																					__snake_case	:    int											=     self.run_trainer(
																					    max_len=128					,			model_name=lowerCamelCase					,			learning_rate=3E-4					,			num_train_epochs=1					,			optim=lowerCamelCase					,			distributed=lowerCamelCase					,			extra_args_str=lowerCamelCase					,			do_eval=lowerCamelCase					,			do_predict=lowerCamelCase					,			n_gpus_to_use=1					,			)

																					# Check metrics
																					__snake_case	:    List[Any]											=     TrainerState.load_from_json(Path(lowerCamelCase					,			"trainer_state.json"       )       ).log_history
																					__snake_case	:    Any											=     int(logs[0]["train_mem_gpu_peaked_delta"] / 2**20       )
																					__snake_case	:    Dict											=     int(logs[0]["train_mem_gpu_alloc_delta"] / 2**20       )

																					__snake_case	:    Any											=     logs[0]["train_loss"]
																					return gpu_peak_mem_mb, gpu_alloc_mem_mb, loss

														__snake_case				,    __snake_case				,    __snake_case	:    Optional[int]											=     train_and_return_metrics(OptimizerNames.ADAMW_TORCH.value       )
														__snake_case				,    __snake_case				,    __snake_case	:    Union[str, Any]											=     train_and_return_metrics(OptimizerNames.ADAMW_BNB.value       )

														__snake_case	:    Optional[int]											=     gpu_alloc_mem_orig - gpu_alloc_mem_bnb

														__snake_case	:    List[str]											=     gpu_peak_mem_orig + gpu_alloc_mem_orig
														__snake_case	:    Optional[int]											=     gpu_peak_mem_bnb + gpu_alloc_mem_bnb
														__snake_case	:    Optional[Any]											=     gpu_total_mem_orig - gpu_total_mem_bnb

														# sshleifer/student_marian_en_ro_6_1 has 54M parameter, 29M of which is `nn.Embedding` which
														# doesn't get quantized and remains in fp32. Therefore we only have 25M parameters quantized
														# in 2 bytes and the diff in optim memory usage is derived as so:
														#
														# - normal 25*8=~200MB (8 bytes per param)
														# - bnb    25*2= ~50MB (2 bytes per param)
														#
														# Thus we should expect ~150MB total memory saved.
														#
														# Peak memory should be the same - the total should be different by about that same margin
														#
														# After leaving a small margin to accommodate for differences between gpus let's check
														# that we have at least 120MB in savings
														__snake_case	:    List[str]											=     120

														# uncomment the following if this test starts failing - requires py38 for a new print feature
														# gpu_peak_mem_diff = gpu_peak_mem_orig - gpu_peak_mem_bnb
														# print(f"{gpu_alloc_mem_orig=}MB {gpu_peak_mem_orig=}MB {gpu_alloc_mem_orig+gpu_peak_mem_orig=}MB")
														# print(f" {gpu_alloc_mem_bnb=}MB  {gpu_peak_mem_bnb=}MB  {gpu_alloc_mem_bnb+gpu_peak_mem_bnb=}MB")
														# print(f"{gpu_alloc_mem_diff=}MB")
														# print(f"{gpu_peak_mem_diff=}MB")
														# print(f"{gpu_total_mem_orig=}MB, {gpu_total_mem_bnb=}MB")
														# print(f"{gpu_total_mem_diff=}MB, {gpu_total_mem_diff=}MB")

														self.assertGreater(
														    lowerCamelCase					,			lowerCamelCase					,			"should use ~150MB less alloc gpu memory with BNB, compared to without it for this model but got"
														    F' a difference of {gpu_alloc_mem_diff}MB, with gpu_alloc_mem_orig={gpu_alloc_mem_orig}MB and'
														    F' gpu_alloc_mem_bnb={gpu_alloc_mem_bnb}MB'					,			)

														self.assertGreater(
														    lowerCamelCase					,			lowerCamelCase					,			"should use ~150MB less total gpu memory with BNB, compared to without it for this model but got"
														    F' a difference of {gpu_total_mem_diff}MB, with gpu_total_mem_orig={gpu_total_mem_orig}MB and'
														    F' gpu_total_mem_bnb={gpu_total_mem_bnb}MB'					,			)

														self.assertEqual(
														    lowerCamelCase					,			lowerCamelCase					,			F'loss should be the same, but got loss_orig={loss_orig}, loss_bnb={loss_bnb}'       )






							def 					__snake_case      (							self :						int					,			lowerCamelCase :						int					,			lowerCamelCase :						str					,			lowerCamelCase :						int					,			lowerCamelCase :						float = 3E-3					,			lowerCamelCase :						str = "adafactor"					,			lowerCamelCase :						bool = False					,			lowerCamelCase :						str = None					,			lowerCamelCase :						int = 0					,			lowerCamelCase :						bool = True					,			lowerCamelCase :						bool = True					,			lowerCamelCase :						bool = True					,			lowerCamelCase :						bool = True					,			lowerCamelCase :						int = None					,			)			->					int:
														__snake_case	:    Union[str, Any]											=     self.test_file_dir / "../fixtures/tests_samples/wmt_en_ro"
														__snake_case	:    str											=     self.get_auto_remove_tmp_dir()
														__snake_case	:    List[Any]											=     F'\n            --model_name_or_path {model_name}\n            --train_file {data_dir}/train.json\n            --validation_file {data_dir}/val.json\n            --test_file {data_dir}/test.json\n            --output_dir {output_dir}\n            --overwrite_output_dir\n            --max_train_samples 8\n            --max_source_length {max_len}\n            --max_target_length {max_len}\n            --do_train\n            --num_train_epochs {str(lowerCamelCase       )}\n            --per_device_train_batch_size 4\n            --learning_rate {learning_rate}\n            --warmup_steps 8\n            --logging_steps 0\n            --logging_strategy no\n            --save_steps {str(lowerCamelCase       )}\n            --group_by_length\n            --label_smoothing_factor 0.1\n            --target_lang ro_RO\n            --source_lang en_XX\n        '.split()

														__snake_case	:    str											=     F'\n            --do_eval\n            --per_device_eval_batch_size 4\n            --max_eval_samples 8\n            --val_max_target_length {max_len}\n            --evaluation_strategy steps\n            --eval_steps {str(lowerCamelCase       )}\n        '.split()

														__snake_case	:    Union[str, Any]											=     "\n            --do_predict\n        ".split()

														__snake_case	:    int											=     []
														if do_train:
																					args += args_train

														if do_eval:
																					args += args_eval

														if do_predict:
																					args += args_predict

														if predict_with_generate:
																					args += "--predict_with_generate".split()

														if do_train:
																					if optim == "adafactor":
																												args += "--adafactor".split()
																					else:
																												args += F'--optim {optim}'.split()

														if extra_args_str is not None:
																					args += extra_args_str.split()

														if distributed:
																					if n_gpus_to_use is None:
																												__snake_case	:    List[str]											=     get_gpu_count()
																					__snake_case	:    int											=     get_torch_dist_unique_port()
																					__snake_case	:    Any											=     F'\n                -m torch.distributed.run\n                --nproc_per_node={n_gpus_to_use}\n                --master_port={master_port}\n                {self.examples_dir_str}/pytorch/translation/run_translation.py\n            '.split()
																					__snake_case	:    Optional[int]											=     [sys.executable] + distributed_args + args
																					# keep for quick debug
																					# print(" ".join([f"\nPYTHONPATH={self.src_dir_str}"] +cmd)); die
																					execute_subprocess_async(lowerCamelCase					,			env=self.get_env()       )
														else:
																					__snake_case	:    List[Any]											=     ["run_translation.py"] + args
																					with patch.object(lowerCamelCase					,			"argv"					,			lowerCamelCase       ):
																												main()

														return output_dir





from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


_snake_case  :	List[Any]			      =       "\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n"

_snake_case  :	Any			      =       "\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n"

_snake_case  :	str			      =       "\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for 'record': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'prediction_text': the predicted answer text\n        - for 'multirc': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question-answer pair as specified by the dataset\n            - 'prediction': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for 'record': list of question-answers dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'answers': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for 'record':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1': F1 score\n    - for 'multirc':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1_m': Per-question macro-F1 score\n        - 'f1_a': Average F1 score over all answers\n    - for 'axb':\n        'matthews_correlation': Matthew Correlation\n    - for 'cb':\n        - 'accuracy': Accuracy\n        - 'f1': F1 score\n    - for all others:\n        - 'accuracy': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')\n    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]\n    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')\n    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'matthews_correlation': 1.0}\n"







def        lowerCAmelCase_  (      __lowerCamelCase							,			__lowerCamelCase			):
							return float((preds == labels).mean()			)







def        lowerCAmelCase_  (      __lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase="binary"			):
							__snake_case	:    Union[str, Any]											=     simple_accuracy(__lowerCamelCase							,			__lowerCamelCase			)
							__snake_case	:    List[str]											=     float(fa_score(y_true=__lowerCamelCase							,			y_pred=__lowerCamelCase							,			average=__lowerCamelCase			)			)
							return {
							    "accuracy": acc,
							    "f1": fa,
							}







def        lowerCAmelCase_  (      __lowerCamelCase							,			__lowerCamelCase			):
							__snake_case	:    List[Any]											=     {}
							for id_pred, label in zip(__lowerCamelCase							,			__lowerCamelCase			):
														__snake_case	:    int											=     F'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'
														__snake_case	:    str											=     id_pred["prediction"]
														if question_id in question_map:
																					question_map[question_id].append((pred, label)			)
														else:
																					__snake_case	:    Optional[Any]											=     [(pred, label)]
							__snake_case				,    __snake_case	:    Dict											=     [], []
							for question, preds_labels in question_map.items():
														__snake_case				,    __snake_case	:    int											=     zip(*__lowerCamelCase			)
														__snake_case	:    Optional[Any]											=     fa_score(y_true=__lowerCamelCase							,			y_pred=__lowerCamelCase							,			average="macro"			)
														fas.append(__lowerCamelCase			)
														__snake_case	:    Union[str, Any]											=     int(sum(pred == label for pred, label in preds_labels			) == len(__lowerCamelCase			)			)
														ems.append(__lowerCamelCase			)
							__snake_case	:    Tuple											=     float(sum(__lowerCamelCase			) / len(__lowerCamelCase			)			)
							__snake_case	:    Any											=     sum(__lowerCamelCase			) / len(__lowerCamelCase			)
							__snake_case	:    int											=     float(fa_score(y_true=__lowerCamelCase							,			y_pred=[id_pred["prediction"] for id_pred in ids_preds]			)			)
							return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,       _KWARGS_DESCRIPTION   )
class        a	(datasets.Metric   ):




							"""simple docstring"""


							def 					__snake_case      (							self :						List[Any]       )			->					Union[str, Any]:
														if self.config_name not in [
														    "boolq",
														    "cb",
														    "copa",
														    "multirc",
														    "record",
														    "rte",
														    "wic",
														    "wsc",
														    "wsc.fixed",
														    "axb",
														    "axg",
														]:
																					raise KeyError(
																					    "You should supply a configuration name selected in "
																					    "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"       )
														return datasets.MetricInfo(
														    description=_DESCRIPTION					,			citation=_CITATION					,			inputs_description=_KWARGS_DESCRIPTION					,			features=datasets.Features(self._get_feature_types()       )					,			codebase_urls=[]					,			reference_urls=[]					,			format="numpy" if not self.config_name == "record" and not self.config_name == "multirc" else None					,			)

							def 					__snake_case      (							self :						Any       )			->					int:
														if self.config_name == "record":
																					return {
																					    "predictions": {
																					        "idx": {
																					            "passage": datasets.Value("int64"       ),
																					            "query": datasets.Value("int64"       ),
																					        },
																					        "prediction_text": datasets.Value("string"       ),
																					    },
																					    "references": {
																					        "idx": {
																					            "passage": datasets.Value("int64"       ),
																					            "query": datasets.Value("int64"       ),
																					        },
																					        "answers": datasets.Sequence(datasets.Value("string"       )       ),
																					    },
																					}
														elif self.config_name == "multirc":
																					return {
																					    "predictions": {
																					        "idx": {
																					            "answer": datasets.Value("int64"       ),
																					            "paragraph": datasets.Value("int64"       ),
																					            "question": datasets.Value("int64"       ),
																					        },
																					        "prediction": datasets.Value("int64"       ),
																					    },
																					    "references": datasets.Value("int64"       ),
																					}
														else:
																					return {
																					    "predictions": datasets.Value("int64"       ),
																					    "references": datasets.Value("int64"       ),
																					}






							def 					__snake_case      (							self :						Optional[Any]					,			lowerCamelCase :						str					,			lowerCamelCase :						Optional[Any]       )			->					str:
														if self.config_name == "axb":
																					return {"matthews_correlation": matthews_corrcoef(lowerCamelCase					,			lowerCamelCase       )}
														elif self.config_name == "cb":
																					return acc_and_fa(lowerCamelCase					,			lowerCamelCase					,			fa_avg="macro"       )
														elif self.config_name == "record":
																					__snake_case	:    Tuple											=     [
																					    {
																					        "qas": [
																					            {"id": ref["idx"]["query"], "answers": [{"text": ans} for ans in ref["answers"]]}
																					            for ref in references
																					        ]
																					    }
																					]
																					__snake_case	:    List[str]											=     {pred["idx"]["query"]: pred["prediction_text"] for pred in predictions}
																					return evaluate_record(lowerCamelCase					,			lowerCamelCase       )[0]
														elif self.config_name == "multirc":
																					return evaluate_multirc(lowerCamelCase					,			lowerCamelCase       )
														elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
																					return {"accuracy": simple_accuracy(lowerCamelCase					,			lowerCamelCase       )}
														else:
																					raise KeyError(
																					    "You should supply a configuration name selected in "
																					    "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"       )

