


from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


_snake_case    		=       '\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n'

_snake_case    		=       '\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n'

_snake_case    		=       '\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for \'record\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'prediction_text\': the predicted answer text\n        - for \'multirc\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question-answer pair as specified by the dataset\n            - \'prediction\': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for \'record\': list of question-answers dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'answers\': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for \'record\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1\': F1 score\n    - for \'multirc\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1_m\': Per-question macro-F1 score\n        - \'f1_a\': Average F1 score over all answers\n    - for \'axb\':\n        \'matthews_correlation\': Matthew Correlation\n    - for \'cb\':\n        - \'accuracy\': Accuracy\n        - \'f1\': F1 score\n    - for all others:\n        - \'accuracy\': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')\n    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]\n    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')\n    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'matthews_correlation\': 1.0}\n'
def        UpperCamelCase_(				_A   :Tuple				,     _A   :Any  )->		int:
			return float((preds == labels).mean()  )
def        UpperCamelCase_(				_A   :Optional[int]				,     _A   :str				,     _A   :Dict="binary"  )->		List[Any]:
			UpperCamelCase__					=       simple_accuracy(_A				,     _A  )
			UpperCamelCase__					=       float(fa_score(y_true=_A				,     y_pred=_A				,     average=_A  )  )
			return {
			    "accuracy": acc,
			    "f1": fa,
			}
def        UpperCamelCase_(				_A   :Optional[Any]				,     _A   :int  )->		List[str]:
			UpperCamelCase__					=       {}
			for id_pred, label in zip(_A				,     _A  ):
						UpperCamelCase__					=       F'''{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}'''
						UpperCamelCase__					=       id_pred["prediction"]
						if question_id in question_map:
									question_map[question_id].append((pred, label)  )
						else:
									UpperCamelCase__					=       [(pred, label)]
			UpperCamelCase__,      UpperCamelCase__					=       [], []
			for question, preds_labels in question_map.items():
						UpperCamelCase__,      UpperCamelCase__					=       zip(*_A  )
						UpperCamelCase__					=       fa_score(y_true=_A				,     y_pred=_A				,     average="macro"  )
						fas.append(_A  )
						UpperCamelCase__					=       int(sum(pred == label for pred, label in preds_labels  ) == len(_A  )  )
						ems.append(_A  )
			UpperCamelCase__					=       float(sum(_A  ) / len(_A  )  )
			UpperCamelCase__					=       sum(_A  ) / len(_A  )
			UpperCamelCase__					=       float(fa_score(y_true=_A				,     y_pred=[id_pred["prediction"] for id_pred in ids_preds]  )  )
			return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION					,	_KWARGS_DESCRIPTION      )
class        lowerCamelCase__	(							datasets.Metric      ):






			"""simple docstring"""






			def 				snake_case__				(							self     ):

						'''simple docstring'''


						if self.config_name not in [
						    "boolq",
						    "cb",
						    "copa",
						    "multirc",
						    "record",
						    "rte",
						    "wic",
						    "wsc",
						    "wsc.fixed",
						    "axb",
						    "axg",
						]:
									raise KeyError(
									    "You should supply a configuration name selected in "
									    "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"     )
						return datasets.MetricInfo(
						    description=_DESCRIPTION      ,						citation=_CITATION      ,						inputs_description=_KWARGS_DESCRIPTION      ,						features=datasets.Features(self._get_feature_types()     )      ,						codebase_urls=[]      ,						reference_urls=[]      ,						format="numpy" if not self.config_name == "record" and not self.config_name == "multirc" else None      ,						)




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						if self.config_name == "record":
									return {
									    "predictions": {
									        "idx": {
									            "passage": datasets.Value("int64"     ),
									            "query": datasets.Value("int64"     ),
									        },
									        "prediction_text": datasets.Value("string"     ),
									    },
									    "references": {
									        "idx": {
									            "passage": datasets.Value("int64"     ),
									            "query": datasets.Value("int64"     ),
									        },
									        "answers": datasets.Sequence(datasets.Value("string"     )     ),
									    },
									}
						elif self.config_name == "multirc":
									return {
									    "predictions": {
									        "idx": {
									            "answer": datasets.Value("int64"     ),
									            "paragraph": datasets.Value("int64"     ),
									            "question": datasets.Value("int64"     ),
									        },
									        "prediction": datasets.Value("int64"     ),
									    },
									    "references": datasets.Value("int64"     ),
									}
						else:
									return {
									    "predictions": datasets.Value("int64"     ),
									    "references": datasets.Value("int64"     ),
									}







			def 				snake_case__				(							self      ,						snake_case      ,						snake_case     ):

						'''simple docstring'''


						if self.config_name == "axb":
									return {"matthews_correlation": matthews_corrcoef(snake_case      ,						snake_case     )}
						elif self.config_name == "cb":
									return acc_and_fa(snake_case      ,						snake_case      ,						fa_avg="macro"     )
						elif self.config_name == "record":
									UpperCamelCase__					=       [
									    {
									        "qas": [
									            {"id": ref["idx"]["query"], "answers": [{"text": ans} for ans in ref["answers"]]}
									            for ref in references
									        ]
									    }
									]
									UpperCamelCase__					=       {pred["idx"]["query"]: pred["prediction_text"] for pred in predictions}
									return evaluate_record(snake_case      ,						snake_case     )[0]
						elif self.config_name == "multirc":
									return evaluate_multirc(snake_case      ,						snake_case     )
						elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
									return {"accuracy": simple_accuracy(snake_case      ,						snake_case     )}
						else:
									raise KeyError(
									    "You should supply a configuration name selected in "
									    "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"     )






import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


__UpperCamelCase    		=       logging.get_logger(__name__)
def        UpperCamelCase_(				)->		Any:
			# Get the sagemaker specific mp parameters from smp_options variable.
			UpperCamelCase__					=       os.getenv("SM_HP_MP_PARAMETERS"				,     "{}"  )
			try:
						# Parse it and check the field "partitions" is included, it is required for model parallel.
						UpperCamelCase__					=       json.loads(_A  )
						if "partitions" not in smp_options:
									return False
			except json.JSONDecodeError:
						return False

			# Get the sagemaker specific framework parameters from mpi_options variable.
			UpperCamelCase__					=       os.getenv("SM_FRAMEWORK_PARAMS"				,     "{}"  )
			try:
						# Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
						UpperCamelCase__					=       json.loads(_A  )
						if not mpi_options.get("sagemaker_mpi_enabled"				,     _A  ):
									return False
			except json.JSONDecodeError:
						return False
			# Lastly, check if the `smdistributed` module is present.
			return importlib.util.find_spec("smdistributed"  ) is not None


if is_sagemaker_model_parallel_available():
						import smdistributed.modelparallel.torch as smp

						smp.init()







@dataclass
class        lowerCamelCase__	(							UpperCAmelCase      ):






			"""simple docstring"""






			_UpperCamelCase		:					str    =		field(
			    default=''					,	metadata={'help': 'Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer'}					,	)




			def 				snake_case__				(							self     ):

						'''simple docstring'''


						super().__post_init__()
						warnings.warn(
						    "`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use "
						    "`TrainingArguments` instead."      ,						snake_case      ,						)




			@cached_property
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						logger.info("PyTorch: setting up devices"     )
						if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
									logger.warning(
									    "torch.distributed process group is initialized, but local_rank == -1. "
									    "In order to use Torch DDP, launch your script with `python -m torch.distributed.launch"     )
						if self.no_cuda:
									UpperCamelCase__					=       torch.device("cpu"     )
									UpperCamelCase__					=       0
						elif is_sagemaker_model_parallel_available():
									UpperCamelCase__					=       smp.local_rank()
									UpperCamelCase__					=       torch.device("cuda"      ,						snake_case     )
									UpperCamelCase__					=       1
						elif is_sagemaker_dp_enabled():
									import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

									torch.distributed.init_process_group(backend="smddp"      ,						timeout=self.ddp_timeout_delta     )
									UpperCamelCase__					=       int(os.getenv("SMDATAPARALLEL_LOCAL_RANK"     )     )
									UpperCamelCase__					=       torch.device("cuda"      ,						self.local_rank     )
									UpperCamelCase__					=       1
						elif self.local_rank == -1:
									# if n_gpu is > 1 we'll use nn.DataParallel.
									# If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
									# Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
									# trigger an error that a device index is missing. Index 0 takes into account the
									# GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
									# will use the first GPU in that env, i.e. GPU#1
									UpperCamelCase__					=       torch.device("cuda:0" if torch.cuda.is_available() else "cpu"     )
									# Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
									# the default value.
									UpperCamelCase__					=       torch.cuda.device_count()
						else:
									# Here, we'll use torch.distributed.
									# Initializes the distributed backend which will take care of synchronizing nodes/GPUs
									if not torch.distributed.is_initialized():
												torch.distributed.init_process_group(backend="nccl"      ,						timeout=self.ddp_timeout_delta     )
									UpperCamelCase__					=       torch.device("cuda"      ,						self.local_rank     )
									UpperCamelCase__					=       1

						if device.type == "cuda":
									torch.cuda.set_device(snake_case     )

						return device




			@property
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						if is_sagemaker_model_parallel_available():
									return smp.dp_size()

						return super().world_size




			@property
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						return not is_sagemaker_model_parallel_available()







			@property
			def 				snake_case__				(							self     ):

						'''simple docstring'''


						return False


