


import logging
import os
from dataclasses import dataclass, field
from typing import Dict, Optional

import numpy as np
from utils_multiple_choice import MultipleChoiceDataset, Split, processors

import transformers
from transformers import (
    AutoConfig,
    AutoModelForMultipleChoice,
    AutoTokenizer,
    DataCollatorWithPadding,
    EvalPrediction,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
)
from transformers.trainer_utils import is_main_process


_snake_case  :	Tuple			      =       logging.getLogger(__name__)







def        lowerCAmelCase_  (      __lowerCamelCase							,			__lowerCamelCase			):
							return (preds == labels).mean()

@dataclass
class        a	:




							"""simple docstring"""


							__UpperCAmelCase					:							str  						=						field(
							    metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}   )
							__UpperCAmelCase					:							Union[str, Any]  						=						field(
							    default=__a  ,       metadata={"help": "Pretrained config name or path if not the same as model_name"}   )
							__UpperCAmelCase					:							List[str]  						=						field(
							    default=__a  ,       metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}   )
							__UpperCAmelCase					:							List[str]  						=						field(
							    default=__a  ,       metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}  ,       )

@dataclass
class        a	:




							"""simple docstring"""


							__UpperCAmelCase					:							int  						=						field(metadata={"help": "The name of the task to train on: " + ", ".join(processors.keys()   )}   )
							__UpperCAmelCase					:							int  						=						field(metadata={"help": "Should contain the data files for the task."}   )
							__UpperCAmelCase					:							Optional[int]  						=						field(
							    default=128  ,       metadata={
							        "help": (
							            "The maximum total input sequence length after tokenization. Sequences longer "
							            "than this will be truncated, sequences shorter will be padded."
							        )
							    }  ,       )
							__UpperCAmelCase					:							Any  						=						field(
							    default=__a  ,       metadata={"help": "Overwrite the cached training and evaluation sets"}   )







def        lowerCAmelCase_  (      ):
							__snake_case	:    Dict											=     HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)			)
							__snake_case	:    List[Any]											=     parser.parse_args_into_dataclasses()

							if (
							    os.path.exists(training_args.output_dir			)
							    and os.listdir(training_args.output_dir			)
							    and training_args.do_train
							    and not training_args.overwrite_output_dir
							):
														raise ValueError(
														    F'Output directory ({training_args.output_dir}) already exists and is not empty. Use'
														    " --overwrite_output_dir to overcome."			)

							# Setup logging
							logging.basicConfig(
							    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"							,			datefmt="%m/%d/%Y %H:%M:%S"							,			level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN							,			)
							logger.warning(
							    "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"							,			training_args.local_rank							,			training_args.device							,			training_args.n_gpu							,			bool(training_args.local_rank != -1			)							,			training_args.fpaa							,			)
							# Set the verbosity to info of the Transformers logger (on main process only):
							if is_main_process(training_args.local_rank			):
														transformers.utils.logging.set_verbosity_info()
														transformers.utils.logging.enable_default_handler()
														transformers.utils.logging.enable_explicit_format()
							logger.info("Training/evaluation parameters %s"							,			snake_case__			)

							# Set seed
							set_seed(training_args.seed			)

							try:
														__snake_case	:    int											=     processors[data_args.task_name]()
														__snake_case	:    Any											=     processor.get_labels()
														__snake_case	:    int											=     len(snake_case__			)
							except KeyError:
														raise ValueError("Task not found: %s" % (data_args.task_name)			)

							# Load pretrained model and tokenizer
							#
							# Distributed training:
							# The .from_pretrained methods guarantee that only one local process can concurrently
							# download model & vocab.

							__snake_case	:    List[Any]											=     AutoConfig.from_pretrained(
							    model_args.config_name if model_args.config_name else model_args.model_name_or_path							,			num_labels=snake_case__							,			finetuning_task=data_args.task_name							,			cache_dir=model_args.cache_dir							,			)
							__snake_case	:    List[Any]											=     AutoTokenizer.from_pretrained(
							    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path							,			cache_dir=model_args.cache_dir							,			)
							__snake_case	:    Optional[int]											=     AutoModelForMultipleChoice.from_pretrained(
							    model_args.model_name_or_path							,			from_tf=bool(".ckpt" in model_args.model_name_or_path			)							,			config=snake_case__							,			cache_dir=model_args.cache_dir							,			)

							# Get datasets
							__snake_case	:    List[str]											=     (
							    MultipleChoiceDataset(
							        data_dir=data_args.data_dir							,			tokenizer=snake_case__							,			task=data_args.task_name							,			max_seq_length=data_args.max_seq_length							,			overwrite_cache=data_args.overwrite_cache							,			mode=Split.train							,			)
							    if training_args.do_train
							    else None
							)
							__snake_case	:    Dict											=     (
							    MultipleChoiceDataset(
							        data_dir=data_args.data_dir							,			tokenizer=snake_case__							,			task=data_args.task_name							,			max_seq_length=data_args.max_seq_length							,			overwrite_cache=data_args.overwrite_cache							,			mode=Split.dev							,			)
							    if training_args.do_eval
							    else None
							)

							def compute_metrics(__lowerCamelCase			) -> Dict:
														__snake_case	:    List[Any]											=     np.argmax(p.predictions							,			axis=1			)
														return {"acc": simple_accuracy(snake_case__							,			p.label_ids			)}

							# Data collator
							__snake_case	:    Any											=     DataCollatorWithPadding(snake_case__							,			pad_to_multiple_of=8			) if training_args.fpaa else None

							# Initialize our Trainer
							__snake_case	:    int											=     Trainer(
							    model=snake_case__							,			args=snake_case__							,			train_dataset=snake_case__							,			eval_dataset=snake_case__							,			compute_metrics=snake_case__							,			data_collator=snake_case__							,			)

							# Training
							if training_args.do_train:
														trainer.train(
														    model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path			) else None			)
														trainer.save_model()
														# For convenience, we also re-save the tokenizer to the same directory,
														# so that you can share your model easily on huggingface.co/models =)
														if trainer.is_world_master():
																					tokenizer.save_pretrained(training_args.output_dir			)

														# Evaluation
							__snake_case	:    Optional[int]											=     {}
							if training_args.do_eval:
														logger.info("*** Evaluate ***"			)

														__snake_case	:    Optional[int]											=     trainer.evaluate()

														__snake_case	:    int											=     os.path.join(training_args.output_dir							,			"eval_results.txt"			)
														if trainer.is_world_master():
																					with open(snake_case__							,			"w"			) as writer:
																												logger.info("***** Eval results *****"			)
																												for key, value in result.items():
																																			logger.info("  %s = %s"							,			snake_case__							,			snake_case__			)
																																			writer.write("%s = %s\n" % (key, value)			)

																												results.update(snake_case__			)

							return results







def        lowerCAmelCase_  (      __lowerCamelCase			):
							main()


if __name__ == "__main__":
				main()





from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


_snake_case  :	List[Any]			      =       "\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n"

_snake_case  :	Any			      =       "\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n"

_snake_case  :	str			      =       "\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for 'record': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'prediction_text': the predicted answer text\n        - for 'multirc': list of question-answer dictionaries with the following keys:\n            - 'idx': index of the question-answer pair as specified by the dataset\n            - 'prediction': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for 'record': list of question-answers dictionaries with the following keys:\n            - 'idx': index of the question as specified by the dataset\n            - 'answers': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for 'record':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1': F1 score\n    - for 'multirc':\n        - 'exact_match': Exact match between answer and gold answer\n        - 'f1_m': Per-question macro-F1 score\n        - 'f1_a': Average F1 score over all answers\n    - for 'axb':\n        'matthews_correlation': Matthew Correlation\n    - for 'cb':\n        - 'accuracy': Accuracy\n        - 'f1': F1 score\n    - for all others:\n        - 'accuracy': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'copa')  # any of [\"copa\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"boolq\", \"axg\"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'cb')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'record')\n    >>> predictions = [{'idx': {'passage': 0, 'query': 0}, 'prediction_text': 'answer'}]\n    >>> references = [{'idx': {'passage': 0, 'query': 0}, 'answers': ['answer', 'another_answer']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'multirc')\n    >>> predictions = [{'idx': {'answer': 0, 'paragraph': 0, 'question': 0}, 'prediction': 0}, {'idx': {'answer': 1, 'paragraph': 2, 'question': 3}, 'prediction': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'exact_match': 1.0, 'f1_m': 1.0, 'f1_a': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric('super_glue', 'axb')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'matthews_correlation': 1.0}\n"







def        lowerCAmelCase_  (      __lowerCamelCase							,			__lowerCamelCase			):
							return float((preds == labels).mean()			)







def        lowerCAmelCase_  (      __lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase="binary"			):
							__snake_case	:    Union[str, Any]											=     simple_accuracy(__lowerCamelCase							,			__lowerCamelCase			)
							__snake_case	:    List[str]											=     float(fa_score(y_true=__lowerCamelCase							,			y_pred=__lowerCamelCase							,			average=__lowerCamelCase			)			)
							return {
							    "accuracy": acc,
							    "f1": fa,
							}







def        lowerCAmelCase_  (      __lowerCamelCase							,			__lowerCamelCase			):
							__snake_case	:    List[Any]											=     {}
							for id_pred, label in zip(__lowerCamelCase							,			__lowerCamelCase			):
														__snake_case	:    int											=     F'{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}'
														__snake_case	:    str											=     id_pred["prediction"]
														if question_id in question_map:
																					question_map[question_id].append((pred, label)			)
														else:
																					__snake_case	:    Optional[Any]											=     [(pred, label)]
							__snake_case				,    __snake_case	:    Dict											=     [], []
							for question, preds_labels in question_map.items():
														__snake_case				,    __snake_case	:    int											=     zip(*__lowerCamelCase			)
														__snake_case	:    Optional[Any]											=     fa_score(y_true=__lowerCamelCase							,			y_pred=__lowerCamelCase							,			average="macro"			)
														fas.append(__lowerCamelCase			)
														__snake_case	:    Union[str, Any]											=     int(sum(pred == label for pred, label in preds_labels			) == len(__lowerCamelCase			)			)
														ems.append(__lowerCamelCase			)
							__snake_case	:    Tuple											=     float(sum(__lowerCamelCase			) / len(__lowerCamelCase			)			)
							__snake_case	:    Any											=     sum(__lowerCamelCase			) / len(__lowerCamelCase			)
							__snake_case	:    int											=     float(fa_score(y_true=__lowerCamelCase							,			y_pred=[id_pred["prediction"] for id_pred in ids_preds]			)			)
							return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,       _KWARGS_DESCRIPTION   )
class        a	(datasets.Metric   ):




							"""simple docstring"""


							def 					__snake_case      (							self :						List[Any]       )			->					Union[str, Any]:
														if self.config_name not in [
														    "boolq",
														    "cb",
														    "copa",
														    "multirc",
														    "record",
														    "rte",
														    "wic",
														    "wsc",
														    "wsc.fixed",
														    "axb",
														    "axg",
														]:
																					raise KeyError(
																					    "You should supply a configuration name selected in "
																					    "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"       )
														return datasets.MetricInfo(
														    description=_DESCRIPTION					,			citation=_CITATION					,			inputs_description=_KWARGS_DESCRIPTION					,			features=datasets.Features(self._get_feature_types()       )					,			codebase_urls=[]					,			reference_urls=[]					,			format="numpy" if not self.config_name == "record" and not self.config_name == "multirc" else None					,			)

							def 					__snake_case      (							self :						Any       )			->					int:
														if self.config_name == "record":
																					return {
																					    "predictions": {
																					        "idx": {
																					            "passage": datasets.Value("int64"       ),
																					            "query": datasets.Value("int64"       ),
																					        },
																					        "prediction_text": datasets.Value("string"       ),
																					    },
																					    "references": {
																					        "idx": {
																					            "passage": datasets.Value("int64"       ),
																					            "query": datasets.Value("int64"       ),
																					        },
																					        "answers": datasets.Sequence(datasets.Value("string"       )       ),
																					    },
																					}
														elif self.config_name == "multirc":
																					return {
																					    "predictions": {
																					        "idx": {
																					            "answer": datasets.Value("int64"       ),
																					            "paragraph": datasets.Value("int64"       ),
																					            "question": datasets.Value("int64"       ),
																					        },
																					        "prediction": datasets.Value("int64"       ),
																					    },
																					    "references": datasets.Value("int64"       ),
																					}
														else:
																					return {
																					    "predictions": datasets.Value("int64"       ),
																					    "references": datasets.Value("int64"       ),
																					}






							def 					__snake_case      (							self :						Optional[Any]					,			lowerCamelCase :						str					,			lowerCamelCase :						Optional[Any]       )			->					str:
														if self.config_name == "axb":
																					return {"matthews_correlation": matthews_corrcoef(lowerCamelCase					,			lowerCamelCase       )}
														elif self.config_name == "cb":
																					return acc_and_fa(lowerCamelCase					,			lowerCamelCase					,			fa_avg="macro"       )
														elif self.config_name == "record":
																					__snake_case	:    Tuple											=     [
																					    {
																					        "qas": [
																					            {"id": ref["idx"]["query"], "answers": [{"text": ans} for ans in ref["answers"]]}
																					            for ref in references
																					        ]
																					    }
																					]
																					__snake_case	:    List[str]											=     {pred["idx"]["query"]: pred["prediction_text"] for pred in predictions}
																					return evaluate_record(lowerCamelCase					,			lowerCamelCase       )[0]
														elif self.config_name == "multirc":
																					return evaluate_multirc(lowerCamelCase					,			lowerCamelCase       )
														elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
																					return {"accuracy": simple_accuracy(lowerCamelCase					,			lowerCamelCase       )}
														else:
																					raise KeyError(
																					    "You should supply a configuration name selected in "
																					    "[\"boolq\", \"cb\", \"copa\", \"multirc\", \"record\", \"rte\", \"wic\", \"wsc\", \"wsc.fixed\", \"axb\", \"axg\",]"       )

