



import datasets

from .evaluate import evaluate


snake_case    :						int  =						'\\n@inproceedings{Rajpurkar2016SQuAD10,\n  title={SQuAD: 100, 000+ Questions for Machine Comprehension of Text},\n  author={Pranav Rajpurkar and Jian Zhang and Konstantin Lopyrev and Percy Liang},\n  booktitle={EMNLP},\n  year={2016}\n}\n'

snake_case    :						Dict  =						'\nThis metric wrap the official scoring script for version 1 of the Stanford Question Answering Dataset (SQuAD).\n\nStanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, consisting of questions posed by\ncrowdworkers on a set of Wikipedia articles, where the answer to every question is a segment of text, or span,\nfrom the corresponding reading passage, or the question might be unanswerable.\n'

snake_case    :						Optional[int]  =						'\nComputes SQuAD scores (F1 and EM).\nArgs:\n    predictions: List of question-answers dictionaries with the following key-values:\n        - \'id\': id of the question-answer pair as given in the references (see below)\n        - \'prediction_text\': the text of the answer\n    references: List of question-answers dictionaries with the following key-values:\n        - \'id\': id of the question-answer pair (see above),\n        - \'answers\': a Dict in the SQuAD dataset format\n            {\n                \'text\': list of possible texts for the answer, as a list of strings\n                \'answer_start\': list of start positions for the answer, as a list of ints\n            }\n            Note that answer_start values are not taken into account to compute the metric.\nReturns:\n    \'exact_match\': Exact match (the normalized answer exactly match the gold answer)\n    \'f1\': The F-score of predicted tokens versus the gold answer\nExamples:\n\n    >>> predictions = [{\'prediction_text\': \'1976\', \'id\': \'56e10a3be3433e1400422b22\'}]\n    >>> references = [{\'answers\': {\'answer_start\': [97], \'text\': [\'1976\']}, \'id\': \'56e10a3be3433e1400422b22\'}]\n    >>> squad_metric = datasets.load_metric("squad")\n    >>> results = squad_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 100.0, \'f1\': 100.0}\n'




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION					,							_KWARGS_DESCRIPTION						)
class 							lowerCAmelCase__  (	datasets.Metric						):







					def  _lowercase			(       self      :     List[Any]):
											return datasets.MetricInfo(
											    description=_DESCRIPTION			,		citation=_CITATION			,		inputs_description=_KWARGS_DESCRIPTION			,		features=datasets.Features(
											        {
											            "predictions": {"id": datasets.Value("string"), "prediction_text": datasets.Value("string")},
											            "references": {
											                "id": datasets.Value("string"),
											                "answers": datasets.features.Sequence(
											                    {
											                        "text": datasets.Value("string"),
											                        "answer_start": datasets.Value("int32"),
											                    }),
											            },
											        })			,		codebase_urls=["https://rajpurkar.github.io/SQuAD-explorer/"]			,		reference_urls=["https://rajpurkar.github.io/SQuAD-explorer/"]			,		)







					def  _lowercase			(       self      :     int			,		_A      :     Union[str, Any]			,		_A      :     List[Any]):
											A__	:							List[Any]						    = {prediction["id"]: prediction["prediction_text"] for prediction in predictions}
											A__	:							List[str]						    = [
											    {
											        "paragraphs": [
											            {
											                "qas": [
											                    {
											                        "answers": [{"text": answer_text} for answer_text in ref["answers"]["text"]],
											                        "id": ref["id"],
											                    }
											                    for ref in references
											                ]
											            }
											        ]
											    }
											]
											A__	:							Any						    = evaluate(dataset=_A			,		predictions=_A)
											return score




import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device


if is_torch_available():
		from transformers import AutoModelForSeqaSeqLM, AutoTokenizer




@require_torch
@require_sentencepiece
@require_tokenizers
class 							lowerCAmelCase__  (	unittest.TestCase						):







					@slow
					def  _lowercase			(       self      :     List[Any]):
											A__	:							Optional[int]						    = AutoModelForSeqaSeqLM.from_pretrained("google/mt5-small"			,		return_dict=_A).to(_A)
											A__	:							Any						    = AutoTokenizer.from_pretrained("google/mt5-small")

											A__	:							int						    = tokenizer("Hello there"			,		return_tensors="pt").input_ids
											A__	:							List[str]						    = tokenizer("Hi I am"			,		return_tensors="pt").input_ids

											A__	:							int						    = model(input_ids.to(_A)			,		labels=labels.to(_A)).loss
											A__	:							Optional[int]						    = -(labels.shape[-1] * loss.item())

											A__	:							List[str]						    = -84.91_27
											self.assertTrue(abs(mtf_score - EXPECTED_SCORE) < 1e-4)