

import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_a:    Tuple 							=			get_tests_dir("""fixtures/test_sentencepiece_no_bos.model""")

@require_sentencepiece
@require_tokenizers
class 						__UpperCamelCase    (					lowercase     ,     unittest.TestCase						):
	SCREAMING_SNAKE_CASE__    				=     PegasusTokenizer
	SCREAMING_SNAKE_CASE__    				=     PegasusTokenizerFast
	SCREAMING_SNAKE_CASE__    				=     True
	SCREAMING_SNAKE_CASE__    				=     True




	def       __A					(		self	:						int							):


		'''simple docstring'''







		super().setUp()

		# We have a SentencePiece fixture for testing
		UpperCAmelCase_       				=			PegasusTokenizer(lowerCAmelCase							)
		tokenizer.save_pretrained(self.tmpdirname							)




	@cached_property
	def       __A					(		self	:						Dict							):


		'''simple docstring'''







		return PegasusTokenizer.from_pretrained("google/pegasus-large"							)




	def       __A					(		self	:						Tuple		,	**lowerCAmelCase	:						str							):


		'''simple docstring'''







		return PegasusTokenizer.from_pretrained(self.tmpdirname		,	**lowerCAmelCase							)




	def       __A					(		self	:						Optional[Any]		,	lowerCAmelCase	:						Union[str, Any]							):


		'''simple docstring'''







		return ("This is a test", "This is a test")




	def       __A					(		self	:						Dict							):


		'''simple docstring'''







		UpperCAmelCase_       				=			"</s>"
		UpperCAmelCase_       				=			1

		self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase							)		,	lowerCAmelCase							)
		self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase							)		,	lowerCAmelCase							)




	def       __A					(		self	:						Dict							):


		'''simple docstring'''







		UpperCAmelCase_       				=			list(self.get_tokenizer().get_vocab().keys()							)

		self.assertEqual(vocab_keys[0]		,	"<pad>"							)
		self.assertEqual(vocab_keys[1]		,	"</s>"							)
		self.assertEqual(vocab_keys[-1]		,	"v"							)
		self.assertEqual(len(lowerCAmelCase							)		,	1_103							)




	def       __A					(		self	:						Optional[int]							):


		'''simple docstring'''







		self.assertEqual(self.get_tokenizer().vocab_size		,	1_103							)




	def       __A					(		self	:						str							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.rust_tokenizer_class.from_pretrained(self.tmpdirname							)
		UpperCAmelCase_       				=			self.tokenizer_class.from_pretrained(self.tmpdirname							)
		UpperCAmelCase_       				=			(
		    "Let's see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important"
		    " </s> <pad> <pad> <pad>"
		)
		UpperCAmelCase_       				=			rust_tokenizer([raw_input_str]		,	return_tensors=lowerCAmelCase		,	add_special_tokens=lowerCAmelCase							).input_ids[0]
		UpperCAmelCase_       				=			py_tokenizer([raw_input_str]		,	return_tensors=lowerCAmelCase		,	add_special_tokens=lowerCAmelCase							).input_ids[0]
		self.assertListEqual(lowerCAmelCase		,	lowerCAmelCase							)




	def       __A					(		self	:						Union[str, Any]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self._large_tokenizer
		# <mask_1> masks whole sentence while <mask_2> masks single word
		UpperCAmelCase_       				=			"<mask_1> To ensure a <mask_2> flow of bank resolutions."
		UpperCAmelCase_       				=			[2, 413, 615, 114, 3, 1_971, 113, 1_679, 10_710, 107, 1]
		UpperCAmelCase_       				=			tokenizer([raw_input_str]		,	return_tensors=lowerCAmelCase							).input_ids[0]
		self.assertListEqual(lowerCAmelCase		,	lowerCAmelCase							)




	def       __A					(		self	:						Dict							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self._large_tokenizer
		# The tracebacks for the following asserts are **better** without messages or self.assertEqual
		assert tokenizer.vocab_size == 96_103
		assert tokenizer.pad_token_id == 0
		assert tokenizer.eos_token_id == 1
		assert tokenizer.offset == 103
		assert tokenizer.unk_token_id == tokenizer.offset + 2 == 105
		assert tokenizer.unk_token == "<unk>"
		assert tokenizer.model_max_length == 1_024
		UpperCAmelCase_       				=			"To ensure a smooth flow of bank resolutions."
		UpperCAmelCase_       				=			[413, 615, 114, 2_291, 1_971, 113, 1_679, 10_710, 107, 1]
		UpperCAmelCase_       				=			tokenizer([raw_input_str]		,	return_tensors=lowerCAmelCase							).input_ids[0]
		self.assertListEqual(lowerCAmelCase		,	lowerCAmelCase							)
		assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3]							) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]




	@require_torch
	def       __A					(		self	:						str							):


		'''simple docstring'''







		UpperCAmelCase_       				=			["This is going to be way too long." * 150, "short example"]
		UpperCAmelCase_       				=			["not super long but more than 5 tokens", "tiny"]
		UpperCAmelCase_       				=			self._large_tokenizer(lowerCAmelCase		,	padding=lowerCAmelCase		,	truncation=lowerCAmelCase		,	return_tensors="pt"							)
		UpperCAmelCase_       				=			self._large_tokenizer(
		    text_target=lowerCAmelCase		,	max_length=5		,	padding=lowerCAmelCase		,	truncation=lowerCAmelCase		,	return_tensors="pt"							)

		assert batch.input_ids.shape == (2, 1_024)
		assert batch.attention_mask.shape == (2, 1_024)
		assert targets["input_ids"].shape == (2, 5)
		assert len(lowerCAmelCase							) == 2  # input_ids, attention_mask.




	@slow
	def       __A					(		self	:						Optional[int]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			{"input_ids": [[38_979, 143, 18_485, 606, 130, 26_669, 87_686, 121, 54_189, 1_129, 111, 26_669, 87_686, 121, 9_114, 14_787, 121, 13_249, 158, 592, 956, 121, 14_621, 31_576, 143, 62_613, 108, 9_688, 930, 43_430, 11_562, 62_613, 304, 108, 11_443, 897, 108, 9_314, 17_415, 63_399, 108, 11_443, 7_614, 18_316, 118, 4_284, 7_148, 12_430, 143, 1_400, 25_703, 158, 111, 4_284, 7_148, 11_772, 143, 21_297, 1_064, 158, 122, 204, 3_506, 1_754, 1_133, 14_787, 1_581, 115, 33_224, 4_482, 111, 1_355, 110, 29_173, 317, 50_833, 108, 20_147, 94_665, 111, 77_198, 107, 1], [110, 62_613, 117, 638, 112, 1_133, 121, 20_098, 1_355, 79_050, 13_872, 135, 1_596, 53_541, 1_352, 141, 13_039, 5_542, 124, 302, 518, 111, 268, 2_956, 115, 149, 4_427, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [139, 1_235, 2_799, 18_289, 17_780, 204, 109, 9_474, 1_296, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
		# fmt: on

		self.tokenizer_integration_test_util(
		    expected_encoding=lowerCAmelCase		,	model_name="google/bigbird-pegasus-large-arxiv"		,	revision="ba85d0851d708441f91440d509690f1ab6353415"		,	)


@require_sentencepiece
@require_tokenizers
class 						__UpperCamelCase    (					lowercase     ,     unittest.TestCase						):
	SCREAMING_SNAKE_CASE__    				=     PegasusTokenizer
	SCREAMING_SNAKE_CASE__    				=     PegasusTokenizerFast
	SCREAMING_SNAKE_CASE__    				=     True
	SCREAMING_SNAKE_CASE__    				=     True




	def       __A					(		self	:						Dict							):


		'''simple docstring'''







		super().setUp()

		# We have a SentencePiece fixture for testing
		UpperCAmelCase_       				=			PegasusTokenizer(lowerCAmelCase		,	offset=0		,	mask_token_sent=lowerCAmelCase		,	mask_token="[MASK]"							)
		tokenizer.save_pretrained(self.tmpdirname							)




	@cached_property
	def       __A					(		self	:						Any							):


		'''simple docstring'''







		return PegasusTokenizer.from_pretrained("google/bigbird-pegasus-large-arxiv"							)




	def       __A					(		self	:						Any		,	**lowerCAmelCase	:						Dict							):


		'''simple docstring'''







		return PegasusTokenizer.from_pretrained(self.tmpdirname		,	**lowerCAmelCase							)




	def       __A					(		self	:						str		,	lowerCAmelCase	:						Tuple							):


		'''simple docstring'''







		return ("This is a test", "This is a test")




	def       __A					(		self	:						Tuple							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.rust_tokenizer_class.from_pretrained(self.tmpdirname							)
		UpperCAmelCase_       				=			self.tokenizer_class.from_pretrained(self.tmpdirname							)
		UpperCAmelCase_       				=			(
		    "Let's see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>"
		    " <pad> <pad> <pad>"
		)
		UpperCAmelCase_       				=			rust_tokenizer([raw_input_str]		,	return_tensors=lowerCAmelCase		,	add_special_tokens=lowerCAmelCase							).input_ids[0]
		UpperCAmelCase_       				=			py_tokenizer([raw_input_str]		,	return_tensors=lowerCAmelCase		,	add_special_tokens=lowerCAmelCase							).input_ids[0]
		self.assertListEqual(lowerCAmelCase		,	lowerCAmelCase							)




	@require_torch
	def       __A					(		self	:						Dict							):


		'''simple docstring'''







		UpperCAmelCase_       				=			["This is going to be way too long." * 1_000, "short example"]
		UpperCAmelCase_       				=			["not super long but more than 5 tokens", "tiny"]
		UpperCAmelCase_       				=			self._large_tokenizer(lowerCAmelCase		,	padding=lowerCAmelCase		,	truncation=lowerCAmelCase		,	return_tensors="pt"							)
		UpperCAmelCase_       				=			self._large_tokenizer(
		    text_target=lowerCAmelCase		,	max_length=5		,	padding=lowerCAmelCase		,	truncation=lowerCAmelCase		,	return_tensors="pt"							)

		assert batch.input_ids.shape == (2, 4_096)
		assert batch.attention_mask.shape == (2, 4_096)
		assert targets["input_ids"].shape == (2, 5)
		assert len(lowerCAmelCase							) == 2  # input_ids, attention_mask.




	def       __A					(		self	:						List[str]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			(
		    "This is an example string that is used to test the original TF implementation against the HF"
		    " implementation"
		)

		UpperCAmelCase_       				=			self._large_tokenizer(lowerCAmelCase							).input_ids

		self.assertListEqual(
		    lowerCAmelCase		,	[182, 117, 142, 587, 4_211, 120, 117, 263, 112, 804, 109, 856, 25_016, 3_137, 464, 109, 26_955, 3_137, 1]		,	)


from sklearn.metrics import mean_squared_error

import datasets


_a:    Any 							=			"""\
@article{scikit-learn,
 title={Scikit-learn: Machine Learning in {P}ython},
 author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
 journal={Journal of Machine Learning Research},
 volume={12},
 pages={2825--2830},
 year={2011}
}
"""

_a:    List[Any] 							=			"""\
Mean Squared Error(MSE) is the average of the square of difference between the predicted
and actual values.
"""


_a:    List[str] 							=			"""
Args:
    predictions: array-like of shape (n_samples,) or (n_samples, n_outputs)
        Estimated target values.
    references: array-like of shape (n_samples,) or (n_samples, n_outputs)
        Ground truth (correct) target values.
    sample_weight: array-like of shape (n_samples,), default=None
        Sample weights.
    multioutput: {\"raw_values\", \"uniform_average\"} or array-like of shape (n_outputs,), default=\"uniform_average\"
        Defines aggregating of multiple output values. Array-like value defines weights used to average errors.

        \"raw_values\" : Returns a full set of errors in case of multioutput input.

        \"uniform_average\" : Errors of all outputs are averaged with uniform weight.

    squared : bool, default=True
        If True returns MSE value, if False returns RMSE (Root Mean Squared Error) value.

Returns:
    mse : mean squared error.
Examples:

    >>> mse_metric = datasets.load_metric(\"mse\")
    >>> predictions = [2.5, 0.0, 2, 8]
    >>> references = [3, -0.5, 2, 7]
    >>> results = mse_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'mse': 0.375}
    >>> rmse_result = mse_metric.compute(predictions=predictions, references=references, squared=False)
    >>> print(rmse_result)
    {'mse': 0.6123724356957945}

    If you're using multi-dimensional lists, then set the config as follows :

    >>> mse_metric = datasets.load_metric(\"mse\", \"multilist\")
    >>> predictions = [[0.5, 1], [-1, 1], [7, -6]]
    >>> references = [[0, 2], [-1, 2], [8, -5]]
    >>> results = mse_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'mse': 0.7083333333333334}
    >>> results = mse_metric.compute(predictions=predictions, references=references, multioutput='raw_values')
    >>> print(results) # doctest: +NORMALIZE_WHITESPACE
    {'mse': array([0.41666667, 1. ])}
"""

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,     _KWARGS_DESCRIPTION						)
class 						__UpperCamelCase    (					datasets.Metric						):




	def       __A					(		self	:						List[str]							):


		'''simple docstring'''







		return datasets.MetricInfo(
		    description=_DESCRIPTION		,	citation=_CITATION		,	inputs_description=_KWARGS_DESCRIPTION		,	features=datasets.Features(self._get_feature_types()							)		,	reference_urls=[
		        "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html"
		    ]		,	)




	def       __A					(		self	:						Optional[Any]							):


		'''simple docstring'''







		if self.config_name == "multilist":
			return {
			    "predictions": datasets.Sequence(datasets.Value("float"							)							),
			    "references": datasets.Sequence(datasets.Value("float"							)							),
			}
		else:
			return {
			    "predictions": datasets.Value("float"							),
			    "references": datasets.Value("float"							),
			}




	def       __A					(		self	:						Tuple		,	lowerCAmelCase	:						int		,	lowerCAmelCase	:						Dict		,	lowerCAmelCase	:						str=None		,	lowerCAmelCase	:						str="uniform_average"		,	lowerCAmelCase	:						Any=True							):


		'''simple docstring'''







		UpperCAmelCase_       				=			mean_squared_error(
		    lowerCAmelCase		,	lowerCAmelCase		,	sample_weight=lowerCAmelCase		,	multioutput=lowerCAmelCase		,	squared=lowerCAmelCase							)

		return {"mse": mse}