
import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin



class    a_ (			_a					,			unittest.TestCase    ):
			a	:			str					   =							ProphetNetTokenizer
			a	:			Any					   =							False






			def    UpperCamelCase_				(      self  ):
									super().setUp()

									_lowercase					       =					[
									    """[UNK]""",
									    """[CLS]""",
									    """[SEP]""",
									    """[PAD]""",
									    """[MASK]""",
									    """want""",
									    """##want""",
									    """##ed""",
									    """wa""",
									    """un""",
									    """runn""",
									    """##ing""",
									    """,""",
									    """low""",
									    """lowest""",
									]
									_lowercase					       =					os.path.join(self.tmpdirname				,			VOCAB_FILES_NAMES["""vocab_file"""]  )
									with open(self.vocab_file				,			"""w"""				,			encoding="""utf-8"""  ) as vocab_writer:
															vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]  )  )






			def    UpperCamelCase_				(      self				,			__UpperCamelCase  ):
									_lowercase					       =					"""UNwant\u00E9d,running"""
									_lowercase					       =					"""unwanted, running"""
									return input_text, output_text






			def    UpperCamelCase_				(      self  ):
									_lowercase					       =					self.tokenizer_class(self.vocab_file  )

									_lowercase					       =					tokenizer.tokenize("""UNwant\u00E9d,running"""  )
									self.assertListEqual(__UpperCamelCase				,			["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]  )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase  )				,			[9, 6, 7, 12, 10, 11]  )






			def    UpperCamelCase_				(      self  ):
									_lowercase					       =					BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""  )				,			["""ah""", """\u535A""", """\u63A8""", """zz"""]  )






			def    UpperCamelCase_				(      self  ):
									_lowercase					       =					BasicTokenizer(do_lower_case=__UpperCamelCase  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """  )				,			["""hello""", """!""", """how""", """are""", """you""", """?"""]  )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""  )				,			["""hello"""]  )






			def    UpperCamelCase_				(      self  ):
									_lowercase					       =					BasicTokenizer(do_lower_case=__UpperCamelCase				,			strip_accents=__UpperCamelCase  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """  )				,			["""hällo""", """!""", """how""", """are""", """you""", """?"""]  )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""  )				,			["""h\u00E9llo"""]  )






			def    UpperCamelCase_				(      self  ):
									_lowercase					       =					BasicTokenizer(do_lower_case=__UpperCamelCase				,			strip_accents=__UpperCamelCase  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """  )				,			["""hallo""", """!""", """how""", """are""", """you""", """?"""]  )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""  )				,			["""hello"""]  )






			def    UpperCamelCase_				(      self  ):
									_lowercase					       =					BasicTokenizer(do_lower_case=__UpperCamelCase  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """  )				,			["""hallo""", """!""", """how""", """are""", """you""", """?"""]  )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""  )				,			["""hello"""]  )






			def    UpperCamelCase_				(      self  ):
									_lowercase					       =					BasicTokenizer(do_lower_case=__UpperCamelCase  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """  )				,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]  )






			def    UpperCamelCase_				(      self  ):
									_lowercase					       =					BasicTokenizer(do_lower_case=__UpperCamelCase				,			strip_accents=__UpperCamelCase  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """  )				,			["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]  )






			def    UpperCamelCase_				(      self  ):
									_lowercase					       =					BasicTokenizer(do_lower_case=__UpperCamelCase				,			strip_accents=__UpperCamelCase  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """  )				,			["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]  )






			def    UpperCamelCase_				(      self  ):
									_lowercase					       =					BasicTokenizer(do_lower_case=__UpperCamelCase				,			never_split=["""[UNK]"""]  )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""  )				,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]  )






			def    UpperCamelCase_				(      self  ):
									_lowercase					       =					["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

									_lowercase					       =					{}
									for i, token in enumerate(__UpperCamelCase  ):
															_lowercase					       =					i
									_lowercase					       =					WordpieceTokenizer(vocab=__UpperCamelCase				,			unk_token="""[UNK]"""  )

									self.assertListEqual(tokenizer.tokenize(""""""  )				,			[]  )

									self.assertListEqual(tokenizer.tokenize("""unwanted running"""  )				,			["""un""", """##want""", """##ed""", """runn""", """##ing"""]  )

									self.assertListEqual(tokenizer.tokenize("""unwantedX running"""  )				,			["""[UNK]""", """runn""", """##ing"""]  )






			@require_torch
			def    UpperCamelCase_				(      self  ):
									_lowercase					       =					self.tokenizer_class.from_pretrained("""microsoft/prophetnet-large-uncased"""  )

									_lowercase					       =					["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
									_lowercase					       =					[1_037, 2_146, 20_423, 2_005, 7_680, 7_849, 3_989, 1_012, 102]
									_lowercase					       =					tokenizer(__UpperCamelCase				,			padding=__UpperCamelCase				,			return_tensors="""pt"""  )
									self.assertIsInstance(__UpperCamelCase				,			__UpperCamelCase  )
									_lowercase					       =					list(batch.input_ids.numpy()[0]  )
									self.assertListEqual(__UpperCamelCase				,			__UpperCamelCase  )

									self.assertEqual((2, 9)				,			batch.input_ids.shape  )
									self.assertEqual((2, 9)				,			batch.attention_mask.shape  )






			def    UpperCamelCase_				(      self  ):
									self.assertTrue(_is_whitespace(""" """  )  )
									self.assertTrue(_is_whitespace("""\t"""  )  )
									self.assertTrue(_is_whitespace("""\r"""  )  )
									self.assertTrue(_is_whitespace("""\n"""  )  )
									self.assertTrue(_is_whitespace("""\u00A0"""  )  )

									self.assertFalse(_is_whitespace("""A"""  )  )
									self.assertFalse(_is_whitespace("""-"""  )  )






			def    UpperCamelCase_				(      self  ):
									self.assertTrue(_is_control("""\u0005"""  )  )

									self.assertFalse(_is_control("""A"""  )  )
									self.assertFalse(_is_control(""" """  )  )
									self.assertFalse(_is_control("""\t"""  )  )
									self.assertFalse(_is_control("""\r"""  )  )






			def    UpperCamelCase_				(      self  ):
									self.assertTrue(_is_punctuation("""-"""  )  )
									self.assertTrue(_is_punctuation("""$"""  )  )
									self.assertTrue(_is_punctuation("""`"""  )  )
									self.assertTrue(_is_punctuation("""."""  )  )

									self.assertFalse(_is_punctuation("""A"""  )  )
									self.assertFalse(_is_punctuation(""" """  )  )






			@slow
			def    UpperCamelCase_				(      self  ):
									_lowercase					       =					self.tokenizer_class.from_pretrained("""microsoft/prophetnet-large-uncased"""  )

									_lowercase					       =					tokenizer.encode("""sequence builders"""				,			add_special_tokens=__UpperCamelCase  )
									_lowercase					       =					tokenizer.encode("""multi-sequence build"""				,			add_special_tokens=__UpperCamelCase  )

									_lowercase					       =					tokenizer.build_inputs_with_special_tokens(__UpperCamelCase  )
									_lowercase					       =					tokenizer.build_inputs_with_special_tokens(__UpperCamelCase				,			__UpperCamelCase  )

									assert encoded_sentence == text + [102]
									assert encoded_pair == text + [102] + text_a + [102]

from sklearn.metrics import mean_squared_error

import datasets


A		: List[Any]								=     '''\
@article{scikit-learn,
 title={Scikit-learn: Machine Learning in {P}ython},
 author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
 journal={Journal of Machine Learning Research},
 volume={12},
 pages={2825--2830},
 year={2011}
}
'''

A		: str								=     '''\
Mean Squared Error(MSE) is the average of the square of difference between the predicted
and actual values.
'''


A		: Any								=     '''
Args:
    predictions: array-like of shape (n_samples,) or (n_samples, n_outputs)
        Estimated target values.
    references: array-like of shape (n_samples,) or (n_samples, n_outputs)
        Ground truth (correct) target values.
    sample_weight: array-like of shape (n_samples,), default=None
        Sample weights.
    multioutput: {"raw_values", "uniform_average"} or array-like of shape (n_outputs,), default="uniform_average"
        Defines aggregating of multiple output values. Array-like value defines weights used to average errors.

        "raw_values" : Returns a full set of errors in case of multioutput input.

        "uniform_average" : Errors of all outputs are averaged with uniform weight.

    squared : bool, default=True
        If True returns MSE value, if False returns RMSE (Root Mean Squared Error) value.

Returns:
    mse : mean squared error.
Examples:

    >>> mse_metric = datasets.load_metric("mse")
    >>> predictions = [2.5, 0.0, 2, 8]
    >>> references = [3, -0.5, 2, 7]
    >>> results = mse_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'mse\': 0.375}
    >>> rmse_result = mse_metric.compute(predictions=predictions, references=references, squared=False)
    >>> print(rmse_result)
    {\'mse\': 0.6123724356957945}

    If you\'re using multi-dimensional lists, then set the config as follows :

    >>> mse_metric = datasets.load_metric("mse", "multilist")
    >>> predictions = [[0.5, 1], [-1, 1], [7, -6]]
    >>> references = [[0, 2], [-1, 2], [8, -5]]
    >>> results = mse_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'mse\': 0.7083333333333334}
    >>> results = mse_metric.compute(predictions=predictions, references=references, multioutput=\'raw_values\')
    >>> print(results) # doctest: +NORMALIZE_WHITESPACE
    {\'mse\': array([0.41666667, 1. ])}
'''



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION					,			_KWARGS_DESCRIPTION    )
class    a_ (			datasets.Metric    ):






			def    UpperCamelCase_				(      self  ):
									return datasets.MetricInfo(
									    description=_DESCRIPTION				,			citation=_CITATION				,			inputs_description=_KWARGS_DESCRIPTION				,			features=datasets.Features(self._get_feature_types()  )				,			reference_urls=[
									        """https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html"""
									    ]				,			)






			def    UpperCamelCase_				(      self  ):
									if self.config_name == "multilist":
															return {
															    "predictions": datasets.Sequence(datasets.Value("""float"""  )  ),
															    "references": datasets.Sequence(datasets.Value("""float"""  )  ),
															}
									else:
															return {
															    "predictions": datasets.Value("""float"""  ),
															    "references": datasets.Value("""float"""  ),
															}






			def    UpperCamelCase_				(      self				,			__UpperCamelCase				,			__UpperCamelCase				,			__UpperCamelCase=None				,			__UpperCamelCase="uniform_average"				,			__UpperCamelCase=True  ):
									_lowercase					       =					mean_squared_error(
									    __UpperCamelCase				,			__UpperCamelCase				,			sample_weight=__UpperCamelCase				,			multioutput=__UpperCamelCase				,			squared=__UpperCamelCase  )

									return {"mse": mse}