





"""simple docstring"""




import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class  _a						(      _lowerCAmelCase	,				unittest.TestCase				):
				UpperCamelCase								=    RealmTokenizer
				UpperCamelCase								=    RealmTokenizerFast
				UpperCamelCase								=    True
				UpperCamelCase								=    True
				UpperCamelCase								=    filter_non_english






				def 				snake_case	( self		:    str							)	->      Any:




										'''simple docstring'''



										super().setUp()

										_UpperCamelCase			:					List[str]     =    [
										    '''[UNK]''',
										    '''[CLS]''',
										    '''[SEP]''',
										    '''[PAD]''',
										    '''[MASK]''',
										    '''want''',
										    '''##want''',
										    '''##ed''',
										    '''wa''',
										    '''un''',
										    '''runn''',
										    '''##ing''',
										    ''',''',
										    '''low''',
										    '''lowest''',
										]
										_UpperCamelCase			:					Tuple     =    os.path.join(self.tmpdirname, VOCAB_FILES_NAMES['''vocab_file''']							)
										with open(self.vocab_file, '''w''', encoding='''utf-8'''							) as vocab_writer:
																vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)






				def 				snake_case	( self		:    Any, lowerCAmelCase__		:    List[str]							)	->      int:




										'''simple docstring'''



										_UpperCamelCase			:					List[str]     =    '''UNwant\u00E9d,running'''
										_UpperCamelCase			:					List[str]     =    '''unwanted, running'''
										return input_text, output_text






				def 				snake_case	( self		:    Any							)	->      int:




										'''simple docstring'''



										_UpperCamelCase			:					Optional[int]     =    self.tokenizer_class(self.vocab_file							)

										_UpperCamelCase			:					Optional[Any]     =    tokenizer.tokenize('''UNwant\u00E9d,running'''							)
										self.assertListEqual(lowerCAmelCase__, ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']							)
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__							), [9, 6, 7, 1_2, 1_0, 1_1]							)






				def 				snake_case	( self		:    Dict							)	->      Optional[int]:




										'''simple docstring'''



										if not self.test_rust_tokenizer:
																return

										_UpperCamelCase			:					Dict     =    self.get_tokenizer()
										_UpperCamelCase			:					Tuple     =    self.get_rust_tokenizer()

										_UpperCamelCase			:					int     =    '''UNwant\u00E9d,running'''

										_UpperCamelCase			:					Any     =    tokenizer.tokenize(lowerCAmelCase__							)
										_UpperCamelCase			:					Dict     =    rust_tokenizer.tokenize(lowerCAmelCase__							)
										self.assertListEqual(lowerCAmelCase__, lowerCAmelCase__							)

										_UpperCamelCase			:					List[str]     =    tokenizer.encode(lowerCAmelCase__, add_special_tokens=lowerCAmelCase__							)
										_UpperCamelCase			:					Dict     =    rust_tokenizer.encode(lowerCAmelCase__, add_special_tokens=lowerCAmelCase__							)
										self.assertListEqual(lowerCAmelCase__, lowerCAmelCase__							)

										_UpperCamelCase			:					Optional[Any]     =    self.get_rust_tokenizer()
										_UpperCamelCase			:					List[Any]     =    tokenizer.encode(lowerCAmelCase__							)
										_UpperCamelCase			:					Any     =    rust_tokenizer.encode(lowerCAmelCase__							)
										self.assertListEqual(lowerCAmelCase__, lowerCAmelCase__							)

										# With lower casing
										_UpperCamelCase			:					Any     =    self.get_tokenizer(do_lower_case=lowerCAmelCase__							)
										_UpperCamelCase			:					int     =    self.get_rust_tokenizer(do_lower_case=lowerCAmelCase__							)

										_UpperCamelCase			:					int     =    '''UNwant\u00E9d,running'''

										_UpperCamelCase			:					str     =    tokenizer.tokenize(lowerCAmelCase__							)
										_UpperCamelCase			:					Optional[Any]     =    rust_tokenizer.tokenize(lowerCAmelCase__							)
										self.assertListEqual(lowerCAmelCase__, lowerCAmelCase__							)

										_UpperCamelCase			:					Optional[int]     =    tokenizer.encode(lowerCAmelCase__, add_special_tokens=lowerCAmelCase__							)
										_UpperCamelCase			:					Optional[int]     =    rust_tokenizer.encode(lowerCAmelCase__, add_special_tokens=lowerCAmelCase__							)
										self.assertListEqual(lowerCAmelCase__, lowerCAmelCase__							)

										_UpperCamelCase			:					Tuple     =    self.get_rust_tokenizer()
										_UpperCamelCase			:					Optional[Any]     =    tokenizer.encode(lowerCAmelCase__							)
										_UpperCamelCase			:					str     =    rust_tokenizer.encode(lowerCAmelCase__							)
										self.assertListEqual(lowerCAmelCase__, lowerCAmelCase__							)






				def 				snake_case	( self		:    Optional[int]							)	->      List[Any]:




										'''simple docstring'''



										_UpperCamelCase			:					Tuple     =    BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''							), ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']							)






				def 				snake_case	( self		:    Tuple							)	->      Optional[Any]:




										'''simple docstring'''



										_UpperCamelCase			:					List[str]     =    BasicTokenizer(do_lower_case=lowerCAmelCase__							)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							), ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							), ['''hello''']							)






				def 				snake_case	( self		:    Any							)	->      Optional[Any]:




										'''simple docstring'''



										_UpperCamelCase			:					List[Any]     =    BasicTokenizer(do_lower_case=lowerCAmelCase__, strip_accents=lowerCAmelCase__							)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							), ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							), ['''h\u00E9llo''']							)






				def 				snake_case	( self		:    List[str]							)	->      List[Any]:




										'''simple docstring'''



										_UpperCamelCase			:					int     =    BasicTokenizer(do_lower_case=lowerCAmelCase__, strip_accents=lowerCAmelCase__							)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							), ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							), ['''hello''']							)






				def 				snake_case	( self		:    Tuple							)	->      List[Any]:




										'''simple docstring'''



										_UpperCamelCase			:					Optional[int]     =    BasicTokenizer(do_lower_case=lowerCAmelCase__							)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							), ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							), ['''hello''']							)






				def 				snake_case	( self		:    List[str]							)	->      Dict:




										'''simple docstring'''



										_UpperCamelCase			:					Dict     =    BasicTokenizer(do_lower_case=lowerCAmelCase__							)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							), ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)






				def 				snake_case	( self		:    List[str]							)	->      Dict:




										'''simple docstring'''



										_UpperCamelCase			:					str     =    BasicTokenizer(do_lower_case=lowerCAmelCase__, strip_accents=lowerCAmelCase__							)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							), ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)






				def 				snake_case	( self		:    str							)	->      List[Any]:




										'''simple docstring'''



										_UpperCamelCase			:					Optional[Any]     =    BasicTokenizer(do_lower_case=lowerCAmelCase__, strip_accents=lowerCAmelCase__							)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							), ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)






				def 				snake_case	( self		:    int							)	->      Optional[int]:




										'''simple docstring'''



										_UpperCamelCase			:					Union[str, Any]     =    BasicTokenizer(do_lower_case=lowerCAmelCase__, never_split=['''[UNK]''']							)

										self.assertListEqual(
										    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''							), ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']							)






				def 				snake_case	( self		:    List[str]							)	->      Dict:




										'''simple docstring'''



										_UpperCamelCase			:					Dict     =    ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

										_UpperCamelCase			:					Any     =    {}
										for i, token in enumerate(lowerCAmelCase__							):
																_UpperCamelCase			:					Union[str, Any]     =    i
										_UpperCamelCase			:					Optional[int]     =    WordpieceTokenizer(vocab=lowerCAmelCase__, unk_token='''[UNK]'''							)

										self.assertListEqual(tokenizer.tokenize(''''''							), []							)

										self.assertListEqual(tokenizer.tokenize('''unwanted running'''							), ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']							)

										self.assertListEqual(tokenizer.tokenize('''unwantedX running'''							), ['''[UNK]''', '''runn''', '''##ing''']							)






				def 				snake_case	( self		:    Tuple							)	->      List[str]:




										'''simple docstring'''



										self.assertTrue(_is_whitespace(''' '''							)							)
										self.assertTrue(_is_whitespace('''\t'''							)							)
										self.assertTrue(_is_whitespace('''\r'''							)							)
										self.assertTrue(_is_whitespace('''\n'''							)							)
										self.assertTrue(_is_whitespace('''\u00A0'''							)							)

										self.assertFalse(_is_whitespace('''A'''							)							)
										self.assertFalse(_is_whitespace('''-'''							)							)






				def 				snake_case	( self		:    Tuple							)	->      Tuple:




										'''simple docstring'''



										self.assertTrue(_is_control('''\u0005'''							)							)

										self.assertFalse(_is_control('''A'''							)							)
										self.assertFalse(_is_control(''' '''							)							)
										self.assertFalse(_is_control('''\t'''							)							)
										self.assertFalse(_is_control('''\r'''							)							)






				def 				snake_case	( self		:    int							)	->      Dict:




										'''simple docstring'''



										self.assertTrue(_is_punctuation('''-'''							)							)
										self.assertTrue(_is_punctuation('''$'''							)							)
										self.assertTrue(_is_punctuation('''`'''							)							)
										self.assertTrue(_is_punctuation('''.'''							)							)

										self.assertFalse(_is_punctuation('''A'''							)							)
										self.assertFalse(_is_punctuation(''' '''							)							)






				def 				snake_case	( self		:    Optional[int]							)	->      Optional[Any]:




										'''simple docstring'''



										_UpperCamelCase			:					Optional[Any]     =    self.get_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(lowerCAmelCase__							) for t in ['''Test''', '''\xad''', '''test''']], [['''[UNK]'''], [], ['''[UNK]''']]							)

										if self.test_rust_tokenizer:
																_UpperCamelCase			:					str     =    self.get_rust_tokenizer()
																self.assertListEqual(
																    [rust_tokenizer.tokenize(lowerCAmelCase__							) for t in ['''Test''', '''\xad''', '''test''']], [['''[UNK]'''], [], ['''[UNK]''']]							)






				@slow
				def 				snake_case	( self		:    Any							)	->      List[str]:




										'''simple docstring'''



										_UpperCamelCase			:					int     =    self.tokenizer_class.from_pretrained('''bert-base-uncased'''							)

										_UpperCamelCase			:					Any     =    tokenizer.encode('''sequence builders''', add_special_tokens=lowerCAmelCase__							)
										_UpperCamelCase			:					Union[str, Any]     =    tokenizer.encode('''multi-sequence build''', add_special_tokens=lowerCAmelCase__							)

										_UpperCamelCase			:					Tuple     =    tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__							)
										_UpperCamelCase			:					Any     =    tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__, lowerCAmelCase__							)

										assert encoded_sentence == [1_0_1] + text + [1_0_2]
										assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]






				def 				snake_case	( self		:    Union[str, Any]							)	->      Union[str, Any]:




										'''simple docstring'''



										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																						_UpperCamelCase			:					Any     =    self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__, **lowerCAmelCase__							)

																						_UpperCamelCase			:					Tuple     =    f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																						_UpperCamelCase			:					str     =    tokenizer_r.encode_plus(
																						    lowerCAmelCase__, return_attention_mask=lowerCAmelCase__, return_token_type_ids=lowerCAmelCase__, return_offsets_mapping=lowerCAmelCase__, add_special_tokens=lowerCAmelCase__, )

																						_UpperCamelCase			:					List[str]     =    tokenizer_r.do_lower_case if hasattr(lowerCAmelCase__, '''do_lower_case'''							) else False
																						_UpperCamelCase			:					str     =    (
																						    [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), '''A'''),
																						        ((1, 2), ''','''),
																						        ((3, 5), '''na'''),
																						        ((5, 6), '''##ï'''),
																						        ((6, 8), '''##ve'''),
																						        ((9, 1_5), tokenizer_r.mask_token),
																						        ((1_6, 2_1), '''Allen'''),
																						        ((2_1, 2_3), '''##NL'''),
																						        ((2_3, 2_4), '''##P'''),
																						        ((2_5, 3_3), '''sentence'''),
																						        ((3_3, 3_4), '''.'''),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						    if not do_lower_case
																						    else [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), '''a'''),
																						        ((1, 2), ''','''),
																						        ((3, 8), '''naive'''),
																						        ((9, 1_5), tokenizer_r.mask_token),
																						        ((1_6, 2_1), '''allen'''),
																						        ((2_1, 2_3), '''##nl'''),
																						        ((2_3, 2_4), '''##p'''),
																						        ((2_5, 3_3), '''sentence'''),
																						        ((3_3, 3_4), '''.'''),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						)

																						self.assertEqual(
																						    [e[1] for e in expected_results], tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']							)							)
																						self.assertEqual([e[0] for e in expected_results], tokens['''offset_mapping''']							)






				@slow
				def 				snake_case	( self		:    Union[str, Any]							)	->      Optional[int]:




										'''simple docstring'''



										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
																						_UpperCamelCase			:					str     =    self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__, **lowerCAmelCase__							)
																						_UpperCamelCase			:					Union[str, Any]     =    self.tokenizer_class.from_pretrained(lowerCAmelCase__, **lowerCAmelCase__							)
																						_UpperCamelCase			:					Any     =    [['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]

																						_UpperCamelCase			:					Dict     =    tokenizer_r.batch_encode_candidates(lowerCAmelCase__, max_length=1_0, return_tensors='''np'''							)
																						_UpperCamelCase			:					Any     =    tokenizer_p.batch_encode_candidates(lowerCAmelCase__, max_length=1_0, return_tensors='''np'''							)

																						_UpperCamelCase			:					Optional[int]     =    (2, 2, 1_0)

																						self.assertEqual(encoded_sentence_r['''input_ids'''].shape, lowerCAmelCase__							)
																						self.assertEqual(encoded_sentence_r['''attention_mask'''].shape, lowerCAmelCase__							)
																						self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape, lowerCAmelCase__							)

																						self.assertEqual(encoded_sentence_p['''input_ids'''].shape, lowerCAmelCase__							)
																						self.assertEqual(encoded_sentence_p['''attention_mask'''].shape, lowerCAmelCase__							)
																						self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape, lowerCAmelCase__							)










"""simple docstring"""




from sklearn.metrics import mean_absolute_error

import datasets


UpperCamelCase_           ="""\
@article{scikit-learn,
 title={Scikit-learn: Machine Learning in {P}ython},
 author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.
         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.
         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and
         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},
 journal={Journal of Machine Learning Research},
 volume={12},
 pages={2825--2830},
 year={2011}
}
"""

UpperCamelCase_           ="""\
Mean Absolute Error (MAE) is the mean of the magnitude of difference between the predicted and actual
values.
"""


UpperCamelCase_           ="""
Args:
    predictions: array-like of shape (n_samples,) or (n_samples, n_outputs)
        Estimated target values.
    references: array-like of shape (n_samples,) or (n_samples, n_outputs)
        Ground truth (correct) target values.
    sample_weight: array-like of shape (n_samples,), default=None
        Sample weights.
    multioutput: {\"raw_values\", \"uniform_average\"} or array-like of shape (n_outputs,), default=\"uniform_average\"
        Defines aggregating of multiple output values. Array-like value defines weights used to average errors.

                 \"raw_values\" : Returns a full set of errors in case of multioutput input.

                 \"uniform_average\" : Errors of all outputs are averaged with uniform weight.

Returns:
    mae : mean absolute error.
        If multioutput is \"raw_values\", then mean absolute error is returned for each output separately. If multioutput is \"uniform_average\" or an ndarray of weights, then the weighted average of all output errors is returned.
        MAE output is non-negative floating point. The best value is 0.0.
Examples:

    >>> mae_metric = datasets.load_metric(\"mae\")
    >>> predictions = [2.5, 0.0, 2, 8]
    >>> references = [3, -0.5, 2, 7]
    >>> results = mae_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'mae': 0.5}

    If you're using multi-dimensional lists, then set the config as follows :

    >>> mae_metric = datasets.load_metric(\"mae\", \"multilist\")
    >>> predictions = [[0.5, 1], [-1, 1], [7, -6]]
    >>> references = [[0, 2], [-1, 2], [8, -5]]
    >>> results = mae_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'mae': 0.75}
    >>> results = mae_metric.compute(predictions=predictions, references=references, multioutput='raw_values')
    >>> print(results)
    {'mae': array([0.5, 1. ])}
"""




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION	,				_KWARGS_DESCRIPTION				)
class  _a						(      datasets.Metric				):






				def 				snake_case	( self		:    Optional[int]							)	->      List[Any]:




										'''simple docstring'''



										return datasets.MetricInfo(
										    description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features(self._get_feature_types()							), reference_urls=[
										        '''https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_absolute_error.html'''
										    ], )






				def 				snake_case	( self		:    Dict							)	->      Union[str, Any]:




										'''simple docstring'''



										if self.config_name == "multilist":
																return {
																    "predictions": datasets.Sequence(datasets.Value('''float'''							)							),
																    "references": datasets.Sequence(datasets.Value('''float'''							)							),
																}
										else:
																return {
																    "predictions": datasets.Value('''float'''							),
																    "references": datasets.Value('''float'''							),
																}






				def 				snake_case	( self		:    List[Any], lowerCAmelCase__		:    Dict, lowerCAmelCase__		:    Union[str, Any], lowerCAmelCase__		:    Tuple=None, lowerCAmelCase__		:    Tuple="uniform_average"							)	->      Optional[int]:




										'''simple docstring'''



										_UpperCamelCase			:					List[str]     =    mean_absolute_error(lowerCAmelCase__, lowerCAmelCase__, sample_weight=lowerCAmelCase__, multioutput=lowerCAmelCase__							)

										return {"mae": mae_score}



