
"""simple docstring"""


import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class  __lowerCamelCase  ( lowerCAmelCase						,	unittest.TestCase			):
				a__:      Union[str, Any]          =    RobertaTokenizer
				a__:      Any          =    RobertaTokenizerFast
				a__:      int          =    True
				a__:      List[str]          =    {'cls_token': '<s>'}




				def        UpperCAmelCase__						(				self   ):
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							lowerCamelCase_					=  [
							    '''l''',
							    '''o''',
							    '''w''',
							    '''e''',
							    '''r''',
							    '''s''',
							    '''t''',
							    '''i''',
							    '''d''',
							    '''n''',
							    '''\u0120''',
							    '''\u0120l''',
							    '''\u0120n''',
							    '''\u0120lo''',
							    '''\u0120low''',
							    '''er''',
							    '''\u0120lowest''',
							    '''\u0120newer''',
							    '''\u0120wider''',
							    '''<unk>''',
							]
							lowerCamelCase_					=  dict(zip(UpperCAmelCase							,						range(len(UpperCAmelCase   )   )   )   )
							lowerCamelCase_					=  ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
							lowerCamelCase_					=  {'''unk_token''': '''<unk>'''}

							lowerCamelCase_					=  os.path.join(self.tmpdirname							,						VOCAB_FILES_NAMES['''vocab_file''']   )
							lowerCamelCase_					=  os.path.join(self.tmpdirname							,						VOCAB_FILES_NAMES['''merges_file''']   )
							with open(self.vocab_file							,						'''w'''							,						encoding='''utf-8'''   ) as fp:
										fp.write(json.dumps(UpperCAmelCase   ) + '''\n'''   )
							with open(self.merges_file							,						'''w'''							,						encoding='''utf-8'''   ) as fp:
										fp.write('''\n'''.join(UpperCAmelCase   )   )




				def        UpperCAmelCase__						(				self							,						**UpperCAmelCase   ):
							kwargs.update(self.special_tokens_map   )
							return self.tokenizer_class.from_pretrained(self.tmpdirname							,						**UpperCAmelCase   )




				def        UpperCAmelCase__						(				self							,						**UpperCAmelCase   ):
							kwargs.update(self.special_tokens_map   )
							return RobertaTokenizerFast.from_pretrained(self.tmpdirname							,						**UpperCAmelCase   )




				def        UpperCAmelCase__						(				self							,						UpperCAmelCase   ):
							lowerCamelCase_					=  '''lower newer'''
							lowerCamelCase_					=  '''lower newer'''
							return input_text, output_text




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  self.tokenizer_class(self.vocab_file							,						self.merges_file							,						**self.special_tokens_map   )
							lowerCamelCase_					=  '''lower newer'''
							lowerCamelCase_					=  ['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
							lowerCamelCase_					=  tokenizer.tokenize(UpperCAmelCase   )  # , add_prefix_space=True)
							self.assertListEqual(UpperCAmelCase							,						UpperCAmelCase   )

							lowerCamelCase_					=  tokens + [tokenizer.unk_token]
							lowerCamelCase_					=  [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase   )							,						UpperCAmelCase   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  self.get_tokenizer()

							self.assertListEqual(tokenizer.encode('''Hello world!'''							,						add_special_tokens=UpperCAmelCase   )							,						[0, 3_1414, 232, 328, 2]   )
							self.assertListEqual(
							    tokenizer.encode('''Hello world! cécé herlolip 418'''							,						add_special_tokens=UpperCAmelCase   )							,						[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]							,						)




				@slow
				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  self.tokenizer_class.from_pretrained('''roberta-base'''   )

							lowerCamelCase_					=  tokenizer.encode('''sequence builders'''							,						add_special_tokens=UpperCAmelCase   )
							lowerCamelCase_					=  tokenizer.encode('''multi-sequence build'''							,						add_special_tokens=UpperCAmelCase   )

							lowerCamelCase_					=  tokenizer.encode(
							    '''sequence builders'''							,						add_special_tokens=UpperCAmelCase							,						add_prefix_space=UpperCAmelCase   )
							lowerCamelCase_					=  tokenizer.encode(
							    '''sequence builders'''							,						'''multi-sequence build'''							,						add_special_tokens=UpperCAmelCase							,						add_prefix_space=UpperCAmelCase   )

							lowerCamelCase_					=  tokenizer.build_inputs_with_special_tokens(UpperCAmelCase   )
							lowerCamelCase_					=  tokenizer.build_inputs_with_special_tokens(UpperCAmelCase							,						UpperCAmelCase   )

							assert encoded_sentence == encoded_text_from_decode
							assert encoded_pair == encoded_pair_from_decode




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  self.get_tokenizer()

							lowerCamelCase_					=  '''Encode this sequence.'''
							lowerCamelCase_					=  tokenizer.byte_encoder[''' '''.encode('''utf-8'''   )[0]]

							# Testing encoder arguments
							lowerCamelCase_					=  tokenizer.encode(UpperCAmelCase							,						add_special_tokens=UpperCAmelCase							,						add_prefix_space=UpperCAmelCase   )
							lowerCamelCase_					=  tokenizer.convert_ids_to_tokens(encoded[0]   )[0]
							self.assertNotEqual(UpperCAmelCase							,						UpperCAmelCase   )

							lowerCamelCase_					=  tokenizer.encode(UpperCAmelCase							,						add_special_tokens=UpperCAmelCase							,						add_prefix_space=UpperCAmelCase   )
							lowerCamelCase_					=  tokenizer.convert_ids_to_tokens(encoded[0]   )[0]
							self.assertEqual(UpperCAmelCase							,						UpperCAmelCase   )

							tokenizer.add_special_tokens({'''bos_token''': '''<s>'''}   )
							lowerCamelCase_					=  tokenizer.encode(UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )
							lowerCamelCase_					=  tokenizer.convert_ids_to_tokens(encoded[1]   )[0]
							self.assertNotEqual(UpperCAmelCase							,						UpperCAmelCase   )

							# Testing spaces after special tokens
							lowerCamelCase_					=  '''<mask>'''
							tokenizer.add_special_tokens(
							    {'''mask_token''': AddedToken(UpperCAmelCase							,						lstrip=UpperCAmelCase							,						rstrip=UpperCAmelCase   )}   )  # mask token has a left space
							lowerCamelCase_					=  tokenizer.convert_tokens_to_ids(UpperCAmelCase   )

							lowerCamelCase_					=  '''Encode <mask> sequence'''
							lowerCamelCase_					=  '''Encode <mask>sequence'''

							lowerCamelCase_					=  tokenizer.encode(UpperCAmelCase   )
							lowerCamelCase_					=  encoded.index(UpperCAmelCase   )
							lowerCamelCase_					=  tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]   )[0]
							self.assertEqual(UpperCAmelCase							,						UpperCAmelCase   )

							lowerCamelCase_					=  tokenizer.encode(UpperCAmelCase   )
							lowerCamelCase_					=  encoded.index(UpperCAmelCase   )
							lowerCamelCase_					=  tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]   )[0]
							self.assertNotEqual(UpperCAmelCase							,						UpperCAmelCase   )




				def        UpperCAmelCase__						(				self   ):
							pass




				def        UpperCAmelCase__						(				self   ):
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
													lowerCamelCase_					=  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase							,						**UpperCAmelCase   )
													lowerCamelCase_					=  self.tokenizer_class.from_pretrained(UpperCAmelCase							,						**UpperCAmelCase   )
													lowerCamelCase_					=  '''A, <mask> AllenNLP sentence.'''
													lowerCamelCase_					=  tokenizer_r.encode_plus(UpperCAmelCase							,						add_special_tokens=UpperCAmelCase							,						return_token_type_ids=UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer_p.encode_plus(UpperCAmelCase							,						add_special_tokens=UpperCAmelCase							,						return_token_type_ids=UpperCAmelCase   )

													# token_type_ids should put 0 everywhere
													self.assertEqual(sum(tokens_r['''token_type_ids''']   )							,						sum(tokens_p['''token_type_ids''']   )   )

													# attention_mask should put 1 everywhere, so sum over length should be 1
													self.assertEqual(
													    sum(tokens_r['''attention_mask''']   ) / len(tokens_r['''attention_mask''']   )							,						sum(tokens_p['''attention_mask''']   ) / len(tokens_p['''attention_mask''']   )							,						)

													lowerCamelCase_					=  tokenizer_r.convert_ids_to_tokens(tokens_r['''input_ids''']   )
													lowerCamelCase_					=  tokenizer_p.convert_ids_to_tokens(tokens_p['''input_ids''']   )

													# Rust correctly handles the space before the mask while python doesnt
													self.assertSequenceEqual(tokens_p['''input_ids''']							,						[0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]   )
													self.assertSequenceEqual(tokens_r['''input_ids''']							,						[0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]   )

													self.assertSequenceEqual(
													    UpperCAmelCase							,						['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']   )
													self.assertSequenceEqual(
													    UpperCAmelCase							,						['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']   )




				def        UpperCAmelCase__						(				self   ):
							for trim_offsets, add_prefix_space in itertools.product([True, False]							,						repeat=2   ):
										lowerCamelCase_					=  self.rust_tokenizer_class.from_pretrained(
										    self.tmpdirname							,						use_fast=UpperCAmelCase							,						add_prefix_space=UpperCAmelCase							,						trim_offsets=UpperCAmelCase   )

										lowerCamelCase_					=  json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()   )
										lowerCamelCase_					=  json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()   )

										self.assertEqual(pre_tokenizer_state['''add_prefix_space''']							,						UpperCAmelCase   )

										self.assertEqual(post_processor_state['''add_prefix_space''']							,						UpperCAmelCase   )
										self.assertEqual(post_processor_state['''trim_offsets''']							,						UpperCAmelCase   )





				def        UpperCAmelCase__						(				self   ):
							# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
							# `trim_offsets`
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
													lowerCamelCase_					=  '''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
													lowerCamelCase_					=  f"{text_of_1_token} {text_of_1_token}"

													lowerCamelCase_					=  self.rust_tokenizer_class.from_pretrained(
													    UpperCAmelCase							,						use_fast=UpperCAmelCase							,						add_prefix_space=UpperCAmelCase							,						trim_offsets=UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer_r(UpperCAmelCase							,						return_offsets_mapping=UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )
													self.assertEqual(encoding.offset_mapping[0]							,						(0, len(UpperCAmelCase   ))   )
													self.assertEqual(
													    encoding.offset_mapping[1]							,						(len(UpperCAmelCase   ) + 1, len(UpperCAmelCase   ) + 1 + len(UpperCAmelCase   ))							,						)

													lowerCamelCase_					=  self.rust_tokenizer_class.from_pretrained(
													    UpperCAmelCase							,						use_fast=UpperCAmelCase							,						add_prefix_space=UpperCAmelCase							,						trim_offsets=UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer_r(UpperCAmelCase							,						return_offsets_mapping=UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )
													self.assertEqual(encoding.offset_mapping[0]							,						(0, len(UpperCAmelCase   ))   )
													self.assertEqual(
													    encoding.offset_mapping[1]							,						(len(UpperCAmelCase   ) + 1, len(UpperCAmelCase   ) + 1 + len(UpperCAmelCase   ))							,						)

													lowerCamelCase_					=  self.rust_tokenizer_class.from_pretrained(
													    UpperCAmelCase							,						use_fast=UpperCAmelCase							,						add_prefix_space=UpperCAmelCase							,						trim_offsets=UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer_r(UpperCAmelCase							,						return_offsets_mapping=UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )
													self.assertEqual(encoding.offset_mapping[0]							,						(0, len(UpperCAmelCase   ))   )
													self.assertEqual(
													    encoding.offset_mapping[1]							,						(len(UpperCAmelCase   ), len(UpperCAmelCase   ) + 1 + len(UpperCAmelCase   ))							,						)

													lowerCamelCase_					=  self.rust_tokenizer_class.from_pretrained(
													    UpperCAmelCase							,						use_fast=UpperCAmelCase							,						add_prefix_space=UpperCAmelCase							,						trim_offsets=UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer_r(UpperCAmelCase							,						return_offsets_mapping=UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )
													self.assertEqual(encoding.offset_mapping[0]							,						(0, len(UpperCAmelCase   ))   )
													self.assertEqual(
													    encoding.offset_mapping[1]							,						(len(UpperCAmelCase   ), len(UpperCAmelCase   ) + 1 + len(UpperCAmelCase   ))							,						)

													lowerCamelCase_					=  f" {text}"

													# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
													#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
													# )
													# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
													# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
													# self.assertEqual(
													#     encoding.offset_mapping[1],
													#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
													# )

													lowerCamelCase_					=  self.rust_tokenizer_class.from_pretrained(
													    UpperCAmelCase							,						use_fast=UpperCAmelCase							,						add_prefix_space=UpperCAmelCase							,						trim_offsets=UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer_r(UpperCAmelCase							,						return_offsets_mapping=UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )
													self.assertEqual(encoding.offset_mapping[0]							,						(1, 1 + len(UpperCAmelCase   ))   )
													self.assertEqual(
													    encoding.offset_mapping[1]							,						(1 + len(UpperCAmelCase   ) + 1, 1 + len(UpperCAmelCase   ) + 1 + len(UpperCAmelCase   ))							,						)

													lowerCamelCase_					=  self.rust_tokenizer_class.from_pretrained(
													    UpperCAmelCase							,						use_fast=UpperCAmelCase							,						add_prefix_space=UpperCAmelCase							,						trim_offsets=UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer_r(UpperCAmelCase							,						return_offsets_mapping=UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )
													self.assertEqual(encoding.offset_mapping[0]							,						(0, 1 + len(UpperCAmelCase   ))   )
													self.assertEqual(
													    encoding.offset_mapping[1]							,						(1 + len(UpperCAmelCase   ), 1 + len(UpperCAmelCase   ) + 1 + len(UpperCAmelCase   ))							,						)

													lowerCamelCase_					=  self.rust_tokenizer_class.from_pretrained(
													    UpperCAmelCase							,						use_fast=UpperCAmelCase							,						add_prefix_space=UpperCAmelCase							,						trim_offsets=UpperCAmelCase   )
													lowerCamelCase_					=  tokenizer_r(UpperCAmelCase							,						return_offsets_mapping=UpperCAmelCase							,						add_special_tokens=UpperCAmelCase   )
													self.assertEqual(encoding.offset_mapping[0]							,						(0, 1 + len(UpperCAmelCase   ))   )
													self.assertEqual(
													    encoding.offset_mapping[1]							,						(1 + len(UpperCAmelCase   ), 1 + len(UpperCAmelCase   ) + 1 + len(UpperCAmelCase   ))							,						)







import math
import time

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
 import torch_xla.core.xla_model as xm
 import torch_xla.debug.metrics as met






class 	__lowercase					(	__snake_case	):





      def __init__(		self    :       Dict       ,							*__lowerCamelCase    :       Dict       ,							__lowerCamelCase    :       Union[str, Any]=None       ,							__lowerCamelCase    :       str=None       ,							**__lowerCamelCase    :       Optional[int]							)     -> Optional[int]:






         """simple docstring"""







         super().__init__(*__lowerCamelCase       ,							**__lowerCamelCase							)
         UpperCAmelCase      					=  eval_examples
         UpperCAmelCase      					=  post_process_function





      def 							_lowercase					(		self    :       Any       ,							__lowerCamelCase    :       int=None       ,							__lowerCamelCase    :       int=None       ,							__lowerCamelCase    :       Tuple=None       ,							__lowerCamelCase    :       str = "eval"							)     -> List[str]:






         """simple docstring"""







         UpperCAmelCase      					=  self.eval_dataset if eval_dataset is None else eval_dataset
         UpperCAmelCase      					=  self.get_eval_dataloader(__lowerCamelCase							)
         UpperCAmelCase      					=  self.eval_examples if eval_examples is None else eval_examples

         # Temporarily disable metric computation, we will do it in the loop here.
         UpperCAmelCase      					=  self.compute_metrics
         UpperCAmelCase      					=  None
         UpperCAmelCase      					=  self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
         UpperCAmelCase      					=  time.time()
         try:
            UpperCAmelCase      					=  eval_loop(
                __lowerCamelCase       ,							description="""Evaluation"""       ,							prediction_loss_only=True if compute_metrics is None else None       ,							ignore_keys=__lowerCamelCase       ,							metric_key_prefix=__lowerCamelCase       ,							)
         finally:
            UpperCAmelCase      					=  compute_metrics
         UpperCAmelCase      					=  self.args.eval_batch_size * self.args.world_size
         if F"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
            start_time += output.metrics[F"""{metric_key_prefix}_jit_compilation_time"""]
         output.metrics.update(
             speed_metrics(
                 __lowerCamelCase       ,							__lowerCamelCase       ,							num_samples=output.num_samples       ,							num_steps=math.ceil(output.num_samples / total_batch_size							)       ,							)							)
         if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
            # Only the main node write the results by default
            UpperCAmelCase      					=  self.post_process_function(__lowerCamelCase       ,							__lowerCamelCase       ,							output.predictions							)
            UpperCAmelCase      					=  self.compute_metrics(__lowerCamelCase							)

            # Prefix all keys with metric_key_prefix + '_'
            for key in list(metrics.keys()							):
               if not key.startswith(F"""{metric_key_prefix}_"""							):
                  UpperCAmelCase      					=  metrics.pop(__lowerCamelCase							)
            metrics.update(output.metrics							)
         else:
            UpperCAmelCase      					=  output.metrics

         if self.args.should_log:
            # Only the main node log the results by default
            self.log(__lowerCamelCase							)

         if self.args.tpu_metrics_debug or self.args.debug:
            # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
            xm.master_print(met.metrics_report()							)

         UpperCAmelCase      					=  self.callback_handler.on_evaluate(self.args       ,							self.state       ,							self.control       ,							__lowerCamelCase							)
         return metrics





      def 							_lowercase					(		self    :       Optional[Any]       ,							__lowerCamelCase    :       Union[str, Any]       ,							__lowerCamelCase    :       Any       ,							__lowerCamelCase    :       Dict=None       ,							__lowerCamelCase    :       str = "test"							)     -> Dict:






         """simple docstring"""







         UpperCAmelCase      					=  self.get_test_dataloader(__lowerCamelCase							)

         # Temporarily disable metric computation, we will do it in the loop here.
         UpperCAmelCase      					=  self.compute_metrics
         UpperCAmelCase      					=  None
         UpperCAmelCase      					=  self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
         UpperCAmelCase      					=  time.time()
         try:
            UpperCAmelCase      					=  eval_loop(
                __lowerCamelCase       ,							description="""Prediction"""       ,							prediction_loss_only=True if compute_metrics is None else None       ,							ignore_keys=__lowerCamelCase       ,							metric_key_prefix=__lowerCamelCase       ,							)
         finally:
            UpperCAmelCase      					=  compute_metrics
         UpperCAmelCase      					=  self.args.eval_batch_size * self.args.world_size
         if F"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
            start_time += output.metrics[F"""{metric_key_prefix}_jit_compilation_time"""]
         output.metrics.update(
             speed_metrics(
                 __lowerCamelCase       ,							__lowerCamelCase       ,							num_samples=output.num_samples       ,							num_steps=math.ceil(output.num_samples / total_batch_size							)       ,							)							)

         if self.post_process_function is None or self.compute_metrics is None:
            return output

         UpperCAmelCase      					=  self.post_process_function(__lowerCamelCase       ,							__lowerCamelCase       ,							output.predictions       ,							"""predict"""							)
         UpperCAmelCase      					=  self.compute_metrics(__lowerCamelCase							)

         # Prefix all keys with metric_key_prefix + '_'
         for key in list(metrics.keys()							):
            if not key.startswith(F"""{metric_key_prefix}_"""							):
               UpperCAmelCase      					=  metrics.pop(__lowerCamelCase							)
         metrics.update(output.metrics							)
         return PredictionOutput(predictions=predictions.predictions       ,							label_ids=predictions.label_ids       ,							metrics=__lowerCamelCase							)



