


'''simple docstring'''






import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowercase				=get_tests_dir('fixtures/test_sentencepiece.model')






@require_sentencepiece
class    __magic_name__				(						lowerCAmelCase   ,unittest.TestCase							):
       UpperCAmelCase		 =XLMProphetNetTokenizer
       UpperCAmelCase		 =False
       UpperCAmelCase		 =True






       def      lowerCAmelCase						(     self)		->    Dict:




          '''simple docstring'''
          super().setUp()

          # We have a SentencePiece fixture for testing
          _UpperCAmelCase  :  int				       =XLMProphetNetTokenizer(snake_case    ,				keep_accents=snake_case)
          tokenizer.save_pretrained(self.tmpdirname)






       def      lowerCAmelCase						(     self)		->    List[Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  Optional[int]				       ='[PAD]'
          _UpperCAmelCase  :  Any				       =0

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case)    ,				snake_case)
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case)    ,				snake_case)






       def      lowerCAmelCase						(     self)		->    Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  Optional[int]				       =list(self.get_tokenizer().get_vocab().keys())

          self.assertEqual(vocab_keys[0]    ,				'[PAD]')
          self.assertEqual(vocab_keys[1]    ,				'[CLS]')
          self.assertEqual(vocab_keys[-1]    ,				'j')
          self.assertEqual(len(snake_case)    ,				1_0_1_2)






       def      lowerCAmelCase						(     self)		->    List[Any]:




          '''simple docstring'''
          self.assertEqual(self.get_tokenizer().vocab_size    ,				1_0_1_2)






       def      lowerCAmelCase						(     self)		->    str:




          '''simple docstring'''
          _UpperCAmelCase  :  str				       =XLMProphetNetTokenizer(snake_case    ,				keep_accents=snake_case)

          _UpperCAmelCase  :  Any				       =tokenizer.tokenize('This is a test')
          self.assertListEqual(snake_case    ,				['▁This', '▁is', '▁a', '▁t', 'est'])

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(snake_case)    ,				[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]    ,				)

          _UpperCAmelCase  :  List[str]				       =tokenizer.tokenize('I was born in 92000, and this is falsé.')
          self.assertListEqual(
              snake_case    ,				[
                  SPIECE_UNDERLINE + 'I',
                  SPIECE_UNDERLINE + 'was',
                  SPIECE_UNDERLINE + 'b',
                  'or',
                  'n',
                  SPIECE_UNDERLINE + 'in',
                  SPIECE_UNDERLINE + '',
                  '9',
                  '2',
                  '0',
                  '0',
                  '0',
                  ',',
                  SPIECE_UNDERLINE + 'and',
                  SPIECE_UNDERLINE + 'this',
                  SPIECE_UNDERLINE + 'is',
                  SPIECE_UNDERLINE + 'f',
                  'al',
                  's',
                  'é',
                  '.',
              ]    ,				)
          _UpperCAmelCase  :  Tuple				       =tokenizer.convert_tokens_to_ids(snake_case)
          self.assertListEqual(
              snake_case    ,				[
                  value + tokenizer.fairseq_offset
                  for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, -9, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, -9, 4]
              ]    ,				)

          _UpperCAmelCase  :  Any				       =tokenizer.convert_ids_to_tokens(snake_case)
          self.assertListEqual(
              snake_case    ,				[
                  SPIECE_UNDERLINE + 'I',
                  SPIECE_UNDERLINE + 'was',
                  SPIECE_UNDERLINE + 'b',
                  'or',
                  'n',
                  SPIECE_UNDERLINE + 'in',
                  SPIECE_UNDERLINE + '',
                  '[UNK]',
                  '2',
                  '0',
                  '0',
                  '0',
                  ',',
                  SPIECE_UNDERLINE + 'and',
                  SPIECE_UNDERLINE + 'this',
                  SPIECE_UNDERLINE + 'is',
                  SPIECE_UNDERLINE + 'f',
                  'al',
                  's',
                  '[UNK]',
                  '.',
              ]    ,				)






       @cached_property
       def      lowerCAmelCase						(     self)		->    int:




          '''simple docstring'''
          return XLMProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased')






       @slow
       def      lowerCAmelCase						(     self)		->    List[Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  List[str]				       ='Hello World!'
          _UpperCAmelCase  :  List[Any]				       =[3_5_3_8_9, 6_6_7_2, 4_9, 2]
          self.assertListEqual(snake_case    ,				self.big_tokenizer.encode(snake_case))






       @slow
       def      lowerCAmelCase						(     self)		->    Optional[Any]:




          '''simple docstring'''
          # fmt: off
          _UpperCAmelCase  :  List[str]				       ={'input_ids': [[1_1_0_7_3, 8_2_7_8_3, 1_8, 2_6, 8_2_7_8_3, 5_4_9, 5_1_5_4_0, 2_4_8, 1_7_2_0_9, 1_3_0_1, 2_1_7, 2_0, 2_1_5_1_8_6, 1_3_2_5, 1_4_7, 1_7_2_0_9, 1_3_0_1, 2_1_7, 2_0, 5_6_3_7_0, 5_3, 1_2_2_0_2_0, 2_0, 1_6_4_7_7, 2_7, 8_7_3_5_5, 4_5_4_8, 2_0, 4_7_2_8, 7_8_3_9_2, 1_7, 1_5_9_9_6_9, 1_8, 2_6, 2_4_4_9_1, 6_2_9, 1_5, 5_3_8, 2_2_7_0_4, 5_4_3_9, 1_5, 2_7_8_8, 2_4_4_9_1, 9_8_8_5, 1_5, 4_3_5_3_4, 6_0_5, 1_5, 8_1_4, 1_8_4_0_3, 3_3_2_0_0, 2_9, 1_5, 4_3_5_3_4, 2_4_4_5_8, 1_2_4_1_0, 1_1_1, 2_4_9_6_6, 8_3_6_6_9, 9_6_3_7, 1_4_4_0_6_8, 2_6, 8_5_0, 2_2_3_4_6, 2_7, 1_4_7, 2_4_9_6_6, 8_3_6_6_9, 8_3_4_9_0, 2_6, 3_9_1_1_3, 7_3_5, 2_7, 6_8_9, 6_5_6, 2_8_0_0, 1_3_3_9, 4_6_0_0, 5_3, 1_2_2_0_2_0, 1_1_5_7_8_5, 3_4, 8_1_6, 1_3_3_9, 4_6_8_8_7, 1_8, 1_4_7, 5_3_9_0_5, 1_9_5_1, 4_2_2_3_8, 4_1_1_7_0, 1_7_7_3_2, 8_3_4, 4_3_6, 1_5, 2_7_5_2_3, 9_8_7_3_3, 2_1_7, 1_4_7, 5_5_4_2, 4_9_8_1, 9_3_0, 1_7_3_4_7, 1_6, 2], [2_0_0_9_1, 6_2_9, 9_4, 8_2_7_8_6, 5_8, 4_9_0, 2_0, 1_5_2_8, 8_4, 5_3_9_0_5, 3_4_4, 8_0_5_9_2, 1_1_0_1_2_8, 1_8_8_2_2, 5_2_6_7, 1_3_0_6, 6_2, 1_5_2_5_3_7, 3_0_8, 7_9_9_7, 4_0_1, 1_2_4_4_2_7, 5_4_9, 3_5_4_4_2, 2_2_5, 1_0_9, 1_5_0_5_5, 2_5_7_4_8, 1_4_7, 7_1_1_9, 4_3_7_1_2, 3_4, 7_6_7, 1_3_5_3_6_6, 1_8, 1_6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_9_2, 6_3_7_8_4, 1_1_9_4_6_6, 1_7, 1_4_7_8_0_8, 8_8_2_1_4, 1_8, 6_5_6, 8_1, 3_2, 3_2_9_6, 1_0_2_8_0, 1_6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=snake_case    ,				model_name='microsoft/xprophetnet-large-wiki100-cased'    ,				revision='1acad1643ddd54a44df6a1b797ada8373685d90e'    ,				)





'''simple docstring'''






import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
 import torch_xla.core.xla_model as xm
 import torch_xla.debug.metrics as met






class    __magic_name__				(						lowerCAmelCase							):






       def __init__(     self    ,				*snake_case    ,				snake_case=None    ,				snake_case=None    ,				**snake_case)		->    Tuple:




          '''simple docstring'''
          super().__init__(*snake_case    ,				**snake_case)
          _UpperCAmelCase  :  Any				       =eval_examples
          _UpperCAmelCase  :  Optional[int]				       =post_process_function






       def      lowerCAmelCase						(     self    ,				snake_case = None    ,				snake_case=None    ,				snake_case = None    ,				snake_case = "eval"    ,				**snake_case    ,				)		->    Dict[str, float]:




          '''simple docstring'''
          _UpperCAmelCase  :  Tuple				       =gen_kwargs.copy()
          _UpperCAmelCase  :  int				       =(
              gen_kwargs['max_length'] if gen_kwargs.get('max_length') is not None else self.args.generation_max_length
          )
          _UpperCAmelCase  :  Dict				       =(
              gen_kwargs['num_beams'] if gen_kwargs.get('num_beams') is not None else self.args.generation_num_beams
          )
          _UpperCAmelCase  :  Dict				       =gen_kwargs

          _UpperCAmelCase  :  Tuple				       =self.eval_dataset if eval_dataset is None else eval_dataset
          _UpperCAmelCase  :  Union[str, Any]				       =self.get_eval_dataloader(snake_case)
          _UpperCAmelCase  :  Tuple				       =self.eval_examples if eval_examples is None else eval_examples

          # Temporarily disable metric computation, we will do it in the loop here.
          _UpperCAmelCase  :  Dict				       =self.compute_metrics
          _UpperCAmelCase  :  Optional[Any]				       =None
          _UpperCAmelCase  :  str				       =time.time()
          _UpperCAmelCase  :  int				       =self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
          try:
             _UpperCAmelCase  :  str				       =eval_loop(
                 snake_case    ,				description='Evaluation'    ,				prediction_loss_only=True if compute_metrics is None else None    ,				ignore_keys=snake_case    ,				metric_key_prefix=snake_case    ,				)
          finally:
             _UpperCAmelCase  :  str				       =compute_metrics
          _UpperCAmelCase  :  Optional[int]				       =self.args.eval_batch_size * self.args.world_size
          if f"{metric_key_prefix}_jit_compilation_time" in output.metrics:
             start_time += output.metrics[f"{metric_key_prefix}_jit_compilation_time"]
          output.metrics.update(
              speed_metrics(
                  snake_case    ,				snake_case    ,				num_samples=output.num_samples    ,				num_steps=math.ceil(output.num_samples / total_batch_size)    ,				))

          if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
             # Only the main node write the results by default
             _UpperCAmelCase  :  Tuple				       =self.post_process_function(snake_case    ,				snake_case    ,				snake_case)
             _UpperCAmelCase  :  Union[str, Any]				       =self.compute_metrics(snake_case)

             # Prefix all keys with metric_key_prefix + '_'
             for key in list(metrics.keys()):
                if not key.startswith(f"{metric_key_prefix}_"):
                   _UpperCAmelCase  :  Any				       =metrics.pop(snake_case)

             metrics.update(output.metrics)
          else:
             _UpperCAmelCase  :  str				       =output.metrics

          if self.args.should_log:
             # Only the main node log the results by default
             self.log(snake_case)

          if self.args.tpu_metrics_debug or self.args.debug:
             # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
             xm.master_print(met.metrics_report())

          _UpperCAmelCase  :  Any				       =self.callback_handler.on_evaluate(self.args    ,				self.state    ,				self.control    ,				snake_case)
          return metrics






       def      lowerCAmelCase						(     self    ,				snake_case    ,				snake_case    ,				snake_case=None    ,				snake_case = "test"    ,				**snake_case)		->    Dict:




          '''simple docstring'''
          _UpperCAmelCase  :  Optional[int]				       =gen_kwargs.copy()

          _UpperCAmelCase  :  int				       =self.get_test_dataloader(snake_case)

          # Temporarily disable metric computation, we will do it in the loop here.
          _UpperCAmelCase  :  str				       =self.compute_metrics
          _UpperCAmelCase  :  Union[str, Any]				       =None
          _UpperCAmelCase  :  Union[str, Any]				       =time.time()
          _UpperCAmelCase  :  Tuple				       =self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
          try:
             _UpperCAmelCase  :  Any				       =eval_loop(
                 snake_case    ,				description='Prediction'    ,				prediction_loss_only=True if compute_metrics is None else None    ,				ignore_keys=snake_case    ,				metric_key_prefix=snake_case    ,				)
          finally:
             _UpperCAmelCase  :  Any				       =compute_metrics

          _UpperCAmelCase  :  Dict				       =self.args.eval_batch_size * self.args.world_size
          if f"{metric_key_prefix}_jit_compilation_time" in output.metrics:
             start_time += output.metrics[f"{metric_key_prefix}_jit_compilation_time"]
          output.metrics.update(
              speed_metrics(
                  snake_case    ,				snake_case    ,				num_samples=output.num_samples    ,				num_steps=math.ceil(output.num_samples / total_batch_size)    ,				))
          if self.post_process_function is None or self.compute_metrics is None:
             return output

          _UpperCAmelCase  :  Tuple				       =self.post_process_function(snake_case    ,				snake_case    ,				snake_case    ,				'predict')
          _UpperCAmelCase  :  Union[str, Any]				       =self.compute_metrics(snake_case)

          # Prefix all keys with metric_key_prefix + '_'
          for key in list(metrics.keys()):
             if not key.startswith(f"{metric_key_prefix}_"):
                _UpperCAmelCase  :  Any				       =metrics.pop(snake_case)
          metrics.update(output.metrics)
          return PredictionOutput(predictions=predictions.predictions    ,				label_ids=predictions.label_ids    ,				metrics=snake_case)

