






import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 							snake_case_       (    __UpperCamelCase		,							unittest.TestCase      ):

						"""simple docstring"""



						snake_case__        =      GPTaTokenizer
						snake_case__        =      GPTaTokenizerFast
						snake_case__        =      True
						snake_case__        =      {"""add_prefix_space""": True}
						snake_case__        =      False



						def        UpperCAmelCase__					(self:					List[Any]   )    ->     Optional[int]:




											'''simple docstring'''



											super().setUp()

											# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
											__a					:						str      =					[
											    "l",
											    "o",
											    "w",
											    "e",
											    "r",
											    "s",
											    "t",
											    "i",
											    "d",
											    "n",
											    "\u0120",
											    "\u0120l",
											    "\u0120n",
											    "\u0120lo",
											    "\u0120low",
											    "er",
											    "\u0120lowest",
											    "\u0120newer",
											    "\u0120wider",
											    "<unk>",
											    "<|endoftext|>",
											]
											__a					:						Any      =					dict(zip(__UpperCAmelCase ,					range(len(__UpperCAmelCase   )   )   )   )
											__a					:						Union[str, Any]      =					["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
											__a					:						List[Any]      =					{"unk_token": "<unk>"}

											__a					:						int      =					os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES["vocab_file"]   )
											__a					:						int      =					os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES["merges_file"]   )
											with open(self.vocab_file ,					"w" ,					encoding="utf-8"   ) as fp:
																fp.write(json.dumps(__UpperCAmelCase   ) + "\n"   )
											with open(self.merges_file ,					"w" ,					encoding="utf-8"   ) as fp:
																fp.write("\n".join(__UpperCAmelCase   )   )



						def        UpperCAmelCase__					(self:					Optional[int] ,					**__UpperCAmelCase:					Optional[Any]   )    ->     Optional[int]:




											'''simple docstring'''



											kwargs.update(self.special_tokens_map   )
											return GPTaTokenizer.from_pretrained(self.tmpdirname ,					**__UpperCAmelCase   )



						def        UpperCAmelCase__					(self:					Dict ,					**__UpperCAmelCase:					Dict   )    ->     Optional[int]:




											'''simple docstring'''



											kwargs.update(self.special_tokens_map   )
											return GPTaTokenizerFast.from_pretrained(self.tmpdirname ,					**__UpperCAmelCase   )



						def        UpperCAmelCase__					(self:					List[str] ,					__UpperCAmelCase:					Optional[Any]   )    ->     List[str]:




											'''simple docstring'''



											__a					:						Any      =					"lower newer"
											__a					:						Dict      =					"lower newer"
											return input_text, output_text



						def        UpperCAmelCase__					(self:					Union[str, Any]   )    ->     str:




											'''simple docstring'''



											__a					:						Dict      =					GPTaTokenizer(self.vocab_file ,					self.merges_file ,					**self.special_tokens_map   )
											__a					:						Any      =					"lower newer"
											__a					:						Optional[int]      =					["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
											__a					:						Any      =					tokenizer.tokenize(__UpperCAmelCase ,					add_prefix_space=__UpperCAmelCase   )
											self.assertListEqual(__UpperCAmelCase ,					__UpperCAmelCase   )

											__a					:						str      =					tokens + [tokenizer.unk_token]
											__a					:						int      =					[14, 15, 10, 9, 3, 2, 15, 19]
											self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase   ) ,					__UpperCAmelCase   )



						def        UpperCAmelCase__					(self:					Tuple   )    ->     Optional[int]:




											'''simple docstring'''



											if not self.test_rust_tokenizer:
																return

											__a					:						Optional[int]      =					self.get_tokenizer()
											__a					:						Tuple      =					self.get_rust_tokenizer(add_prefix_space=__UpperCAmelCase   )

											__a					:						Optional[int]      =					"lower newer"

											# Testing tokenization
											__a					:						int      =					tokenizer.tokenize(__UpperCAmelCase ,					add_prefix_space=__UpperCAmelCase   )
											__a					:						List[str]      =					rust_tokenizer.tokenize(__UpperCAmelCase   )
											self.assertListEqual(__UpperCAmelCase ,					__UpperCAmelCase   )

											# Testing conversion to ids without special tokens
											__a					:						Dict      =					tokenizer.encode(__UpperCAmelCase ,					add_special_tokens=__UpperCAmelCase ,					add_prefix_space=__UpperCAmelCase   )
											__a					:						Union[str, Any]      =					rust_tokenizer.encode(__UpperCAmelCase ,					add_special_tokens=__UpperCAmelCase   )
											self.assertListEqual(__UpperCAmelCase ,					__UpperCAmelCase   )

											# Testing conversion to ids with special tokens
											__a					:						Dict      =					self.get_rust_tokenizer(add_prefix_space=__UpperCAmelCase   )
											__a					:						List[str]      =					tokenizer.encode(__UpperCAmelCase ,					add_prefix_space=__UpperCAmelCase   )
											__a					:						List[str]      =					rust_tokenizer.encode(__UpperCAmelCase   )
											self.assertListEqual(__UpperCAmelCase ,					__UpperCAmelCase   )

											# Testing the unknown token
											__a					:						Any      =					tokens + [rust_tokenizer.unk_token]
											__a					:						int      =					[14, 15, 10, 9, 3, 2, 15, 19]
											self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__UpperCAmelCase   ) ,					__UpperCAmelCase   )



						def        UpperCAmelCase__					(self:					str ,					*__UpperCAmelCase:					List[Any] ,					**__UpperCAmelCase:					Optional[int]   )    ->     List[str]:




											'''simple docstring'''



											pass



						def        UpperCAmelCase__					(self:					Union[str, Any] ,					__UpperCAmelCase:					List[str]=15   )    ->     int:




											'''simple docstring'''



											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
																					__a					:						Any      =					self.rust_tokenizer_class.from_pretrained(__UpperCAmelCase ,					**__UpperCAmelCase   )

																					# Simple input
																					__a					:						int      =					"This is a simple input"
																					__a					:						List[Any]      =					["This is a simple input 1", "This is a simple input 2"]
																					__a					:						str      =					("This is a simple input", "This is a pair")
																					__a					:						Any      =					[
																					    ("This is a simple input 1", "This is a simple input 2"),
																					    ("This is a simple pair 1", "This is a simple pair 2"),
																					]

																					# Simple input tests
																					self.assertRaises(__UpperCAmelCase ,					tokenizer_r.encode ,					__UpperCAmelCase ,					max_length=__UpperCAmelCase ,					padding="max_length"   )

																					# Simple input
																					self.assertRaises(__UpperCAmelCase ,					tokenizer_r.encode_plus ,					__UpperCAmelCase ,					max_length=__UpperCAmelCase ,					padding="max_length"   )

																					# Simple input
																					self.assertRaises(
																					    __UpperCAmelCase ,					tokenizer_r.batch_encode_plus ,					__UpperCAmelCase ,					max_length=__UpperCAmelCase ,					padding="max_length" ,					)

																					# Pair input
																					self.assertRaises(__UpperCAmelCase ,					tokenizer_r.encode ,					__UpperCAmelCase ,					max_length=__UpperCAmelCase ,					padding="max_length"   )

																					# Pair input
																					self.assertRaises(__UpperCAmelCase ,					tokenizer_r.encode_plus ,					__UpperCAmelCase ,					max_length=__UpperCAmelCase ,					padding="max_length"   )

																					# Pair input
																					self.assertRaises(
																					    __UpperCAmelCase ,					tokenizer_r.batch_encode_plus ,					__UpperCAmelCase ,					max_length=__UpperCAmelCase ,					padding="max_length" ,					)



						def        UpperCAmelCase__					(self:					List[Any]   )    ->     Optional[int]:




											'''simple docstring'''



											__a					:						List[Any]      =					GPTaTokenizer.from_pretrained(self.tmpdirname ,					pad_token="<pad>"   )

											# Simple input
											__a					:						Tuple      =					"This is a simple input"
											__a					:						int      =					["This is a simple input looooooooong", "This is a simple input"]
											__a					:						str      =					("This is a simple input", "This is a pair")
											__a					:						Any      =					[
											    ("This is a simple input loooooong", "This is a simple input"),
											    ("This is a simple pair loooooong", "This is a simple pair"),
											]

											__a					:						Optional[int]      =					tokenizer.pad_token_id

											__a					:						Union[str, Any]      =					tokenizer(__UpperCAmelCase ,					padding="max_length" ,					max_length=30 ,					return_tensors="np"   )
											__a					:						Any      =					tokenizer(__UpperCAmelCase ,					padding=__UpperCAmelCase ,					truncate=__UpperCAmelCase ,					return_tensors="np"   )
											__a					:						Union[str, Any]      =					tokenizer(*__UpperCAmelCase ,					padding="max_length" ,					max_length=60 ,					return_tensors="np"   )
											__a					:						Tuple      =					tokenizer(__UpperCAmelCase ,					padding=__UpperCAmelCase ,					truncate=__UpperCAmelCase ,					return_tensors="np"   )

											# s
											# test single string max_length padding
											self.assertEqual(out_s["input_ids"].shape[-1] ,					30   )
											self.assertTrue(pad_token_id in out_s["input_ids"]   )
											self.assertTrue(0 in out_s["attention_mask"]   )

											# s2
											# test automatic padding
											self.assertEqual(out_sa["input_ids"].shape[-1] ,					33   )
											# long slice doesn't have padding
											self.assertFalse(pad_token_id in out_sa["input_ids"][0]   )
											self.assertFalse(0 in out_sa["attention_mask"][0]   )
											# short slice does have padding
											self.assertTrue(pad_token_id in out_sa["input_ids"][1]   )
											self.assertTrue(0 in out_sa["attention_mask"][1]   )

											# p
											# test single pair max_length padding
											self.assertEqual(out_p["input_ids"].shape[-1] ,					60   )
											self.assertTrue(pad_token_id in out_p["input_ids"]   )
											self.assertTrue(0 in out_p["attention_mask"]   )

											# p2
											# test automatic padding pair
											self.assertEqual(out_pa["input_ids"].shape[-1] ,					52   )
											# long slice pair doesn't have padding
											self.assertFalse(pad_token_id in out_pa["input_ids"][0]   )
											self.assertFalse(0 in out_pa["attention_mask"][0]   )
											# short slice pair does have padding
											self.assertTrue(pad_token_id in out_pa["input_ids"][1]   )
											self.assertTrue(0 in out_pa["attention_mask"][1]   )



						def        UpperCAmelCase__					(self:					int   )    ->     List[Any]:




											'''simple docstring'''



											__a					:						Dict      =					"$$$"
											__a					:						str      =					GPTaTokenizer.from_pretrained(self.tmpdirname ,					bos_token=__UpperCAmelCase ,					add_bos_token=__UpperCAmelCase   )

											__a					:						Optional[int]      =					"This is a simple input"
											__a					:						Tuple      =					["This is a simple input 1", "This is a simple input 2"]

											__a					:						Optional[int]      =					tokenizer.bos_token_id

											__a					:						Any      =					tokenizer(__UpperCAmelCase   )
											__a					:						Any      =					tokenizer(__UpperCAmelCase   )

											self.assertEqual(out_s.input_ids[0] ,					__UpperCAmelCase   )
											self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids   )   )

											__a					:						Union[str, Any]      =					tokenizer.decode(out_s.input_ids   )
											__a					:						str      =					tokenizer.batch_decode(out_sa.input_ids   )

											self.assertEqual(decode_s.split()[0] ,					__UpperCAmelCase   )
											self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa   )   )



						def        UpperCAmelCase__					(self:					Union[str, Any]   )    ->     int:




											'''simple docstring'''



											pass



						def        UpperCAmelCase__					(self:					int   )    ->     Optional[int]:




											'''simple docstring'''



											__a					:						Dict      =					[self.get_tokenizer(do_lower_case=__UpperCAmelCase ,					add_bos_token=__UpperCAmelCase   )]
											for tokenizer in tokenizers:
																with self.subTest(f'{tokenizer.__class__.__name__}'   ):
																					__a					:						str      =					"Encode this."
																					__a					:						int      =					"This one too please."
																					__a					:						Optional[Any]      =					tokenizer.encode(__UpperCAmelCase ,					add_special_tokens=__UpperCAmelCase   )
																					encoded_sequence += tokenizer.encode(__UpperCAmelCase ,					add_special_tokens=__UpperCAmelCase   )
																					__a					:						Dict      =					tokenizer.encode_plus(
																					    __UpperCAmelCase ,					__UpperCAmelCase ,					add_special_tokens=__UpperCAmelCase ,					return_special_tokens_mask=__UpperCAmelCase ,					)
																					__a					:						List[Any]      =					encoded_sequence_dict["input_ids"]
																					__a					:						Any      =					encoded_sequence_dict["special_tokens_mask"]
																					self.assertEqual(len(__UpperCAmelCase   ) ,					len(__UpperCAmelCase   )   )

																					__a					:						Optional[int]      =					[
																					    (x if not special_tokens_mask[i] else None) for i, x in enumerate(__UpperCAmelCase   )
																					]
																					__a					:						List[str]      =					[x for x in filtered_sequence if x is not None]
																					self.assertEqual(__UpperCAmelCase ,					__UpperCAmelCase   )




@require_tokenizers
class 							snake_case_       (    unittest.TestCase      ):

						"""simple docstring"""



						def        UpperCAmelCase__					(self:					Optional[int]   )    ->     Optional[int]:




											'''simple docstring'''



											__a					:						Dict      =					AutoTokenizer.from_pretrained("facebook/opt-350m" ,					from_slow=__UpperCAmelCase   )
											__a					:						str      =					"A photo of a cat"

											__a					:						Union[str, Any]      =					tokenizer.encode(
											    __UpperCAmelCase ,					)
											self.assertEqual(__UpperCAmelCase ,					[2, 250, 1345, 9, 10, 4758]   )
											tokenizer.save_pretrained("test_opt"   )

											__a					:						List[str]      =					AutoTokenizer.from_pretrained("./test_opt"   )
											__a					:						List[Any]      =					tokenizer.encode(
											    __UpperCAmelCase ,					)
											self.assertEqual(__UpperCAmelCase ,					[2, 250, 1345, 9, 10, 4758]   )



						def        UpperCAmelCase__					(self:					List[str]   )    ->     Union[str, Any]:




											'''simple docstring'''



											__a					:						Tuple      =					AutoTokenizer.from_pretrained("facebook/opt-350m" ,					use_slow=__UpperCAmelCase   )
											__a					:						Optional[Any]      =					"A photo of a cat"

											__a					:						Tuple      =					tokenizer.encode(
											    __UpperCAmelCase ,					)
											# Same as above
											self.assertEqual(__UpperCAmelCase ,					[2, 250, 1345, 9, 10, 4758]   )



						@unittest.skip("This test is failing because of a bug in the fast tokenizer"   )
						def        UpperCAmelCase__					(self:					int   )    ->     Any:




											'''simple docstring'''



											__a					:						int      =					AutoTokenizer.from_pretrained("facebook/opt-350m" ,					from_slow=__UpperCAmelCase   )

											__a					:						Tuple      =					"bos"
											__a					:						int      =					tokenizer.get_vocab()["bos"]

											__a					:						Union[str, Any]      =					"A photo of a cat"
											__a					:						Optional[int]      =					tokenizer.encode(
											    __UpperCAmelCase ,					)
											# We changed the bos token
											self.assertEqual(__UpperCAmelCase ,					[31957, 250, 1345, 9, 10, 4758]   )
											tokenizer.save_pretrained("./tok"   )
											__a					:						str      =					AutoTokenizer.from_pretrained("./tok"   )
											self.assertTrue(tokenizer.is_fast   )
											__a					:						List[Any]      =					tokenizer.encode(
											    __UpperCAmelCase ,					)
											self.assertEqual(__UpperCAmelCase ,					[31957, 250, 1345, 9, 10, 4758]   )













import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
			import torch_xla.core.xla_model as xm
			import torch_xla.debug.metrics as met




class 							snake_case_       (    __UpperCamelCase      ):

						"""simple docstring"""



						def __init__(self:					str ,					*__UpperCAmelCase:					Optional[int] ,					__UpperCAmelCase:					Optional[int]=None ,					__UpperCAmelCase:					Dict=None ,					**__UpperCAmelCase:					Tuple   )    ->     Dict:




											'''simple docstring'''



											super().__init__(*__UpperCAmelCase ,					**__UpperCAmelCase   )
											__a					:						str      =					eval_examples
											__a					:						List[Any]      =					post_process_function



						def        UpperCAmelCase__					(self:					Optional[int] ,					__UpperCAmelCase:					Optional[Dataset] = None ,					__UpperCAmelCase:					Optional[int]=None ,					__UpperCAmelCase:					Optional[List[str]] = None ,					__UpperCAmelCase:					str = "eval" ,					**__UpperCAmelCase:					Any ,					)    ->     Dict[str, float]:




											'''simple docstring'''



											__a					:						Optional[int]      =					gen_kwargs.copy()
											__a					:						Union[str, Any]      =					(
											    gen_kwargs["max_length"] if gen_kwargs.get("max_length"   ) is not None else self.args.generation_max_length
											)
											__a					:						Dict      =					(
											    gen_kwargs["num_beams"] if gen_kwargs.get("num_beams"   ) is not None else self.args.generation_num_beams
											)
											__a					:						Optional[Any]      =					gen_kwargs

											__a					:						Union[str, Any]      =					self.eval_dataset if eval_dataset is None else eval_dataset
											__a					:						Optional[int]      =					self.get_eval_dataloader(__UpperCAmelCase   )
											__a					:						Tuple      =					self.eval_examples if eval_examples is None else eval_examples

											# Temporarily disable metric computation, we will do it in the loop here.
											__a					:						int      =					self.compute_metrics
											__a					:						Dict      =					None
											__a					:						List[Any]      =					time.time()
											__a					:						str      =					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																__a					:						str      =					eval_loop(
																    __UpperCAmelCase ,					description="Evaluation" ,					prediction_loss_only=True if compute_metrics is None else None ,					ignore_keys=__UpperCAmelCase ,					metric_key_prefix=__UpperCAmelCase ,					)
											finally:
																__a					:						Union[str, Any]      =					compute_metrics
											__a					:						str      =					self.args.eval_batch_size * self.args.world_size
											if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
																start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
											output.metrics.update(
											    speed_metrics(
											        __UpperCAmelCase ,					__UpperCAmelCase ,					num_samples=output.num_samples ,					num_steps=math.ceil(output.num_samples / total_batch_size   ) ,					)   )

											if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
																# Only the main node write the results by default
																__a					:						Union[str, Any]      =					self.post_process_function(__UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase   )
																__a					:						Dict      =					self.compute_metrics(__UpperCAmelCase   )

																# Prefix all keys with metric_key_prefix + '_'
																for key in list(metrics.keys()   ):
																					if not key.startswith(f'{metric_key_prefix}_'   ):
																										__a					:						int      =					metrics.pop(__UpperCAmelCase   )

																metrics.update(output.metrics   )
											else:
																__a					:						Optional[int]      =					output.metrics

											if self.args.should_log:
																# Only the main node log the results by default
																self.log(__UpperCAmelCase   )

											if self.args.tpu_metrics_debug or self.args.debug:
																# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																xm.master_print(met.metrics_report()   )

											__a					:						Tuple      =					self.callback_handler.on_evaluate(self.args ,					self.state ,					self.control ,					__UpperCAmelCase   )
											return metrics



						def        UpperCAmelCase__					(self:					Dict ,					__UpperCAmelCase:					List[str] ,					__UpperCAmelCase:					List[str] ,					__UpperCAmelCase:					Union[str, Any]=None ,					__UpperCAmelCase:					str = "test" ,					**__UpperCAmelCase:					List[Any]   )    ->     Union[str, Any]:




											'''simple docstring'''



											__a					:						Any      =					gen_kwargs.copy()

											__a					:						Tuple      =					self.get_test_dataloader(__UpperCAmelCase   )

											# Temporarily disable metric computation, we will do it in the loop here.
											__a					:						Optional[Any]      =					self.compute_metrics
											__a					:						int      =					None
											__a					:						List[Any]      =					time.time()
											__a					:						Dict      =					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																__a					:						Optional[Any]      =					eval_loop(
																    __UpperCAmelCase ,					description="Prediction" ,					prediction_loss_only=True if compute_metrics is None else None ,					ignore_keys=__UpperCAmelCase ,					metric_key_prefix=__UpperCAmelCase ,					)
											finally:
																__a					:						List[Any]      =					compute_metrics

											__a					:						Optional[Any]      =					self.args.eval_batch_size * self.args.world_size
											if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
																start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
											output.metrics.update(
											    speed_metrics(
											        __UpperCAmelCase ,					__UpperCAmelCase ,					num_samples=output.num_samples ,					num_steps=math.ceil(output.num_samples / total_batch_size   ) ,					)   )
											if self.post_process_function is None or self.compute_metrics is None:
																return output

											__a					:						Optional[int]      =					self.post_process_function(__UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase ,					"predict"   )
											__a					:						Dict      =					self.compute_metrics(__UpperCAmelCase   )

											# Prefix all keys with metric_key_prefix + '_'
											for key in list(metrics.keys()   ):
																if not key.startswith(f'{metric_key_prefix}_'   ):
																					__a					:						List[str]      =					metrics.pop(__UpperCAmelCase   )
											metrics.update(output.metrics   )
											return PredictionOutput(predictions=predictions.predictions ,					label_ids=predictions.label_ids ,					metrics=__UpperCAmelCase   )





