'''simple docstring'''




import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
						from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
						__lowercase        =    get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
						from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__lowercase        =    1_2_8_0_2_2
__lowercase        =    1_2_8_0_2_8






@require_sentencepiece
class 							a__(   lowerCAmelCase__					,		unittest.TestCase      ):


			'''simple docstring'''






			UpperCAmelCase_     :	List[Any]						=			MaMaaaTokenizer
			UpperCAmelCase_     :	Tuple						=			False
			UpperCAmelCase_     :	Dict						=			False
			UpperCAmelCase_     :	int						=			True



			def 				a_			(  self):




					"""simple docstring"""


					super().setUp()

					lowerCAmelCase     =    ["""</s>""", """<unk>""", """▁This""", """▁is""", """▁a""", """▁t""", """est""", """\u0120""", """<pad>"""]
					lowerCAmelCase     =    dict(zip(__lowerCAmelCase			,       range(len(__lowerCAmelCase))))
					lowerCAmelCase     =    Path(self.tmpdirname)
					save_json(__lowerCAmelCase			,       save_dir / VOCAB_FILES_NAMES["""vocab_file"""])
					if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
							copyfile(__lowerCAmelCase			,       save_dir / VOCAB_FILES_NAMES["""spm_file"""])

					lowerCAmelCase     =    MaMaaaTokenizer.from_pretrained(self.tmpdirname)
					tokenizer.save_pretrained(self.tmpdirname)



			def 				a_			(  self			,       **__lowerCAmelCase):




					"""simple docstring"""


					return MaMaaaTokenizer.from_pretrained(self.tmpdirname			,       **__lowerCAmelCase)



			def 				a_			(  self			,       __lowerCAmelCase):




					"""simple docstring"""


					return (
					    "This is a test",
					    "This is a test",
					)



			def 				a_			(  self):




					"""simple docstring"""


					lowerCAmelCase     =    """</s>"""
					lowerCAmelCase     =    0

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCAmelCase)			,       __lowerCAmelCase)
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCAmelCase)			,       __lowerCAmelCase)



			def 				a_			(  self):




					"""simple docstring"""


					lowerCAmelCase     =    self.get_tokenizer()
					lowerCAmelCase     =    list(tokenizer.get_vocab().keys())

					self.assertEqual(vocab_keys[0]			,       """</s>""")
					self.assertEqual(vocab_keys[1]			,       """<unk>""")
					self.assertEqual(vocab_keys[-1]			,       """<s>""")
					self.assertEqual(len(__lowerCAmelCase)			,       tokenizer.vocab_size + len(tokenizer.get_added_vocab()))



			@unittest.skip("""Skip this test while all models are still to be uploaded.""")
			def 				a_			(  self):




					"""simple docstring"""


					pass



			def 				a_			(  self):




					"""simple docstring"""


					lowerCAmelCase     =    self.get_tokenizer()

					lowerCAmelCase     =    tokenizer.tokenize("""This is a test""")
					self.assertListEqual(__lowerCAmelCase			,       ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""])

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(__lowerCAmelCase)			,       [2, 3, 4, 5, 6]			,       )

					lowerCAmelCase     =    tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6])
					self.assertListEqual(__lowerCAmelCase			,       ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""])

					lowerCAmelCase     =    tokenizer.convert_tokens_to_string(__lowerCAmelCase)
					self.assertEqual(__lowerCAmelCase			,       """This is a test""")







			@slow
			def 				a_			(  self):




					"""simple docstring"""


					lowerCAmelCase     =    {"""input_ids""": [[128022, 110108, 397, 11, 38272, 2247, 124811, 285, 18105, 1586, 207, 7, 39534, 4428, 397, 1019, 18105, 1586, 207, 7, 41337, 16786, 241, 7, 20214, 17, 125690, 10398, 7, 44378, 58069, 68342, 7798, 7343, 11, 299, 33310, 4, 158, 37350, 94077, 4569, 299, 33310, 90, 4, 52840, 290, 4, 31270, 112, 299, 682, 4, 52840, 39953, 14079, 193, 52519, 90894, 17894, 120697, 11, 40445, 551, 17, 1019, 52519, 90894, 17756, 963, 11, 40445, 480, 17, 9792, 1120, 5173, 1393, 6240, 16786, 241, 120996, 28, 1245, 1393, 118240, 11123, 1019, 93612, 2691, 10618, 98058, 120409, 1928, 279, 4, 40683, 367, 178, 207, 1019, 103, 103121, 506, 65296, 5, 2], [128022, 21217, 367, 117, 125450, 128, 719, 7, 7308, 40, 93612, 12669, 1116, 16704, 71, 17785, 3699, 15592, 35, 144, 9584, 241, 11943, 713, 950, 799, 2247, 88427, 150, 149, 118813, 120706, 1019, 106906, 81518, 28, 1224, 22799, 397, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [128022, 1658, 123311, 5155, 5578, 4722, 279, 14947, 2366, 1120, 1197, 14, 1348, 9232, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
					# fmt: on

					self.tokenizer_integration_test_util(
					    expected_encoding=__lowerCAmelCase			,       model_name="""facebook/m2m100_418M"""			,       revision="""c168bae485c864188cf9aa0e4108b0b6934dc91e"""			,       )







@require_torch
@require_sentencepiece
@require_tokenizers
class 							a__(   unittest.TestCase      ):


			'''simple docstring'''






			UpperCAmelCase_     :	Any						=			'''facebook/m2m100_418M'''
			UpperCAmelCase_     :	Dict						=			[
			    '''In my opinion, there are two levels of response from the French government.''',
			    '''NSA Affair Emphasizes Complete Lack of Debate on Intelligence''',
			]
			UpperCAmelCase_     :	Tuple						=			[
			    '''Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.''',
			    '''L\'affaire NSA souligne l\'absence totale de débat sur le renseignement''',
			]

			# fmt: off
			UpperCAmelCase_     :	Optional[int]						=			[EN_CODE, 5_9_3, 1_9_4_9, 1_1_5_7_8_1, 4, 7_1_5_8_6, 4_2_3_4, 6_0_6_3_3, 1_2_6_2_3_3, 4_3_2, 1_2_3_8_0_8, 1_5_5_9_2, 1_1_9_7, 1_1_7_1_3_2, 1_2_0_6_1_8, 5, 2]



			@classmethod
			def 				a_			(  cls):




					"""simple docstring"""


					lowerCAmelCase     =    MaMaaaTokenizer.from_pretrained(
					    cls.checkpoint_name			,       src_lang="""en"""			,       tgt_lang="""fr""")
					lowerCAmelCase     =    1
					return cls



			def 				a_			(  self):




					"""simple docstring"""


					self.assertEqual(self.tokenizer.get_lang_id("""ar""")			,       128006)
					self.assertEqual(self.tokenizer.get_lang_id("""en""")			,       128022)
					self.assertEqual(self.tokenizer.get_lang_id("""ro""")			,       128076)
					self.assertEqual(self.tokenizer.get_lang_id("""mr""")			,       128063)



			def 				a_			(  self):




					"""simple docstring"""


					lowerCAmelCase     =    self.tokenizer.get_vocab()
					self.assertEqual(len(__lowerCAmelCase)			,       self.tokenizer.vocab_size)
					self.assertEqual(vocab["""<unk>"""]			,       3)
					self.assertIn(self.tokenizer.get_lang_token("""en""")			,       __lowerCAmelCase)



			def 				a_			(  self):




					"""simple docstring"""


					lowerCAmelCase     =    """en"""
					lowerCAmelCase     =    self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]
					self.assertListEqual(self.expected_src_tokens			,       __lowerCAmelCase)



			def 				a_			(  self):




					"""simple docstring"""


					self.assertIn(__lowerCAmelCase			,       self.tokenizer.all_special_ids)
					# fmt: off
					lowerCAmelCase     =    [FR_CODE, 5364, 82, 8642, 4, 294, 47, 8, 14028, 136, 3286, 9706, 6, 90797, 6, 144012, 162, 88128, 30061, 5, 2]
					# fmt: on
					lowerCAmelCase     =    self.tokenizer.decode(__lowerCAmelCase			,       skip_special_tokens=__lowerCAmelCase)
					lowerCAmelCase     =    self.tokenizer.decode(generated_ids[1:]			,       skip_special_tokens=__lowerCAmelCase)
					self.assertEqual(__lowerCAmelCase			,       __lowerCAmelCase)
					self.assertNotIn(self.tokenizer.eos_token			,       __lowerCAmelCase)



			def 				a_			(  self):




					"""simple docstring"""


					lowerCAmelCase     =    tempfile.mkdtemp()
					lowerCAmelCase     =    self.tokenizer.lang_token_to_id
					self.tokenizer.save_pretrained(__lowerCAmelCase)
					lowerCAmelCase     =    MaMaaaTokenizer.from_pretrained(__lowerCAmelCase)
					self.assertDictEqual(new_tok.lang_token_to_id			,       __lowerCAmelCase)



			@require_torch
			def 				a_			(  self):




					"""simple docstring"""


					lowerCAmelCase     =    """en"""
					lowerCAmelCase     =    """fr"""

					lowerCAmelCase     =    self.tokenizer(self.src_text			,       text_target=self.tgt_text			,       padding=__lowerCAmelCase			,       return_tensors="""pt""")

					lowerCAmelCase     =    shift_tokens_right(
					    batch["""labels"""]			,       self.tokenizer.pad_token_id			,       self.tokenizer.eos_token_id)

					for k in batch:
							lowerCAmelCase     =    batch[k].tolist()
					# batch = {k: v.tolist() for k,v in batch.items()}
					# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
					# batch.decoder_inputs_ids[0][0] ==
					assert batch.input_ids[1][0] == EN_CODE
					assert batch.input_ids[1][-1] == 2
					assert batch.labels[1][0] == FR_CODE
					assert batch.labels[1][-1] == 2
					assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]



			@require_torch
			def 				a_			(  self):




					"""simple docstring"""


					lowerCAmelCase     =    """mr"""
					self.assertListEqual(self.tokenizer.prefix_tokens			,       [self.tokenizer.get_lang_id("""mr""")])
					self.assertListEqual(self.tokenizer.suffix_tokens			,       [self.tokenizer.eos_token_id])

					lowerCAmelCase     =    """zh"""
					self.assertListEqual(self.tokenizer.prefix_tokens			,       [self.tokenizer.get_lang_id("""zh""")])
					self.assertListEqual(self.tokenizer.suffix_tokens			,       [self.tokenizer.eos_token_id])



			@require_torch
			def 				a_			(  self):




					"""simple docstring"""


					lowerCAmelCase     =    """mr"""
					self.tokenizer._switch_to_target_mode()
					self.assertListEqual(self.tokenizer.prefix_tokens			,       [self.tokenizer.get_lang_id("""mr""")])
					self.assertListEqual(self.tokenizer.suffix_tokens			,       [self.tokenizer.eos_token_id])
					self.tokenizer._switch_to_input_mode()
					self.assertListEqual(self.tokenizer.prefix_tokens			,       [self.tokenizer.get_lang_id(self.tokenizer.src_lang)])

					lowerCAmelCase     =    """zh"""
					self.tokenizer._switch_to_target_mode()
					self.assertListEqual(self.tokenizer.prefix_tokens			,       [self.tokenizer.get_lang_id("""zh""")])
					self.assertListEqual(self.tokenizer.suffix_tokens			,       [self.tokenizer.eos_token_id])
					self.tokenizer._switch_to_input_mode()
					self.assertListEqual(self.tokenizer.prefix_tokens			,       [self.tokenizer.get_lang_id(self.tokenizer.src_lang)])







			@require_torch
			def 				a_			(  self):




					"""simple docstring"""


					lowerCAmelCase     =    self.tokenizer._build_translation_inputs("""A test"""			,       return_tensors="""pt"""			,       src_lang="""en"""			,       tgt_lang="""ar""")

					self.assertEqual(
					    nested_simplify(__lowerCAmelCase)			,       {
					        # en_XX, A, test, EOS
					        """input_ids""": [[128022, 58, 4183, 2]],
					        """attention_mask""": [[1, 1, 1, 1]],
					        # ar_AR
					        """forced_bos_token_id""": 128006,
					    }			,       )





'''simple docstring'''




import math
import time

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
						import torch_xla.core.xla_model as xm
						import torch_xla.debug.metrics as met






class 							a__(   lowerCAmelCase__      ):


			'''simple docstring'''






			def __init__(  self			,       *__lowerCAmelCase			,       __lowerCAmelCase=None			,       __lowerCAmelCase=None			,       **__lowerCAmelCase):




					"""simple docstring"""


					super().__init__(*__lowerCAmelCase			,       **__lowerCAmelCase)
					lowerCAmelCase     =    eval_examples
					lowerCAmelCase     =    post_process_function



			def 				a_			(  self			,       __lowerCAmelCase=None			,       __lowerCAmelCase=None			,       __lowerCAmelCase=None			,       __lowerCAmelCase = "eval"):




					"""simple docstring"""


					lowerCAmelCase     =    self.eval_dataset if eval_dataset is None else eval_dataset
					lowerCAmelCase     =    self.get_eval_dataloader(__lowerCAmelCase)
					lowerCAmelCase     =    self.eval_examples if eval_examples is None else eval_examples

					# Temporarily disable metric computation, we will do it in the loop here.
					lowerCAmelCase     =    self.compute_metrics
					lowerCAmelCase     =    None
					lowerCAmelCase     =    self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
					lowerCAmelCase     =    time.time()
					try:
							lowerCAmelCase     =    eval_loop(
							    __lowerCAmelCase			,       description="""Evaluation"""			,       prediction_loss_only=True if compute_metrics is None else None			,       ignore_keys=__lowerCAmelCase			,       metric_key_prefix=__lowerCAmelCase			,       )
					finally:
							lowerCAmelCase     =    compute_metrics
					lowerCAmelCase     =    self.args.eval_batch_size * self.args.world_size
					if f"{metric_key_prefix}_jit_compilation_time" in output.metrics:
							start_time += output.metrics[f"{metric_key_prefix}_jit_compilation_time"]
					output.metrics.update(
					    speed_metrics(
					        __lowerCAmelCase			,       __lowerCAmelCase			,       num_samples=output.num_samples			,       num_steps=math.ceil(output.num_samples / total_batch_size)			,       ))
					if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
							# Only the main node write the results by default
							lowerCAmelCase     =    self.post_process_function(__lowerCAmelCase			,       __lowerCAmelCase			,       output.predictions)
							lowerCAmelCase     =    self.compute_metrics(__lowerCAmelCase)

							# Prefix all keys with metric_key_prefix + '_'
							for key in list(metrics.keys()):
									if not key.startswith(f"{metric_key_prefix}_"):
											lowerCAmelCase     =    metrics.pop(__lowerCAmelCase)
							metrics.update(output.metrics)
					else:
							lowerCAmelCase     =    output.metrics

					if self.args.should_log:
							# Only the main node log the results by default
							self.log(__lowerCAmelCase)

					if self.args.tpu_metrics_debug or self.args.debug:
							# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
							xm.master_print(met.metrics_report())

					lowerCAmelCase     =    self.callback_handler.on_evaluate(self.args			,       self.state			,       self.control			,       __lowerCAmelCase)
					return metrics







			def 				a_			(  self			,       __lowerCAmelCase			,       __lowerCAmelCase			,       __lowerCAmelCase=None			,       __lowerCAmelCase = "test"):




					"""simple docstring"""


					lowerCAmelCase     =    self.get_test_dataloader(__lowerCAmelCase)

					# Temporarily disable metric computation, we will do it in the loop here.
					lowerCAmelCase     =    self.compute_metrics
					lowerCAmelCase     =    None
					lowerCAmelCase     =    self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
					lowerCAmelCase     =    time.time()
					try:
							lowerCAmelCase     =    eval_loop(
							    __lowerCAmelCase			,       description="""Prediction"""			,       prediction_loss_only=True if compute_metrics is None else None			,       ignore_keys=__lowerCAmelCase			,       metric_key_prefix=__lowerCAmelCase			,       )
					finally:
							lowerCAmelCase     =    compute_metrics
					lowerCAmelCase     =    self.args.eval_batch_size * self.args.world_size
					if f"{metric_key_prefix}_jit_compilation_time" in output.metrics:
							start_time += output.metrics[f"{metric_key_prefix}_jit_compilation_time"]
					output.metrics.update(
					    speed_metrics(
					        __lowerCAmelCase			,       __lowerCAmelCase			,       num_samples=output.num_samples			,       num_steps=math.ceil(output.num_samples / total_batch_size)			,       ))

					if self.post_process_function is None or self.compute_metrics is None:
							return output

					lowerCAmelCase     =    self.post_process_function(__lowerCAmelCase			,       __lowerCAmelCase			,       output.predictions			,       """predict""")
					lowerCAmelCase     =    self.compute_metrics(__lowerCAmelCase)

					# Prefix all keys with metric_key_prefix + '_'
					for key in list(metrics.keys()):
							if not key.startswith(f"{metric_key_prefix}_"):
									lowerCAmelCase     =    metrics.pop(__lowerCAmelCase)
					metrics.update(output.metrics)
					return PredictionOutput(predictions=predictions.predictions			,       label_ids=predictions.label_ids			,       metrics=__lowerCAmelCase)




