





import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartaaTokenizer, MBartaaTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import TokenizerTesterMixin


A_												=     get_tests_dir("fixtures/test_sentencepiece.model")

if is_torch_available():
						from transformers.models.mbart.modeling_mbart import shift_tokens_right

A_												=     2_5_0_0_0_4
A_												=     2_5_0_0_2_0

@require_sentencepiece
@require_tokenizers
class SCREAMING_SNAKE_CASE_ (					lowercase_							,       unittest.TestCase   ):





					"""simple docstring"""


					A__				 =    MBartaaTokenizer
					A__				 =    MBartaaTokenizerFast
					A__				 =    True
					A__				 =    True





					def 					__magic_name__ ( self	):
										super().setUp()

										# We have a SentencePiece fixture for testing
										lowerCamelCase__       =       MBartaaTokenizer(_lowerCAmelCase	,  src_lang="en_XX"	,  tgt_lang="ro_RO"	,  keep_accents=_lowerCAmelCase	)
										tokenizer.save_pretrained(self.tmpdirname	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       "<s>"
										lowerCamelCase__       =       0

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCAmelCase	)	,  _lowerCAmelCase	)
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCAmelCase	)	,  _lowerCAmelCase	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       list(self.get_tokenizer().get_vocab().keys()	)

										self.assertEqual(vocab_keys[0]	,  "<s>"	)
										self.assertEqual(vocab_keys[1]	,  "<pad>"	)
										self.assertEqual(vocab_keys[-1]	,  "<mask>"	)
										self.assertEqual(len(_lowerCAmelCase	)	,  1054	)





					def 					__magic_name__ ( self	):
										self.assertEqual(self.get_tokenizer().vocab_size	,  1054	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       MBartaaTokenizer(_lowerCAmelCase	,  src_lang="en_XX"	,  tgt_lang="ro_RO"	,  keep_accents=_lowerCAmelCase	)

										lowerCamelCase__       =       tokenizer.tokenize("This is a test"	)
										self.assertListEqual(_lowerCAmelCase	,  ["▁This", "▁is", "▁a", "▁t", "est"]	)

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(_lowerCAmelCase	)	,  [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]	,  )

										lowerCamelCase__       =       tokenizer.tokenize("I was born in 92000, and this is falsé."	)
										self.assertListEqual(
										    _lowerCAmelCase	,  [SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "9", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "é", "."]	,  )
										lowerCamelCase__       =       tokenizer.convert_tokens_to_ids(_lowerCAmelCase	)
										self.assertListEqual(
										    _lowerCAmelCase	,  [
										        value + tokenizer.fairseq_offset
										        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
										    ]	,  )

										lowerCamelCase__       =       tokenizer.convert_ids_to_tokens(_lowerCAmelCase	)
										self.assertListEqual(
										    _lowerCAmelCase	,  [SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "<unk>", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "<unk>", "."]	,  )





					@slow
					def 					__magic_name__ ( self	):
										# fmt: off
										lowerCamelCase__       =       {"input_ids": [[25_0004, 1_1062, 8_2772, 7, 15, 8_2772, 538, 5_1529, 237, 1_7198, 1290, 206, 9, 21_5175, 1314, 136, 1_7198, 1290, 206, 9, 5_6359, 42, 12_2009, 9, 1_6466, 16, 8_7344, 4537, 9, 4717, 7_8381, 6, 15_9958, 7, 15, 2_4480, 618, 4, 527, 2_2693, 5428, 4, 2777, 2_4480, 9874, 4, 4_3523, 594, 4, 803, 1_8392, 3_3189, 18, 4, 4_3523, 2_4447, 1_2399, 100, 2_4955, 8_3658, 9626, 14_4057, 15, 839, 2_2335, 16, 136, 2_4955, 8_3658, 8_3479, 15, 3_9102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 12_2009, 11_5774, 23, 805, 1328, 4_6876, 7, 136, 5_3894, 1940, 4_2227, 4_1159, 1_7721, 823, 425, 4, 2_7512, 9_8722, 206, 136, 5531, 4970, 919, 1_7336, 5, 2], [25_0004, 2_0080, 618, 83, 8_2775, 47, 479, 9, 1517, 73, 5_3894, 333, 8_0581, 11_0117, 1_8811, 5256, 1295, 51, 15_2526, 297, 7986, 390, 12_4416, 538, 3_5431, 214, 98, 1_5044, 2_5737, 136, 7108, 4_3701, 23, 756, 13_5355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [25_0004, 581, 6_3773, 11_9455, 6, 14_7797, 8_8203, 7, 645, 70, 21, 3285, 1_0269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=_lowerCAmelCase	,  model_name="facebook/mbart-large-50"	,  revision="d3913889c59cd5c9e456b269c376325eabad57e2"	,  )






					def 					__magic_name__ ( self	):
										if not self.test_slow_tokenizer:
															# as we don't have a slow version, we can't compare the outputs between slow and fast versions
															return

										lowerCamelCase__       =       (self.rust_tokenizer_class, "hf-internal-testing/tiny-random-mbart50", {})
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"	):
																				lowerCamelCase__       =       self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase	,  **_lowerCAmelCase	)
																				lowerCamelCase__       =       self.tokenizer_class.from_pretrained(_lowerCAmelCase	,  **_lowerCAmelCase	)

																				lowerCamelCase__       =       tempfile.mkdtemp()

																				lowerCamelCase__       =       tokenizer_r.save_pretrained(_lowerCAmelCase	)
																				lowerCamelCase__       =       tokenizer_p.save_pretrained(_lowerCAmelCase	)

																				# Checks it save with the same files + the tokenizer.json file for the fast one
																				self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files	)	)
																				lowerCamelCase__       =       tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f	)
																				self.assertSequenceEqual(_lowerCAmelCase	,  _lowerCAmelCase	)

																				# Checks everything loads correctly in the same way
																				lowerCamelCase__       =       tokenizer_r.from_pretrained(_lowerCAmelCase	)
																				lowerCamelCase__       =       tokenizer_p.from_pretrained(_lowerCAmelCase	)

																				# Check special tokens are set accordingly on Rust and Python
																				for key in tokenizer_pp.special_tokens_map:
																									self.assertTrue(hasattr(_lowerCAmelCase	,  _lowerCAmelCase	)	)
																									# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																									# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

																				shutil.rmtree(_lowerCAmelCase	)

																				# Save tokenizer rust, legacy_format=True
																				lowerCamelCase__       =       tempfile.mkdtemp()

																				lowerCamelCase__       =       tokenizer_r.save_pretrained(_lowerCAmelCase	,  legacy_format=_lowerCAmelCase	)
																				lowerCamelCase__       =       tokenizer_p.save_pretrained(_lowerCAmelCase	)

																				# Checks it save with the same files
																				self.assertSequenceEqual(_lowerCAmelCase	,  _lowerCAmelCase	)

																				# Checks everything loads correctly in the same way
																				lowerCamelCase__       =       tokenizer_r.from_pretrained(_lowerCAmelCase	)
																				lowerCamelCase__       =       tokenizer_p.from_pretrained(_lowerCAmelCase	)

																				# Check special tokens are set accordingly on Rust and Python
																				for key in tokenizer_pp.special_tokens_map:
																									self.assertTrue(hasattr(_lowerCAmelCase	,  _lowerCAmelCase	)	)

																				shutil.rmtree(_lowerCAmelCase	)

																				# Save tokenizer rust, legacy_format=False
																				lowerCamelCase__       =       tempfile.mkdtemp()

																				lowerCamelCase__       =       tokenizer_r.save_pretrained(_lowerCAmelCase	,  legacy_format=_lowerCAmelCase	)
																				lowerCamelCase__       =       tokenizer_p.save_pretrained(_lowerCAmelCase	)

																				# Checks it saved the tokenizer.json file
																				self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files	)	)

																				# Checks everything loads correctly in the same way
																				lowerCamelCase__       =       tokenizer_r.from_pretrained(_lowerCAmelCase	)
																				lowerCamelCase__       =       tokenizer_p.from_pretrained(_lowerCAmelCase	)

																				# Check special tokens are set accordingly on Rust and Python
																				for key in tokenizer_pp.special_tokens_map:
																									self.assertTrue(hasattr(_lowerCAmelCase	,  _lowerCAmelCase	)	)

																				shutil.rmtree(_lowerCAmelCase	)

@require_torch
@require_sentencepiece
@require_tokenizers
class SCREAMING_SNAKE_CASE_ (					unittest.TestCase   ):





					"""simple docstring"""


					A__				 =    "facebook/mbart-large-50-one-to-many-mmt"
					A__				 =    [
					    " UN Chief Says There Is No Military Solution in Syria",
					    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
					]
					A__				 =    [
					    "Şeful ONU declară că nu există o soluţie militară în Siria",
					    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
					    " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
					    " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
					]
					A__				 =    [EN_CODE, 8_2_7_4, 1_2_7_8_7_3, 2_5_9_1_6, 7, 8_6_2_2, 2_0_7_1, 4_3_8, 6_7_4_8_5, 5_3, 1_8_7_8_9_5, 2_3, 5_1_7_1_2, 2]





					@classmethod
					def 					__magic_name__ ( cls	):
										lowerCamelCase__       =       MBartaaTokenizer.from_pretrained(
										    cls.checkpoint_name	,  src_lang="en_XX"	,  tgt_lang="ro_RO"	)
										lowerCamelCase__       =       1
										return cls





					def 					__magic_name__ ( self	):
										self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ar_AR"]	,  25_0001	)
										self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["en_EN"]	,  25_0004	)
										self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ro_RO"]	,  25_0020	)
										self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["mr_IN"]	,  25_0038	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.tokenizer.batch_encode_plus(self.src_text	).input_ids[0]
										self.assertListEqual(self.expected_src_tokens	,  _lowerCAmelCase	)





					def 					__magic_name__ ( self	):
										self.assertIn(_lowerCAmelCase	,  self.tokenizer.all_special_ids	)
										lowerCamelCase__       =       [RO_CODE, 884, 9019, 96, 9, 916, 8_6792, 36, 1_8743, 1_5596, 5, 2]
										lowerCamelCase__       =       self.tokenizer.decode(_lowerCAmelCase	,  skip_special_tokens=_lowerCAmelCase	)
										lowerCamelCase__       =       self.tokenizer.decode(generated_ids[1:]	,  skip_special_tokens=_lowerCAmelCase	)
										self.assertEqual(_lowerCAmelCase	,  _lowerCAmelCase	)
										self.assertNotIn(self.tokenizer.eos_token	,  _lowerCAmelCase	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       ["this is gunna be a long sentence " * 20]
										assert isinstance(src_text[0]	,  _lowerCAmelCase	)
										lowerCamelCase__       =       10
										lowerCamelCase__       =       self.tokenizer(_lowerCAmelCase	,  max_length=_lowerCAmelCase	,  truncation=_lowerCAmelCase	).input_ids[0]
										self.assertEqual(ids[0]	,  _lowerCAmelCase	)
										self.assertEqual(ids[-1]	,  2	)
										self.assertEqual(len(_lowerCAmelCase	)	,  _lowerCAmelCase	)





					def 					__magic_name__ ( self	):
										self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "ar_AR"]	)	,  [25_0053, 25_0001]	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       tempfile.mkdtemp()
										lowerCamelCase__       =       self.tokenizer.fairseq_tokens_to_ids
										self.tokenizer.save_pretrained(_lowerCAmelCase	)
										lowerCamelCase__       =       MBartaaTokenizer.from_pretrained(_lowerCAmelCase	)
										self.assertDictEqual(new_tok.fairseq_tokens_to_ids	,  _lowerCAmelCase	)





					@require_torch
					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.tokenizer(self.src_text	,  text_target=self.tgt_text	,  padding=_lowerCAmelCase	,  return_tensors="pt"	)
										lowerCamelCase__       =       shift_tokens_right(batch["labels"]	,  self.tokenizer.pad_token_id	)

										# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
										assert batch.input_ids[1][0] == EN_CODE
										assert batch.input_ids[1][-1] == 2
										assert batch.labels[1][0] == RO_CODE
										assert batch.labels[1][-1] == 2
										assert batch.decoder_input_ids[1][:2].tolist() == [2, RO_CODE]





					@require_torch
					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.tokenizer(
										    self.src_text	,  text_target=self.tgt_text	,  padding=_lowerCAmelCase	,  truncation=_lowerCAmelCase	,  max_length=len(self.expected_src_tokens	)	,  return_tensors="pt"	,  )
										lowerCamelCase__       =       shift_tokens_right(batch["labels"]	,  self.tokenizer.pad_token_id	)

										self.assertIsInstance(_lowerCAmelCase	,  _lowerCAmelCase	)

										self.assertEqual((2, 14)	,  batch.input_ids.shape	)
										self.assertEqual((2, 14)	,  batch.attention_mask.shape	)
										lowerCamelCase__       =       batch.input_ids.tolist()[0]
										self.assertListEqual(self.expected_src_tokens	,  _lowerCAmelCase	)
										self.assertEqual(2	,  batch.decoder_input_ids[0, 0]	)  # decoder_start_token_id
										# Test that special tokens are reset
										self.assertEqual(self.tokenizer.prefix_tokens	,  [EN_CODE]	)
										self.assertEqual(self.tokenizer.suffix_tokens	,  [self.tokenizer.eos_token_id]	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.tokenizer(self.src_text	,  padding=_lowerCAmelCase	,  truncation=_lowerCAmelCase	,  max_length=3	,  return_tensors="pt"	)
										lowerCamelCase__       =       self.tokenizer(
										    text_target=self.tgt_text	,  padding=_lowerCAmelCase	,  truncation=_lowerCAmelCase	,  max_length=10	,  return_tensors="pt"	)
										lowerCamelCase__       =       targets["input_ids"]
										lowerCamelCase__       =       shift_tokens_right(_lowerCAmelCase	,  self.tokenizer.pad_token_id	)

										self.assertEqual(batch.input_ids.shape[1]	,  3	)
										self.assertEqual(batch.decoder_input_ids.shape[1]	,  10	)






					@require_torch
					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.tokenizer._build_translation_inputs(
										    "A test"	,  return_tensors="pt"	,  src_lang="en_XX"	,  tgt_lang="ar_AR"	)

										self.assertEqual(
										    nested_simplify(_lowerCAmelCase	)	,  {
										        # en_XX, A, test, EOS
										        "input_ids": [[25_0004, 62, 3034, 2]],
										        "attention_mask": [[1, 1, 1, 1]],
										        # ar_AR
										        "forced_bos_token_id": 25_0001,
										    }	,  )








import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin

class SCREAMING_SNAKE_CASE_ (					unittest.TestCase							,       lowercase_   ):





					"""simple docstring"""


					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       load_tool("text-classification"	)
										self.tool.setup()
										lowerCamelCase__       =       load_tool("text-classification"	,  remote=_lowerCAmelCase	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.tool("That's quite cool"	,  ["positive", "negative"]	)
										self.assertEqual(_lowerCAmelCase	,  "positive"	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.remote_tool("That's quite cool"	,  ["positive", "negative"]	)
										self.assertEqual(_lowerCAmelCase	,  "positive"	)





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.tool(text="That's quite cool"	,  labels=["positive", "negative"]	)
										self.assertEqual(_lowerCAmelCase	,  "positive"	)






					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.remote_tool(text="That's quite cool"	,  labels=["positive", "negative"]	)
										self.assertEqual(_lowerCAmelCase	,  "positive"	)

