


import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


snake_case__			 =						get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
		from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

snake_case__			 =						25_6047
snake_case__			 =						25_6145


@require_sentencepiece
@require_tokenizers
class 					lowerCAmelCase_ (       __lowercase			,unittest.TestCase):
						lowerCamelCase_         =     NllbTokenizer
						lowerCamelCase_         =     NllbTokenizerFast
						lowerCamelCase_         =     True
						lowerCamelCase_         =     True
						lowerCamelCase_         =     {}




						def  _snake_case					(	self						:   Optional[Any]						)      ->List[str]:






									"""simple docstring"""






									super().setUp()

									# We have a SentencePiece fixture for testing
									a__ :Any						  =    NllbTokenizer(__A   ,     keep_accents=__A						)
									tokenizer.save_pretrained(self.tmpdirname						)




						def  _snake_case					(	self						:   Union[str, Any]						)      ->Optional[int]:






									"""simple docstring"""






									a__ :Tuple						  =    NllbTokenizer(__A   ,     keep_accents=__A						)

									a__ :Optional[int]						  =    tokenizer.tokenize("This is a test"						)
									self.assertListEqual(__A   ,     ["▁This", "▁is", "▁a", "▁t", "est"]						)

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(__A						)   ,     [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]   ,     )

									a__ :str						  =    tokenizer.tokenize("I was born in 92000, and this is falsé."						)
									self.assertListEqual(
									    __A   ,     [
									        SPIECE_UNDERLINE + "I",
									        SPIECE_UNDERLINE + "was",
									        SPIECE_UNDERLINE + "b",
									        "or",
									        "n",
									        SPIECE_UNDERLINE + "in",
									        SPIECE_UNDERLINE + "",
									        "9",
									        "2",
									        "0",
									        "0",
									        "0",
									        ",",
									        SPIECE_UNDERLINE + "and",
									        SPIECE_UNDERLINE + "this",
									        SPIECE_UNDERLINE + "is",
									        SPIECE_UNDERLINE + "f",
									        "al",
									        "s",
									        "é",
									        ".",
									    ]   ,     )
									a__ :Dict						  =    tokenizer.convert_tokens_to_ids(__A						)
									self.assertListEqual(
									    __A   ,     [
									        value + tokenizer.fairseq_offset
									        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
									    ]   ,     )

									a__ :Any						  =    tokenizer.convert_ids_to_tokens(__A						)
									self.assertListEqual(
									    __A   ,     [
									        SPIECE_UNDERLINE + "I",
									        SPIECE_UNDERLINE + "was",
									        SPIECE_UNDERLINE + "b",
									        "or",
									        "n",
									        SPIECE_UNDERLINE + "in",
									        SPIECE_UNDERLINE + "",
									        "<unk>",
									        "2",
									        "0",
									        "0",
									        "0",
									        ",",
									        SPIECE_UNDERLINE + "and",
									        SPIECE_UNDERLINE + "this",
									        SPIECE_UNDERLINE + "is",
									        SPIECE_UNDERLINE + "f",
									        "al",
									        "s",
									        "<unk>",
									        ".",
									    ]   ,     )




						def  _snake_case					(	self						:   Optional[Any]						)      ->str:






									"""simple docstring"""






									a__ :str						  =    (self.rust_tokenizer_class, "hf-internal-testing/tiny-random-nllb", {})
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
															a__ :Optional[Any]						  =    self.rust_tokenizer_class.from_pretrained(__A   ,     **__A						)
															a__ :List[Any]						  =    self.tokenizer_class.from_pretrained(__A   ,     **__A						)

															a__ :List[str]						  =    tempfile.mkdtemp()

															a__ :List[Any]						  =    tokenizer_r.save_pretrained(__A						)
															a__ :List[str]						  =    tokenizer_p.save_pretrained(__A						)

															# Checks it save with the same files + the tokenizer.json file for the fast one
															self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files						)						)
															a__ :str						  =    tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f						)
															self.assertSequenceEqual(__A   ,     __A						)

															# Checks everything loads correctly in the same way
															a__ :Optional[Any]						  =    tokenizer_r.from_pretrained(__A						)
															a__ :int						  =    tokenizer_p.from_pretrained(__A						)

															# Check special tokens are set accordingly on Rust and Python
															for key in tokenizer_pp.special_tokens_map:
																		self.assertTrue(hasattr(__A   ,     __A						)						)

															shutil.rmtree(__A						)

															# Save tokenizer rust, legacy_format=True
															a__ :List[str]						  =    tempfile.mkdtemp()

															a__ :Dict						  =    tokenizer_r.save_pretrained(__A   ,     legacy_format=__A						)
															a__ :int						  =    tokenizer_p.save_pretrained(__A						)

															# Checks it save with the same files
															self.assertSequenceEqual(__A   ,     __A						)

															# Checks everything loads correctly in the same way
															a__ :List[str]						  =    tokenizer_r.from_pretrained(__A						)
															a__ :List[str]						  =    tokenizer_p.from_pretrained(__A						)

															# Check special tokens are set accordingly on Rust and Python
															for key in tokenizer_pp.special_tokens_map:
																		self.assertTrue(hasattr(__A   ,     __A						)						)

															shutil.rmtree(__A						)

															# Save tokenizer rust, legacy_format=False
															a__ :Any						  =    tempfile.mkdtemp()

															a__ :Optional[Any]						  =    tokenizer_r.save_pretrained(__A   ,     legacy_format=__A						)
															a__ :List[str]						  =    tokenizer_p.save_pretrained(__A						)

															# Checks it saved the tokenizer.json file
															self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files						)						)

															# Checks everything loads correctly in the same way
															a__ :Union[str, Any]						  =    tokenizer_r.from_pretrained(__A						)
															a__ :Union[str, Any]						  =    tokenizer_p.from_pretrained(__A						)

															# Check special tokens are set accordingly on Rust and Python
															for key in tokenizer_pp.special_tokens_map:
																		self.assertTrue(hasattr(__A   ,     __A						)						)

															shutil.rmtree(__A						)




						@require_torch
						def  _snake_case					(	self						:   Optional[Any]						)      ->str:






									"""simple docstring"""






									if not self.test_seqaseq:
												return

									a__ :List[Any]						  =    self.get_tokenizers()
									for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''						):
															# Longer text that will definitely require truncation.
															a__ :Union[str, Any]						  =    [
															    " UN Chief Says There Is No Military Solution in Syria",
															    " Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for"
															    " Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons"
															    " will only worsen the violence and misery for millions of people.",
															]
															a__ :str						  =    [
															    "Şeful ONU declară că nu există o soluţie militară în Siria",
															    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"
															    " Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"
															    " că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
															]
															try:
																		a__ :Optional[int]						  =    tokenizer.prepare_seqaseq_batch(
																		    src_texts=__A   ,     tgt_texts=__A   ,     max_length=3   ,     max_target_length=10   ,     return_tensors="pt"   ,     src_lang="eng_Latn"   ,     tgt_lang="ron_Latn"   ,     )
															except NotImplementedError:
																		return
															self.assertEqual(batch.input_ids.shape[1]   ,     3						)
															self.assertEqual(batch.labels.shape[1]   ,     10						)
															# max_target_length will default to max_length if not specified
															a__ :str						  =    tokenizer.prepare_seqaseq_batch(
															    __A   ,     tgt_texts=__A   ,     max_length=3   ,     return_tensors="pt"						)
															self.assertEqual(batch.input_ids.shape[1]   ,     3						)
															self.assertEqual(batch.labels.shape[1]   ,     3						)

															a__ :str						  =    tokenizer.prepare_seqaseq_batch(
															    src_texts=__A   ,     max_length=3   ,     max_target_length=10   ,     return_tensors="pt"						)
															self.assertEqual(batch_encoder_only.input_ids.shape[1]   ,     3						)
															self.assertEqual(batch_encoder_only.attention_mask.shape[1]   ,     3						)
															self.assertNotIn("decoder_input_ids"   ,     __A						)




						@unittest.skip("Unfortunately way too slow to build a BPE with SentencePiece."						)
						def  _snake_case					(	self						:   int						)      ->Any:






									"""simple docstring"""






									pass






						def  _snake_case					(	self						:   Union[str, Any]						)      ->Optional[Any]:






									"""simple docstring"""






									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
															a__ :List[Any]						  =    [AddedToken("<special>"   ,     lstrip=__A						)]

															a__ :Union[str, Any]						  =    self.rust_tokenizer_class.from_pretrained(
															    __A   ,     additional_special_tokens=__A   ,     **__A						)
															a__ :Optional[int]						  =    tokenizer_r.encode("Hey this is a <special> token"						)

															a__ :Tuple						  =    tokenizer_r.encode("<special>"   ,     add_special_tokens=__A						)[0]

															self.assertTrue(special_token_id in r_output						)

															if self.test_slow_tokenizer:
																		a__ :Any						  =    self.rust_tokenizer_class.from_pretrained(
																		    __A   ,     additional_special_tokens=__A   ,     **__A   ,     )
																		a__ :Optional[Any]						  =    self.tokenizer_class.from_pretrained(
																		    __A   ,     additional_special_tokens=__A   ,     **__A						)

																		a__ :Optional[Any]						  =    tokenizer_p.encode("Hey this is a <special> token"						)

																		a__ :Union[str, Any]						  =    tokenizer_cr.encode("Hey this is a <special> token"						)

																		self.assertEqual(__A   ,     __A						)
																		self.assertEqual(__A   ,     __A						)
																		self.assertTrue(special_token_id in p_output						)
																		self.assertTrue(special_token_id in cr_output						)






@require_torch
@require_sentencepiece
@require_tokenizers
class 					lowerCAmelCase_ (       unittest.TestCase):
						lowerCamelCase_         =     'facebook/nllb-200-distilled-600M'
						lowerCamelCase_         =     [
						    ' UN Chief Says There Is No Military Solution in Syria',
						    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
						]
						lowerCamelCase_         =     [
						    'Şeful ONU declară că nu există o soluţie militară în Siria',
						    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
						    ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
						    ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
						]
						lowerCamelCase_         =     [
						    256_047,
						    16_297,
						    134_408,
						    8_165,
						    248_066,
						    14_734,
						    950,
						    1_135,
						    105_721,
						    3_573,
						    83,
						    27_352,
						    108,
						    49_486,
						    2,
						]




						@classmethod
						def  _snake_case					(	cls						:   Optional[int]						)      ->Optional[Any]:






									"""simple docstring"""






									a__ :Tuple						  =    NllbTokenizer.from_pretrained(
									    cls.checkpoint_name   ,     src_lang="eng_Latn"   ,     tgt_lang="ron_Latn"						)
									a__ :Union[str, Any]						  =    1
									return cls




						def  _snake_case					(	self						:   Tuple						)      ->List[Any]:






									"""simple docstring"""






									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Arab"]   ,     256001						)
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["ace_Latn"]   ,     256002						)
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["fra_Latn"]   ,     256057						)




						def  _snake_case					(	self						:   Union[str, Any]						)      ->Any:






									"""simple docstring"""






									a__ :Optional[Any]						  =    self.tokenizer.batch_encode_plus(self.src_text						).input_ids[0]
									self.assertListEqual(self.expected_src_tokens   ,     __A						)




						def  _snake_case					(	self						:   int						)      ->List[str]:






									"""simple docstring"""






									self.assertIn(__A   ,     self.tokenizer.all_special_ids						)
									# fmt: off
									a__ :Union[str, Any]						  =    [RO_CODE, 4254, 98068, 112923, 39072, 3909, 713, 102767, 26, 17314, 35642, 14683, 33118, 2022, 66987, 2, 256047]
									# fmt: on

									a__ :str						  =    self.tokenizer.decode(__A   ,     skip_special_tokens=__A						)
									a__ :Union[str, Any]						  =    self.tokenizer.decode(generated_ids[1:]   ,     skip_special_tokens=__A						)
									self.assertEqual(__A   ,     __A						)
									self.assertNotIn(self.tokenizer.eos_token   ,     __A						)




						def  _snake_case					(	self						:   Tuple						)      ->Optional[Any]:






									"""simple docstring"""






									a__ :int						  =    ["this is gunna be a long sentence " * 20]
									assert isinstance(src_text[0]   ,     __A						)
									a__ :Tuple						  =    10
									a__ :str						  =    self.tokenizer(__A   ,     max_length=__A   ,     truncation=__A						).input_ids[0]
									self.assertEqual(ids[-1]   ,     2						)
									self.assertEqual(ids[0]   ,     __A						)
									self.assertEqual(len(__A						)   ,     __A						)




						def  _snake_case					(	self						:   str						)      ->Optional[Any]:






									"""simple docstring"""






									self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "ar_AR"]						)   ,     [256203, 3]						)




						def  _snake_case					(	self						:   Dict						)      ->Optional[int]:






									"""simple docstring"""






									a__ :Any						  =    tempfile.mkdtemp()
									a__ :Dict						  =    self.tokenizer.fairseq_tokens_to_ids
									self.tokenizer.save_pretrained(__A						)
									a__ :Optional[Any]						  =    NllbTokenizer.from_pretrained(__A						)
									self.assertDictEqual(new_tok.fairseq_tokens_to_ids   ,     __A						)




						@require_torch
						def  _snake_case					(	self						:   Optional[Any]						)      ->List[str]:






									"""simple docstring"""






									a__ :Optional[Any]						  =    self.tokenizer(
									    self.src_text   ,     text_target=self.tgt_text   ,     padding=__A   ,     truncation=__A   ,     max_length=len(self.expected_src_tokens						)   ,     return_tensors="pt"   ,     )
									a__ :List[str]						  =    shift_tokens_right(
									    batch["labels"]   ,     self.tokenizer.pad_token_id   ,     self.tokenizer.lang_code_to_id["ron_Latn"]						)

									self.assertIsInstance(__A   ,     __A						)

									self.assertEqual((2, 15)   ,     batch.input_ids.shape						)
									self.assertEqual((2, 15)   ,     batch.attention_mask.shape						)
									a__ :List[str]						  =    batch.input_ids.tolist()[0]
									self.assertListEqual(self.expected_src_tokens   ,     __A						)
									self.assertEqual(__A   ,     batch.decoder_input_ids[0, 0]						)  # EOS
									# Test that special tokens are reset
									self.assertEqual(self.tokenizer.prefix_tokens   ,     [EN_CODE]						)
									self.assertEqual(self.tokenizer.suffix_tokens   ,     [self.tokenizer.eos_token_id]						)




						def  _snake_case					(	self						:   Union[str, Any]						)      ->Union[str, Any]:






									"""simple docstring"""






									a__ :Tuple						  =    self.tokenizer(self.src_text   ,     padding=__A   ,     truncation=__A   ,     max_length=3   ,     return_tensors="pt"						)
									a__ :Optional[int]						  =    self.tokenizer(
									    text_target=self.tgt_text   ,     padding=__A   ,     truncation=__A   ,     max_length=10   ,     return_tensors="pt"						)
									a__ :List[str]						  =    targets["input_ids"]
									a__ :str						  =    shift_tokens_right(
									    __A   ,     self.tokenizer.pad_token_id   ,     decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]   ,     )

									self.assertEqual(batch.input_ids.shape[1]   ,     3						)
									self.assertEqual(batch.decoder_input_ids.shape[1]   ,     10						)




						@require_torch
						def  _snake_case					(	self						:   Optional[Any]						)      ->List[str]:






									"""simple docstring"""






									a__ :Union[str, Any]						  =    self.tokenizer._build_translation_inputs(
									    "A test"   ,     return_tensors="pt"   ,     src_lang="eng_Latn"   ,     tgt_lang="fra_Latn"						)

									self.assertEqual(
									    nested_simplify(__A						)   ,     {
									        # A, test, EOS, en_XX
									        "input_ids": [[256047, 70, 7356, 2]],
									        "attention_mask": [[1, 1, 1, 1]],
									        # ar_AR
									        "forced_bos_token_id": 256057,
									    }   ,     )






						@require_torch
						def  _snake_case					(	self						:   int						)      ->List[Any]:






									"""simple docstring"""






									a__ :Any						  =    True
									a__ :Any						  =    self.tokenizer(
									    "UN Chief says there is no military solution in Syria"   ,     src_lang="eng_Latn"   ,     tgt_lang="fra_Latn"						)
									self.assertEqual(
									    inputs.input_ids   ,     [16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2, 256047]						)

									a__ :Dict						  =    False
									a__ :Tuple						  =    self.tokenizer(
									    "UN Chief says there is no military solution in Syria"   ,     src_lang="eng_Latn"   ,     tgt_lang="fra_Latn"						)
									self.assertEqual(
									    inputs.input_ids   ,     [256047, 16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2]						)






import numpy as np


def 				lowerCamelCase__	(      a	:   np.ndarray	,  a	:   np.ndarray	,  a	:   float = 1e-12	,  a	:   int = 100	,  )    ->	tuple[float, np.ndarray]:





			"""simple docstring"""
			assert np.shape(a				)[0] == np.shape(a				)[1]
			# Ensure proper dimensionality.
			assert np.shape(a				)[0] == np.shape(a				)[0]
			# Ensure inputs are either both complex or both real
			assert np.iscomplexobj(a				) == np.iscomplexobj(a				)
			a__ :List[str]						  =    np.iscomplexobj(a				)
			if is_complex:
						# Ensure complex input_matrix is Hermitian
						assert np.array_equal(a	,  input_matrix.conj().T				)

			# Set convergence to False. Will define convergence when we exceed max_iterations
			# or when we have small changes from one iteration to next.

			a__ :List[str]						  =    False
			a__ :str						  =    0
			a__ :Dict						  =    0
			a__ :Tuple						  =    1e12

			while not convergence:
						# Multiple matrix by the vector.
						a__ :Tuple						  =    np.dot(a	,  a				)
						# Normalize the resulting output vector.
						a__ :List[Any]						  =    w / np.linalg.norm(a				)
						# Find rayleigh quotient
						# (faster than usual b/c we know vector is normalized already)
						a__ :List[str]						  =    vector.conj().T if is_complex else vector.T
						a__ :Any						  =    np.dot(a	,  np.dot(a	,  a				)				)

						# Check convergence.
						a__ :Optional[Any]						  =    np.abs(lambda_ - lambda_previous				) / lambda_
						iterations += 1

						if error <= error_tol or iterations >= max_iterations:
									a__ :List[str]						  =    True

						a__ :Optional[int]						  =    lambda_

			if is_complex:
						a__ :Union[str, Any]						  =    np.real(lambda_				)

			return lambda_, vector


def 				lowerCamelCase__	(      )    ->	None:





			"""simple docstring"""
			a__ :List[str]						  =    np.array([[41, 4, 20], [4, 26, 30], [20, 30, 50]]				)
			a__ :Dict						  =    np.array([41, 4, 20]				)
			a__ :Union[str, Any]						  =    real_input_matrix.astype(np.complexaaa				)
			a__ :Optional[Any]						  =    np.triu(1j * complex_input_matrix	,  1				)
			complex_input_matrix += imag_matrix
			complex_input_matrix += -1 * imag_matrix.T
			a__ :Union[str, Any]						  =    np.array([41, 4, 20]				).astype(np.complexaaa				)

			for problem_type in ["real", "complex"]:
						if problem_type == "real":
									a__ :Union[str, Any]						  =    real_input_matrix
									a__ :int						  =    real_vector
						elif problem_type == "complex":
									a__ :Optional[Any]						  =    complex_input_matrix
									a__ :List[Any]						  =    complex_vector

						# Our implementation.
						a__  ,  a__ :Any						  =    power_iteration(a	,  a				)

						# Numpy implementation.

						# Get eigenvalues and eigenvectors using built-in numpy
						# eigh (eigh used for symmetric or hermetian matrices).
						a__  ,  a__ :int						  =    np.linalg.eigh(a				)
						# Last eigenvalue is the maximum one.
						a__ :int						  =    eigen_values[-1]
						# Last column in this matrix is eigenvector corresponding to largest eigenvalue.
						a__ :Optional[Any]						  =    eigen_vectors[:, -1]

						# Check our implementation and numpy gives close answers.
						assert np.abs(eigen_value - eigen_value_max				) <= 1e-6
						# Take absolute values element wise of each eigenvector.
						# as they are only unique to a minus sign.
						assert np.linalg.norm(np.abs(a				) - np.abs(a				)				) <= 1e-6


if __name__ == "__main__":
		import doctest

		doctest.testmod()
		test_power_iteration()


