



import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE__   			=   get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
							from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

SCREAMING_SNAKE_CASE__   			=   2_5_6_0_4_7
SCREAMING_SNAKE_CASE__   			=   2_5_6_1_4_5


@require_sentencepiece
@require_tokenizers
class   __lowerCAmelCase  (		UpperCAmelCase_				,  unittest.TestCase   ):
					"""simple docstring"""






					A__					:		int      							=     NllbTokenizer
					A__					:		Optional[int]      							=     NllbTokenizerFast
					A__					:		Optional[int]      							=     True
					A__					:		Tuple      							=     True
					A__					:		Optional[int]      							=     {}



					def      _a				(  self       :							Any       ):






								"""simple docstring"""


								super().setUp()

								# We have a SentencePiece fixture for testing
								A__	  =     NllbTokenizer(_snake_case					,       keep_accents=_snake_case       )
								tokenizer.save_pretrained(self.tmpdirname       )



					def      _a				(  self       :							Optional[int]       ):






								"""simple docstring"""


								A__	  =     NllbTokenizer(_snake_case					,       keep_accents=_snake_case       )

								A__	  =     tokenizer.tokenize('This is a test'       )
								self.assertListEqual(_snake_case					,       ['▁This', '▁is', '▁a', '▁t', 'est']       )

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(_snake_case       )					,       [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]					,       )

								A__	  =     tokenizer.tokenize('I was born in 92000, and this is falsé.'       )
								self.assertListEqual(
								    _snake_case					,       [
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '9',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        's',
								        'é',
								        '.',
								    ]					,       )
								A__	  =     tokenizer.convert_tokens_to_ids(_snake_case       )
								self.assertListEqual(
								    _snake_case					,       [
								        value + tokenizer.fairseq_offset
								        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
								    ]					,       )

								A__	  =     tokenizer.convert_ids_to_tokens(_snake_case       )
								self.assertListEqual(
								    _snake_case					,       [
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '<unk>',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        's',
								        '<unk>',
								        '.',
								    ]					,       )



					def      _a				(  self       :							Any       ):






								"""simple docstring"""


								A__	  =     (self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
														A__	  =     self.rust_tokenizer_class.from_pretrained(_snake_case					,       **_snake_case       )
														A__	  =     self.tokenizer_class.from_pretrained(_snake_case					,       **_snake_case       )

														A__	  =     tempfile.mkdtemp()

														A__	  =     tokenizer_r.save_pretrained(_snake_case       )
														A__	  =     tokenizer_p.save_pretrained(_snake_case       )

														# Checks it save with the same files + the tokenizer.json file for the fast one
														self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files       )       )
														A__	  =     tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f       )
														self.assertSequenceEqual(_snake_case					,       _snake_case       )

														# Checks everything loads correctly in the same way
														A__	  =     tokenizer_r.from_pretrained(_snake_case       )
														A__	  =     tokenizer_p.from_pretrained(_snake_case       )

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																	self.assertTrue(hasattr(_snake_case					,       _snake_case       )       )

														shutil.rmtree(_snake_case       )

														# Save tokenizer rust, legacy_format=True
														A__	  =     tempfile.mkdtemp()

														A__	  =     tokenizer_r.save_pretrained(_snake_case					,       legacy_format=_snake_case       )
														A__	  =     tokenizer_p.save_pretrained(_snake_case       )

														# Checks it save with the same files
														self.assertSequenceEqual(_snake_case					,       _snake_case       )

														# Checks everything loads correctly in the same way
														A__	  =     tokenizer_r.from_pretrained(_snake_case       )
														A__	  =     tokenizer_p.from_pretrained(_snake_case       )

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																	self.assertTrue(hasattr(_snake_case					,       _snake_case       )       )

														shutil.rmtree(_snake_case       )

														# Save tokenizer rust, legacy_format=False
														A__	  =     tempfile.mkdtemp()

														A__	  =     tokenizer_r.save_pretrained(_snake_case					,       legacy_format=_snake_case       )
														A__	  =     tokenizer_p.save_pretrained(_snake_case       )

														# Checks it saved the tokenizer.json file
														self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files       )       )

														# Checks everything loads correctly in the same way
														A__	  =     tokenizer_r.from_pretrained(_snake_case       )
														A__	  =     tokenizer_p.from_pretrained(_snake_case       )

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																	self.assertTrue(hasattr(_snake_case					,       _snake_case       )       )

														shutil.rmtree(_snake_case       )



					@require_torch
					def      _a				(  self       :							str       ):






								"""simple docstring"""


								if not self.test_seqaseq:
											return

								A__	  =     self.get_tokenizers()
								for tokenizer in tokenizers:
											with self.subTest(F'''{tokenizer.__class__.__name__}'''       ):
														# Longer text that will definitely require truncation.
														A__	  =     [
														    ' UN Chief Says There Is No Military Solution in Syria',
														    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
														    ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
														    ' will only worsen the violence and misery for millions of people.',
														]
														A__	  =     [
														    'Şeful ONU declară că nu există o soluţie militară în Siria',
														    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
														    ' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'
														    ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
														]
														try:
																	A__	  =     tokenizer.prepare_seqaseq_batch(
																	    src_texts=_snake_case					,       tgt_texts=_snake_case					,       max_length=3					,       max_target_length=10					,       return_tensors='pt'					,       src_lang='eng_Latn'					,       tgt_lang='ron_Latn'					,       )
														except NotImplementedError:
																	return
														self.assertEqual(batch.input_ids.shape[1]					,       3       )
														self.assertEqual(batch.labels.shape[1]					,       10       )
														# max_target_length will default to max_length if not specified
														A__	  =     tokenizer.prepare_seqaseq_batch(
														    _snake_case					,       tgt_texts=_snake_case					,       max_length=3					,       return_tensors='pt'       )
														self.assertEqual(batch.input_ids.shape[1]					,       3       )
														self.assertEqual(batch.labels.shape[1]					,       3       )

														A__	  =     tokenizer.prepare_seqaseq_batch(
														    src_texts=_snake_case					,       max_length=3					,       max_target_length=10					,       return_tensors='pt'       )
														self.assertEqual(batch_encoder_only.input_ids.shape[1]					,       3       )
														self.assertEqual(batch_encoder_only.attention_mask.shape[1]					,       3       )
														self.assertNotIn('decoder_input_ids'					,       _snake_case       )



					@unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'       )
					def      _a				(  self       :							Dict       ):






								"""simple docstring"""


								pass



					def      _a				(  self       :							List[Any]       ):






								"""simple docstring"""


								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
														A__	  =     [AddedToken('<special>'					,       lstrip=_snake_case       )]

														A__	  =     self.rust_tokenizer_class.from_pretrained(
														    _snake_case					,       additional_special_tokens=_snake_case					,       **_snake_case       )
														A__	  =     tokenizer_r.encode('Hey this is a <special> token'       )

														A__	  =     tokenizer_r.encode('<special>'					,       add_special_tokens=_snake_case       )[0]

														self.assertTrue(special_token_id in r_output       )

														if self.test_slow_tokenizer:
																	A__	  =     self.rust_tokenizer_class.from_pretrained(
																	    _snake_case					,       additional_special_tokens=_snake_case					,       **_snake_case					,       )
																	A__	  =     self.tokenizer_class.from_pretrained(
																	    _snake_case					,       additional_special_tokens=_snake_case					,       **_snake_case       )

																	A__	  =     tokenizer_p.encode('Hey this is a <special> token'       )

																	A__	  =     tokenizer_cr.encode('Hey this is a <special> token'       )

																	self.assertEqual(_snake_case					,       _snake_case       )
																	self.assertEqual(_snake_case					,       _snake_case       )
																	self.assertTrue(special_token_id in p_output       )
																	self.assertTrue(special_token_id in cr_output       )







@require_torch
@require_sentencepiece
@require_tokenizers
class   __lowerCAmelCase  (		unittest.TestCase   ):
					"""simple docstring"""






					A__					:		int      							=     "facebook/nllb-200-distilled-600M"
					A__					:		List[Any]      							=     [
					    " UN Chief Says There Is No Military Solution in Syria",
					    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
					]
					A__					:		Dict      							=     [
					    "Şeful ONU declară că nu există o soluţie militară în Siria",
					    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
					    " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
					    " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
					]
					A__					:		List[Any]      							=     [
					    25_60_47,
					    1_62_97,
					    13_44_08,
					    81_65,
					    24_80_66,
					    1_47_34,
					    9_50,
					    11_35,
					    10_57_21,
					    35_73,
					    83,
					    2_73_52,
					    1_08,
					    4_94_86,
					    2,
					]



					@classmethod
					def      _a				(  cls       :							Tuple       ):






								"""simple docstring"""


								A__	  =     NllbTokenizer.from_pretrained(
								    cls.checkpoint_name					,       src_lang='eng_Latn'					,       tgt_lang='ron_Latn'       )
								A__	  =     1
								return cls



					def      _a				(  self       :							Optional[Any]       ):






								"""simple docstring"""


								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']					,       25_60_01       )
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']					,       25_60_02       )
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']					,       25_60_57       )



					def      _a				(  self       :							str       ):






								"""simple docstring"""


								A__	  =     self.tokenizer.batch_encode_plus(self.src_text       ).input_ids[0]
								self.assertListEqual(self.expected_src_tokens					,       _snake_case       )



					def      _a				(  self       :							Tuple       ):






								"""simple docstring"""


								self.assertIn(_snake_case					,       self.tokenizer.all_special_ids       )
								# fmt: off
								A__	  =     [RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
								# fmt: on

								A__	  =     self.tokenizer.decode(_snake_case					,       skip_special_tokens=_snake_case       )
								A__	  =     self.tokenizer.decode(generated_ids[1:]					,       skip_special_tokens=_snake_case       )
								self.assertEqual(_snake_case					,       _snake_case       )
								self.assertNotIn(self.tokenizer.eos_token					,       _snake_case       )



					def      _a				(  self       :							Dict       ):






								"""simple docstring"""


								A__	  =     ['this is gunna be a long sentence ' * 20]
								assert isinstance(src_text[0]					,       _snake_case       )
								A__	  =     10
								A__	  =     self.tokenizer(_snake_case					,       max_length=_snake_case					,       truncation=_snake_case       ).input_ids[0]
								self.assertEqual(ids[-1]					,       2       )
								self.assertEqual(ids[0]					,       _snake_case       )
								self.assertEqual(len(_snake_case       )					,       _snake_case       )



					def      _a				(  self       :							str       ):






								"""simple docstring"""


								self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']       )					,       [25_62_03, 3]       )



					def      _a				(  self       :							Union[str, Any]       ):






								"""simple docstring"""


								A__	  =     tempfile.mkdtemp()
								A__	  =     self.tokenizer.fairseq_tokens_to_ids
								self.tokenizer.save_pretrained(_snake_case       )
								A__	  =     NllbTokenizer.from_pretrained(_snake_case       )
								self.assertDictEqual(new_tok.fairseq_tokens_to_ids					,       _snake_case       )



					@require_torch
					def      _a				(  self       :							Union[str, Any]       ):






								"""simple docstring"""


								A__	  =     self.tokenizer(
								    self.src_text					,       text_target=self.tgt_text					,       padding=_snake_case					,       truncation=_snake_case					,       max_length=len(self.expected_src_tokens       )					,       return_tensors='pt'					,       )
								A__	  =     shift_tokens_right(
								    batch['labels']					,       self.tokenizer.pad_token_id					,       self.tokenizer.lang_code_to_id['ron_Latn']       )

								self.assertIsInstance(_snake_case					,       _snake_case       )

								self.assertEqual((2, 15)					,       batch.input_ids.shape       )
								self.assertEqual((2, 15)					,       batch.attention_mask.shape       )
								A__	  =     batch.input_ids.tolist()[0]
								self.assertListEqual(self.expected_src_tokens					,       _snake_case       )
								self.assertEqual(_snake_case					,       batch.decoder_input_ids[0, 0]       )  # EOS
								# Test that special tokens are reset
								self.assertEqual(self.tokenizer.prefix_tokens					,       [EN_CODE]       )
								self.assertEqual(self.tokenizer.suffix_tokens					,       [self.tokenizer.eos_token_id]       )



					def      _a				(  self       :							List[Any]       ):






								"""simple docstring"""


								A__	  =     self.tokenizer(self.src_text					,       padding=_snake_case					,       truncation=_snake_case					,       max_length=3					,       return_tensors='pt'       )
								A__	  =     self.tokenizer(
								    text_target=self.tgt_text					,       padding=_snake_case					,       truncation=_snake_case					,       max_length=10					,       return_tensors='pt'       )
								A__	  =     targets['input_ids']
								A__	  =     shift_tokens_right(
								    _snake_case					,       self.tokenizer.pad_token_id					,       decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]					,       )

								self.assertEqual(batch.input_ids.shape[1]					,       3       )
								self.assertEqual(batch.decoder_input_ids.shape[1]					,       10       )



					@require_torch
					def      _a				(  self       :							str       ):






								"""simple docstring"""


								A__	  =     self.tokenizer._build_translation_inputs(
								    'A test'					,       return_tensors='pt'					,       src_lang='eng_Latn'					,       tgt_lang='fra_Latn'       )

								self.assertEqual(
								    nested_simplify(_snake_case       )					,       {
								        # A, test, EOS, en_XX
								        'input_ids': [[25_60_47, 70, 73_56, 2]],
								        'attention_mask': [[1, 1, 1, 1]],
								        # ar_AR
								        'forced_bos_token_id': 25_60_57,
								    }					,       )



					@require_torch
					def      _a				(  self       :							List[str]       ):






								"""simple docstring"""


								A__	  =     True
								A__	  =     self.tokenizer(
								    'UN Chief says there is no military solution in Syria'					,       src_lang='eng_Latn'					,       tgt_lang='fra_Latn'       )
								self.assertEqual(
								    inputs.input_ids					,       [1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]       )

								A__	  =     False
								A__	  =     self.tokenizer(
								    'UN Chief says there is no military solution in Syria'					,       src_lang='eng_Latn'					,       tgt_lang='fra_Latn'       )
								self.assertEqual(
								    inputs.input_ids					,       [25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]       )









def A    (							__UpperCamelCase			,						__UpperCamelCase			,						__UpperCamelCase       )    ->   int:

			if exponent == 1:
						return base
			if exponent % 2 == 0:
						A__	  =     _modexpt(__UpperCamelCase			,						exponent // 2			,						__UpperCamelCase       ) % modulo_value
						return (x * x) % modulo_value
			else:
						return (base * _modexpt(__UpperCamelCase			,						exponent - 1			,						__UpperCamelCase       )) % modulo_value


def A    (							__UpperCamelCase = 1_777			,						__UpperCamelCase = 1_855			,						__UpperCamelCase = 8       )    ->   int:
			A__	  =     base
			for _ in range(1			,						__UpperCamelCase       ):
						A__	  =     _modexpt(__UpperCamelCase			,						__UpperCamelCase			,						10**digits       )

			return result


if __name__ == "__main__":
							print(f'{solution() = }')




