

'''simple docstring'''





import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


A_	= get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
				from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

A_	= 256_047
A_	= 256_145


@require_sentencepiece
@require_tokenizers
class  UpperCAmelCase (							snake_case_  , unittest.TestCase    ):

		'''simple docstring'''






		SCREAMING_SNAKE_CASE_           =					NllbTokenizer
		SCREAMING_SNAKE_CASE_           =					NllbTokenizerFast
		SCREAMING_SNAKE_CASE_           =					True
		SCREAMING_SNAKE_CASE_           =					True
		SCREAMING_SNAKE_CASE_           =					{}
		def 			UpperCamelCase(					self		)  ->     Dict:






							'''simple docstring'''



							super().setUp()

							# We have a SentencePiece fixture for testing
							lowerCamelCase_  				=      NllbTokenizer(lowercase__   ,			keep_accents=lowercase__		)
							tokenizer.save_pretrained(self.tmpdirname		)
		def 			UpperCamelCase(					self		)  ->     Dict:






							'''simple docstring'''



							lowerCamelCase_  				=      NllbTokenizer(lowercase__   ,			keep_accents=lowercase__		)

							lowerCamelCase_  				=      tokenizer.tokenize('This is a test'		)
							self.assertListEqual(lowercase__   ,			['▁This', '▁is', '▁a', '▁t', 'est']		)

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(lowercase__		)   ,			[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]   ,			)

							lowerCamelCase_  				=      tokenizer.tokenize('I was born in 92000, and this is falsé.'		)
							self.assertListEqual(
							    lowercase__   ,			[
							        SPIECE_UNDERLINE + 'I',
							        SPIECE_UNDERLINE + 'was',
							        SPIECE_UNDERLINE + 'b',
							        'or',
							        'n',
							        SPIECE_UNDERLINE + 'in',
							        SPIECE_UNDERLINE + '',
							        '9',
							        '2',
							        '0',
							        '0',
							        '0',
							        ',',
							        SPIECE_UNDERLINE + 'and',
							        SPIECE_UNDERLINE + 'this',
							        SPIECE_UNDERLINE + 'is',
							        SPIECE_UNDERLINE + 'f',
							        'al',
							        's',
							        'é',
							        '.',
							    ]   ,			)
							lowerCamelCase_  				=      tokenizer.convert_tokens_to_ids(lowercase__		)
							self.assertListEqual(
							    lowercase__   ,			[
							        value + tokenizer.fairseq_offset
							        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
							    ]   ,			)

							lowerCamelCase_  				=      tokenizer.convert_ids_to_tokens(lowercase__		)
							self.assertListEqual(
							    lowercase__   ,			[
							        SPIECE_UNDERLINE + 'I',
							        SPIECE_UNDERLINE + 'was',
							        SPIECE_UNDERLINE + 'b',
							        'or',
							        'n',
							        SPIECE_UNDERLINE + 'in',
							        SPIECE_UNDERLINE + '',
							        '<unk>',
							        '2',
							        '0',
							        '0',
							        '0',
							        ',',
							        SPIECE_UNDERLINE + 'and',
							        SPIECE_UNDERLINE + 'this',
							        SPIECE_UNDERLINE + 'is',
							        SPIECE_UNDERLINE + 'f',
							        'al',
							        's',
							        '<unk>',
							        '.',
							    ]   ,			)
		def 			UpperCamelCase(					self		)  ->     Optional[Any]:






							'''simple docstring'''



							lowerCamelCase_  				=      (self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
																	lowerCamelCase_  				=      self.rust_tokenizer_class.from_pretrained(lowercase__   ,			**lowercase__		)
																	lowerCamelCase_  				=      self.tokenizer_class.from_pretrained(lowercase__   ,			**lowercase__		)

																	lowerCamelCase_  				=      tempfile.mkdtemp()

																	lowerCamelCase_  				=      tokenizer_r.save_pretrained(lowercase__		)
																	lowerCamelCase_  				=      tokenizer_p.save_pretrained(lowercase__		)

																	# Checks it save with the same files + the tokenizer.json file for the fast one
																	self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files		)		)
																	lowerCamelCase_  				=      tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f		)
																	self.assertSequenceEqual(lowercase__   ,			lowercase__		)

																	# Checks everything loads correctly in the same way
																	lowerCamelCase_  				=      tokenizer_r.from_pretrained(lowercase__		)
																	lowerCamelCase_  				=      tokenizer_p.from_pretrained(lowercase__		)

																	# Check special tokens are set accordingly on Rust and Python
																	for key in tokenizer_pp.special_tokens_map:
																						self.assertTrue(hasattr(lowercase__   ,			lowercase__		)		)

																	shutil.rmtree(lowercase__		)

																	# Save tokenizer rust, legacy_format=True
																	lowerCamelCase_  				=      tempfile.mkdtemp()

																	lowerCamelCase_  				=      tokenizer_r.save_pretrained(lowercase__   ,			legacy_format=lowercase__		)
																	lowerCamelCase_  				=      tokenizer_p.save_pretrained(lowercase__		)

																	# Checks it save with the same files
																	self.assertSequenceEqual(lowercase__   ,			lowercase__		)

																	# Checks everything loads correctly in the same way
																	lowerCamelCase_  				=      tokenizer_r.from_pretrained(lowercase__		)
																	lowerCamelCase_  				=      tokenizer_p.from_pretrained(lowercase__		)

																	# Check special tokens are set accordingly on Rust and Python
																	for key in tokenizer_pp.special_tokens_map:
																						self.assertTrue(hasattr(lowercase__   ,			lowercase__		)		)

																	shutil.rmtree(lowercase__		)

																	# Save tokenizer rust, legacy_format=False
																	lowerCamelCase_  				=      tempfile.mkdtemp()

																	lowerCamelCase_  				=      tokenizer_r.save_pretrained(lowercase__   ,			legacy_format=lowercase__		)
																	lowerCamelCase_  				=      tokenizer_p.save_pretrained(lowercase__		)

																	# Checks it saved the tokenizer.json file
																	self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files		)		)

																	# Checks everything loads correctly in the same way
																	lowerCamelCase_  				=      tokenizer_r.from_pretrained(lowercase__		)
																	lowerCamelCase_  				=      tokenizer_p.from_pretrained(lowercase__		)

																	# Check special tokens are set accordingly on Rust and Python
																	for key in tokenizer_pp.special_tokens_map:
																						self.assertTrue(hasattr(lowercase__   ,			lowercase__		)		)

																	shutil.rmtree(lowercase__		)
		@require_torch
		def 			UpperCamelCase(					self		)  ->     Any:






							'''simple docstring'''



							if not self.test_seqaseq:
												return

							lowerCamelCase_  				=      self.get_tokenizers()
							for tokenizer in tokenizers:
												with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
																	# Longer text that will definitely require truncation.
																	lowerCamelCase_  				=      [
																	    """ UN Chief Says There Is No Military Solution in Syria""",
																	    """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
																	    """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
																	    """ will only worsen the violence and misery for millions of people.""",
																	]
																	lowerCamelCase_  				=      [
																	    """Şeful ONU declară că nu există o soluţie militară în Siria""",
																	    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
																	    """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
																	    """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
																	]
																	try:
																						lowerCamelCase_  				=      tokenizer.prepare_seqaseq_batch(
																						    src_texts=lowercase__   ,			tgt_texts=lowercase__   ,			max_length=3   ,			max_target_length=10   ,			return_tensors='pt'   ,			src_lang='eng_Latn'   ,			tgt_lang='ron_Latn'   ,			)
																	except NotImplementedError:
																						return
																	self.assertEqual(batch.input_ids.shape[1]   ,			3		)
																	self.assertEqual(batch.labels.shape[1]   ,			10		)
																	# max_target_length will default to max_length if not specified
																	lowerCamelCase_  				=      tokenizer.prepare_seqaseq_batch(
																	    lowercase__   ,			tgt_texts=lowercase__   ,			max_length=3   ,			return_tensors='pt'		)
																	self.assertEqual(batch.input_ids.shape[1]   ,			3		)
																	self.assertEqual(batch.labels.shape[1]   ,			3		)

																	lowerCamelCase_  				=      tokenizer.prepare_seqaseq_batch(
																	    src_texts=lowercase__   ,			max_length=3   ,			max_target_length=10   ,			return_tensors='pt'		)
																	self.assertEqual(batch_encoder_only.input_ids.shape[1]   ,			3		)
																	self.assertEqual(batch_encoder_only.attention_mask.shape[1]   ,			3		)
																	self.assertNotIn('decoder_input_ids'   ,			lowercase__		)
		@unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'		)
		def 			UpperCamelCase(					self		)  ->     Optional[Any]:






							'''simple docstring'''



							pass






		def 			UpperCamelCase(					self		)  ->     Tuple:






							'''simple docstring'''



							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
																	lowerCamelCase_  				=      [AddedToken('<special>'   ,			lstrip=lowercase__		)]

																	lowerCamelCase_  				=      self.rust_tokenizer_class.from_pretrained(
																	    lowercase__   ,			additional_special_tokens=lowercase__   ,			**lowercase__		)
																	lowerCamelCase_  				=      tokenizer_r.encode('Hey this is a <special> token'		)

																	lowerCamelCase_  				=      tokenizer_r.encode('<special>'   ,			add_special_tokens=lowercase__		)[0]

																	self.assertTrue(special_token_id in r_output		)

																	if self.test_slow_tokenizer:
																						lowerCamelCase_  				=      self.rust_tokenizer_class.from_pretrained(
																						    lowercase__   ,			additional_special_tokens=lowercase__   ,			**lowercase__   ,			)
																						lowerCamelCase_  				=      self.tokenizer_class.from_pretrained(
																						    lowercase__   ,			additional_special_tokens=lowercase__   ,			**lowercase__		)

																						lowerCamelCase_  				=      tokenizer_p.encode('Hey this is a <special> token'		)

																						lowerCamelCase_  				=      tokenizer_cr.encode('Hey this is a <special> token'		)

																						self.assertEqual(lowercase__   ,			lowercase__		)
																						self.assertEqual(lowercase__   ,			lowercase__		)
																						self.assertTrue(special_token_id in p_output		)
																						self.assertTrue(special_token_id in cr_output		)


@require_torch
@require_sentencepiece
@require_tokenizers
class  UpperCAmelCase (							unittest.TestCase    ):

		'''simple docstring'''






		SCREAMING_SNAKE_CASE_           =					'''facebook/nllb-200-distilled-600M'''
		SCREAMING_SNAKE_CASE_           =					[
		    ''' UN Chief Says There Is No Military Solution in Syria''',
		    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
		]
		SCREAMING_SNAKE_CASE_           =					[
		    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
		    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
		    ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
		    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
		]
		SCREAMING_SNAKE_CASE_           =					[
		    25_60_47,
		    1_62_97,
		    13_44_08,
		    81_65,
		    24_80_66,
		    1_47_34,
		    9_50,
		    11_35,
		    10_57_21,
		    35_73,
		    83,
		    2_73_52,
		    1_08,
		    4_94_86,
		    2,
		]
		@classmethod
		def 			UpperCamelCase(					cls		)  ->     List[str]:






							'''simple docstring'''



							lowerCamelCase_  				=      NllbTokenizer.from_pretrained(
							    cls.checkpoint_name   ,			src_lang='eng_Latn'   ,			tgt_lang='ron_Latn'		)
							lowerCamelCase_  				=      1
							return cls
		def 			UpperCamelCase(					self		)  ->     int:






							'''simple docstring'''



							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']   ,			256001		)
							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']   ,			256002		)
							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']   ,			256057		)
		def 			UpperCamelCase(					self		)  ->     str:






							'''simple docstring'''



							lowerCamelCase_  				=      self.tokenizer.batch_encode_plus(self.src_text		).input_ids[0]
							self.assertListEqual(self.expected_src_tokens   ,			lowercase__		)
		def 			UpperCamelCase(					self		)  ->     List[str]:






							'''simple docstring'''



							self.assertIn(lowercase__   ,			self.tokenizer.all_special_ids		)
							# fmt: off
							lowerCamelCase_  				=      [RO_CODE, 4254, 98068, 112923, 39072, 3909, 713, 102767, 26, 17314, 35642, 14683, 33118, 2022, 66987, 2, 256047]
							# fmt: on

							lowerCamelCase_  				=      self.tokenizer.decode(lowercase__   ,			skip_special_tokens=lowercase__		)
							lowerCamelCase_  				=      self.tokenizer.decode(generated_ids[1:]   ,			skip_special_tokens=lowercase__		)
							self.assertEqual(lowercase__   ,			lowercase__		)
							self.assertNotIn(self.tokenizer.eos_token   ,			lowercase__		)
		def 			UpperCamelCase(					self		)  ->     Optional[Any]:






							'''simple docstring'''



							lowerCamelCase_  				=      ["""this is gunna be a long sentence """ * 20]
							assert isinstance(src_text[0]   ,			lowercase__		)
							lowerCamelCase_  				=      10
							lowerCamelCase_  				=      self.tokenizer(lowercase__   ,			max_length=lowercase__   ,			truncation=lowercase__		).input_ids[0]
							self.assertEqual(ids[-1]   ,			2		)
							self.assertEqual(ids[0]   ,			lowercase__		)
							self.assertEqual(len(lowercase__		)   ,			lowercase__		)
		def 			UpperCamelCase(					self		)  ->     Dict:






							'''simple docstring'''



							self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']		)   ,			[256203, 3]		)
		def 			UpperCamelCase(					self		)  ->     Any:






							'''simple docstring'''



							lowerCamelCase_  				=      tempfile.mkdtemp()
							lowerCamelCase_  				=      self.tokenizer.fairseq_tokens_to_ids
							self.tokenizer.save_pretrained(lowercase__		)
							lowerCamelCase_  				=      NllbTokenizer.from_pretrained(lowercase__		)
							self.assertDictEqual(new_tok.fairseq_tokens_to_ids   ,			lowercase__		)
		@require_torch
		def 			UpperCamelCase(					self		)  ->     str:






							'''simple docstring'''



							lowerCamelCase_  				=      self.tokenizer(
							    self.src_text   ,			text_target=self.tgt_text   ,			padding=lowercase__   ,			truncation=lowercase__   ,			max_length=len(self.expected_src_tokens		)   ,			return_tensors='pt'   ,			)
							lowerCamelCase_  				=      shift_tokens_right(
							    batch['labels']   ,			self.tokenizer.pad_token_id   ,			self.tokenizer.lang_code_to_id['ron_Latn']		)

							self.assertIsInstance(lowercase__   ,			lowercase__		)

							self.assertEqual((2, 15)   ,			batch.input_ids.shape		)
							self.assertEqual((2, 15)   ,			batch.attention_mask.shape		)
							lowerCamelCase_  				=      batch.input_ids.tolist()[0]
							self.assertListEqual(self.expected_src_tokens   ,			lowercase__		)
							self.assertEqual(lowercase__   ,			batch.decoder_input_ids[0, 0]		)  # EOS
							# Test that special tokens are reset
							self.assertEqual(self.tokenizer.prefix_tokens   ,			[EN_CODE]		)
							self.assertEqual(self.tokenizer.suffix_tokens   ,			[self.tokenizer.eos_token_id]		)
		def 			UpperCamelCase(					self		)  ->     int:






							'''simple docstring'''



							lowerCamelCase_  				=      self.tokenizer(self.src_text   ,			padding=lowercase__   ,			truncation=lowercase__   ,			max_length=3   ,			return_tensors='pt'		)
							lowerCamelCase_  				=      self.tokenizer(
							    text_target=self.tgt_text   ,			padding=lowercase__   ,			truncation=lowercase__   ,			max_length=10   ,			return_tensors='pt'		)
							lowerCamelCase_  				=      targets["""input_ids"""]
							lowerCamelCase_  				=      shift_tokens_right(
							    lowercase__   ,			self.tokenizer.pad_token_id   ,			decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]   ,			)

							self.assertEqual(batch.input_ids.shape[1]   ,			3		)
							self.assertEqual(batch.decoder_input_ids.shape[1]   ,			10		)
		@require_torch
		def 			UpperCamelCase(					self		)  ->     List[str]:






							'''simple docstring'''



							lowerCamelCase_  				=      self.tokenizer._build_translation_inputs(
							    'A test'   ,			return_tensors='pt'   ,			src_lang='eng_Latn'   ,			tgt_lang='fra_Latn'		)

							self.assertEqual(
							    nested_simplify(lowercase__		)   ,			{
							        # A, test, EOS, en_XX
							        'input_ids': [[256047, 70, 7356, 2]],
							        'attention_mask': [[1, 1, 1, 1]],
							        # ar_AR
							        'forced_bos_token_id': 256057,
							    }   ,			)






		@require_torch
		def 			UpperCamelCase(					self		)  ->     Dict:






							'''simple docstring'''



							lowerCamelCase_  				=      True
							lowerCamelCase_  				=      self.tokenizer(
							    'UN Chief says there is no military solution in Syria'   ,			src_lang='eng_Latn'   ,			tgt_lang='fra_Latn'		)
							self.assertEqual(
							    inputs.input_ids   ,			[16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2, 256047]		)

							lowerCamelCase_  				=      False
							lowerCamelCase_  				=      self.tokenizer(
							    'UN Chief says there is no military solution in Syria'   ,			src_lang='eng_Latn'   ,			tgt_lang='fra_Latn'		)
							self.assertEqual(
							    inputs.input_ids   ,			[256047, 16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2]		)









'''simple docstring'''


import os
from typing import Dict, List, Union

import tensorflow as tf
from keras_nlp.tokenizers import BytePairTokenizer
from tensorflow_text import pad_model_inputs

from .tokenization_gpta import GPTaTokenizer







class      SCREAMING_SNAKE_CASE    (						tf.keras.layers.Layer				):






      def __init__( self   :  List[Any]						,						lowercase__   :  Dict[str, int]						,						lowercase__   :  List[str]						,						lowercase__   :  int = None						,						lowercase__   :  int = None       ):


           '''simple docstring'''

           super().__init__()
           a_     :							Optional[Any]     						=  pad_token_id
           a_     :							str     						=  max_length
           a_     :							List[str]     						=  vocab
           a_     :							Union[str, Any]     						=  merges
           a_     :							Union[str, Any]     						=  BytePairTokenizer(lowercase__						,						lowercase__						,						sequence_length=lowercase__       )






      @classmethod
      def 							lowercase_						( cls   :  List[Any]						,						lowercase__   :  GPTaTokenizer						,						*lowercase__   :  Optional[Any]						,						**lowercase__   :  int       ):


           '''simple docstring'''

           a_     :							str     						=  [""" """.join(lowercase__       ) for m in tokenizer.bpe_ranks.keys()]
           a_     :							Any     						=  tokenizer.get_vocab()
           return cls(lowercase__						,						lowercase__						,						*lowercase__						,						**lowercase__       )






      @classmethod
      def 							lowercase_						( cls   :  Dict						,						lowercase__   :  Union[str, os.PathLike]						,						*lowercase__   :  str						,						**lowercase__   :  Optional[int]       ):


           '''simple docstring'''

           a_     :							Optional[int]     						=  GPTaTokenizer.from_pretrained(lowercase__						,						*lowercase__						,						**lowercase__       )
           return cls.from_tokenizer(lowercase__						,						*lowercase__						,						**lowercase__       )






      @classmethod
      def 							lowercase_						( cls   :  Dict						,						lowercase__   :  Optional[Any]       ):


           '''simple docstring'''

           return cls(**lowercase__       )






      def 							lowercase_						( self   :  Optional[Any]       ):


           '''simple docstring'''

           return {
               "vocab": self.vocab,
               "merges": self.merges,
               "max_length": self.max_length,
               "pad_token_id": self.pad_token_id,
           }






      def 							lowercase_						( self   :  Dict						,						lowercase__   :  Union[str, Any]						,						lowercase__   :  int = None       ):


           '''simple docstring'''

           a_     :							List[str]     						=  self.tf_tokenizer(lowercase__       )
           a_     :							int     						=  tf.ones_like(lowercase__       )

           if self.pad_token_id is not None:
                # pad the tokens up to max length
                a_     :							int     						=  max_length if max_length is not None else self.max_length

                if max_length is not None:
                     a_						,     a_     :							int     						=  pad_model_inputs(
                         lowercase__						,						max_seq_length=lowercase__						,						pad_value=self.pad_token_id       )

           return {"attention_mask": attention_mask, "input_ids": input_ids}
