




'''simple docstring'''



import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
      from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
      UpperCamelCase_    	=				get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
      from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

UpperCamelCase_    	=				1_2_8_0_2_2
UpperCamelCase_    	=				1_2_8_0_2_8






@require_sentencepiece
class 							_a				(			SCREAMING_SNAKE_CASE ,      unittest.TestCase ):





       '''simple docstring'''






       A						:				List[str]        =						MaMaaaTokenizer
       A						:				Optional[int]        =						False
       A						:				Tuple        =						False
       A						:				Optional[Any]        =						True







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             super().setUp()

             SCREAMING_SNAKE_CASE   :      str    					=						['</s>', '<unk>', '▁This', '▁is', '▁a', '▁t', 'est', '\u0120', '<pad>']
             SCREAMING_SNAKE_CASE   :      List[str]    					=						dict(zip(A,							range(len(A  )  )  )  )
             SCREAMING_SNAKE_CASE   :      List[str]    					=						Path(self.tmpdirname  )
             save_json(A,							save_dir / VOCAB_FILES_NAMES['vocab_file']  )
             if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
                   copyfile(A,							save_dir / VOCAB_FILES_NAMES['spm_file']  )

             SCREAMING_SNAKE_CASE   :      Any    					=						MaMaaaTokenizer.from_pretrained(self.tmpdirname  )
             tokenizer.save_pretrained(self.tmpdirname  )







       def        UpperCamelCase_							(      self,							**A  ):







             '''simple docstring'''







             return MaMaaaTokenizer.from_pretrained(self.tmpdirname,							**A  )







       def        UpperCamelCase_							(      self,							A  ):







             '''simple docstring'''







             return (
                 "This is a test",
                 "This is a test",
             )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Tuple    					=						'</s>'
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						0

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(A  ),							A  )
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(A  ),							A  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      List[Any]    					=						self.get_tokenizer()
             SCREAMING_SNAKE_CASE   :      Tuple    					=						list(tokenizer.get_vocab().keys()  )

             self.assertEqual(vocab_keys[0],							'</s>'  )
             self.assertEqual(vocab_keys[1],							'<unk>'  )
             self.assertEqual(vocab_keys[-1],							'<s>'  )
             self.assertEqual(len(A  ),							tokenizer.vocab_size + len(tokenizer.get_added_vocab()  )  )







       @unittest.skip('Skip this test while all models are still to be uploaded.'  )
       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             pass







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Tuple    					=						self.get_tokenizer()

             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						tokenizer.tokenize('This is a test'  )
             self.assertListEqual(A,							['▁This', '▁is', '▁a', '▁t', 'est']  )

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(A  ),							[2, 3, 4, 5, 6],							)

             SCREAMING_SNAKE_CASE   :      str    					=						tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6]  )
             self.assertListEqual(A,							['▁This', '▁is', '▁a', '▁t', 'est']  )

             SCREAMING_SNAKE_CASE   :      Tuple    					=						tokenizer.convert_tokens_to_string(A  )
             self.assertEqual(A,							'This is a test'  )







       @slow
       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						{'input_ids': [[128_022, 110_108, 397, 11, 38_272, 2_247, 124_811, 285, 18_105, 1_586, 207, 7, 39_534, 4_428, 397, 1_019, 18_105, 1_586, 207, 7, 41_337, 16_786, 241, 7, 20_214, 17, 125_690, 10_398, 7, 44_378, 58_069, 68_342, 7_798, 7_343, 11, 299, 33_310, 4, 158, 37_350, 94_077, 4_569, 299, 33_310, 90, 4, 52_840, 290, 4, 31_270, 112, 299, 682, 4, 52_840, 39_953, 14_079, 193, 52_519, 90_894, 17_894, 120_697, 11, 40_445, 551, 17, 1_019, 52_519, 90_894, 17_756, 963, 11, 40_445, 480, 17, 9_792, 1_120, 5_173, 1_393, 6_240, 16_786, 241, 120_996, 28, 1_245, 1_393, 118_240, 11_123, 1_019, 93_612, 2_691, 10_618, 98_058, 120_409, 1_928, 279, 4, 40_683, 367, 178, 207, 1_019, 103, 103_121, 506, 65_296, 5, 2], [128_022, 21_217, 367, 117, 125_450, 128, 719, 7, 7_308, 40, 93_612, 12_669, 1_116, 16_704, 71, 17_785, 3_699, 15_592, 35, 144, 9_584, 241, 11_943, 713, 950, 799, 2_247, 88_427, 150, 149, 118_813, 120_706, 1_019, 106_906, 81_518, 28, 1_224, 22_799, 397, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [128_022, 1_658, 123_311, 5_155, 5_578, 4_722, 279, 14_947, 2_366, 1_120, 1_197, 14, 1_348, 9_232, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=A,							model_name='facebook/m2m100_418M',							revision='c168bae485c864188cf9aa0e4108b0b6934dc91e',							)






@require_torch
@require_sentencepiece
@require_tokenizers
class 							_a				(			unittest.TestCase ):





       '''simple docstring'''






       A						:				Dict        =						'''facebook/m2m100_418M'''
       A						:				Dict        =						[
           '''In my opinion, there are two levels of response from the French government.''',
           '''NSA Affair Emphasizes Complete Lack of Debate on Intelligence''',
       ]
       A						:				Optional[int]        =						[
           '''Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.''',
           '''L\'affaire NSA souligne l\'absence totale de débat sur le renseignement''',
       ]

       # fmt: off
       A						:				Dict        =						[EN_CODE, 593, 1_949, 115_781, 4, 71_586, 4_234, 60_633, 126_233, 432, 123_808, 15_592, 1_197, 117_132, 120_618, 5, 2]







       @classmethod
       def        UpperCamelCase_							(      cls  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      MaMaaaTokenizer    					=						MaMaaaTokenizer.from_pretrained(
                 cls.checkpoint_name,							src_lang='en',							tgt_lang='fr'  )
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						1
             return cls







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             self.assertEqual(self.tokenizer.get_lang_id('ar'  ),							128_006  )
             self.assertEqual(self.tokenizer.get_lang_id('en'  ),							128_022  )
             self.assertEqual(self.tokenizer.get_lang_id('ro'  ),							128_076  )
             self.assertEqual(self.tokenizer.get_lang_id('mr'  ),							128_063  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						self.tokenizer.get_vocab()
             self.assertEqual(len(A  ),							self.tokenizer.vocab_size  )
             self.assertEqual(vocab['<unk>'],							3  )
             self.assertIn(self.tokenizer.get_lang_token('en'  ),							A  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Any    					=						'en'
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						self.tokenizer.batch_encode_plus(self.src_text  ).input_ids[0]
             self.assertListEqual(self.expected_src_tokens,							A  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             self.assertIn(A,							self.tokenizer.all_special_ids  )
             # fmt: off
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						[FR_CODE, 5_364, 82, 8_642, 4, 294, 47, 8, 14_028, 136, 3_286, 9_706, 6, 90_797, 6, 144_012, 162, 88_128, 30_061, 5, 2]
             # fmt: on
             SCREAMING_SNAKE_CASE   :      int    					=						self.tokenizer.decode(A,							skip_special_tokens=A  )
             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						self.tokenizer.decode(generated_ids[1:],							skip_special_tokens=A  )
             self.assertEqual(A,							A  )
             self.assertNotIn(self.tokenizer.eos_token,							A  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						tempfile.mkdtemp()
             SCREAMING_SNAKE_CASE   :      List[str]    					=						self.tokenizer.lang_token_to_id
             self.tokenizer.save_pretrained(A  )
             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						MaMaaaTokenizer.from_pretrained(A  )
             self.assertDictEqual(new_tok.lang_token_to_id,							A  )







       @require_torch
       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      List[Any]    					=						'en'
             SCREAMING_SNAKE_CASE   :      Tuple    					=						'fr'

             SCREAMING_SNAKE_CASE   :      Any    					=						self.tokenizer(self.src_text,							text_target=self.tgt_text,							padding=A,							return_tensors='pt'  )

             SCREAMING_SNAKE_CASE   :      List[str]    					=						shift_tokens_right(
                 batch['labels'],							self.tokenizer.pad_token_id,							self.tokenizer.eos_token_id  )

             for k in batch:
                   SCREAMING_SNAKE_CASE   :      Any    					=						batch[k].tolist()
             # batch = {k: v.tolist() for k,v in batch.items()}
             # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
             # batch.decoder_inputs_ids[0][0] ==
             assert batch.input_ids[1][0] == EN_CODE
             assert batch.input_ids[1][-1] == 2
             assert batch.labels[1][0] == FR_CODE
             assert batch.labels[1][-1] == 2
             assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]







       @require_torch
       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						'mr'
             self.assertListEqual(self.tokenizer.prefix_tokens,							[self.tokenizer.get_lang_id('mr'  )]  )
             self.assertListEqual(self.tokenizer.suffix_tokens,							[self.tokenizer.eos_token_id]  )

             SCREAMING_SNAKE_CASE   :      str    					=						'zh'
             self.assertListEqual(self.tokenizer.prefix_tokens,							[self.tokenizer.get_lang_id('zh'  )]  )
             self.assertListEqual(self.tokenizer.suffix_tokens,							[self.tokenizer.eos_token_id]  )







       @require_torch
       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						'mr'
             self.tokenizer._switch_to_target_mode()
             self.assertListEqual(self.tokenizer.prefix_tokens,							[self.tokenizer.get_lang_id('mr'  )]  )
             self.assertListEqual(self.tokenizer.suffix_tokens,							[self.tokenizer.eos_token_id]  )
             self.tokenizer._switch_to_input_mode()
             self.assertListEqual(self.tokenizer.prefix_tokens,							[self.tokenizer.get_lang_id(self.tokenizer.src_lang  )]  )

             SCREAMING_SNAKE_CASE   :      int    					=						'zh'
             self.tokenizer._switch_to_target_mode()
             self.assertListEqual(self.tokenizer.prefix_tokens,							[self.tokenizer.get_lang_id('zh'  )]  )
             self.assertListEqual(self.tokenizer.suffix_tokens,							[self.tokenizer.eos_token_id]  )
             self.tokenizer._switch_to_input_mode()
             self.assertListEqual(self.tokenizer.prefix_tokens,							[self.tokenizer.get_lang_id(self.tokenizer.src_lang  )]  )







       @require_torch
       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Any    					=						self.tokenizer._build_translation_inputs('A test',							return_tensors='pt',							src_lang='en',							tgt_lang='ar'  )

             self.assertEqual(
                 nested_simplify(A  ),							{
                     # en_XX, A, test, EOS
                     'input_ids': [[128_022, 58, 4_183, 2]],
                     'attention_mask': [[1, 1, 1, 1]],
                     # ar_AR
                     'forced_bos_token_id': 128_006,
                 },							)





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


UpperCAmelCase_							:					Optional[Any]		 =   {
    '''configuration_wav2vec2''': ['''WAV_2_VEC_2_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''Wav2Vec2Config'''],
    '''feature_extraction_wav2vec2''': ['''Wav2Vec2FeatureExtractor'''],
    '''processing_wav2vec2''': ['''Wav2Vec2Processor'''],
    '''tokenization_wav2vec2''': ['''Wav2Vec2CTCTokenizer''', '''Wav2Vec2Tokenizer'''],
}


try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCAmelCase_							:					Optional[Any]		 =   [
			    '''WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''Wav2Vec2ForAudioFrameClassification''',
			    '''Wav2Vec2ForCTC''',
			    '''Wav2Vec2ForMaskedLM''',
			    '''Wav2Vec2ForPreTraining''',
			    '''Wav2Vec2ForSequenceClassification''',
			    '''Wav2Vec2ForXVector''',
			    '''Wav2Vec2Model''',
			    '''Wav2Vec2PreTrainedModel''',
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCAmelCase_							:					List[Any]		 =   [
			    '''TF_WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST''',
			    '''TFWav2Vec2ForCTC''',
			    '''TFWav2Vec2Model''',
			    '''TFWav2Vec2PreTrainedModel''',
			    '''TFWav2Vec2ForSequenceClassification''',
			]

try:
			if not is_flax_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			UpperCAmelCase_							:					Dict		 =   [
			    '''FlaxWav2Vec2ForCTC''',
			    '''FlaxWav2Vec2ForPreTraining''',
			    '''FlaxWav2Vec2Model''',
			    '''FlaxWav2Vec2PreTrainedModel''',
			]


if TYPE_CHECKING:
			from .configuration_wavaveca import WAV_2_VEC_2_PRETRAINED_CONFIG_ARCHIVE_MAP, WavaVecaConfig
			from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
			from .processing_wavaveca import WavaVecaProcessor
			from .tokenization_wavaveca import WavaVecaCTCTokenizer, WavaVecaTokenizer

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_wavaveca import (
						    WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST,
						    WavaVecaForAudioFrameClassification,
						    WavaVecaForCTC,
						    WavaVecaForMaskedLM,
						    WavaVecaForPreTraining,
						    WavaVecaForSequenceClassification,
						    WavaVecaForXVector,
						    WavaVecaModel,
						    WavaVecaPreTrainedModel,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_wavaveca import (
						    TF_WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFWavaVecaForCTC,
						    TFWavaVecaForSequenceClassification,
						    TFWavaVecaModel,
						    TFWavaVecaPreTrainedModel,
						)

			try:
						if not is_flax_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_wavaveca import (
						    FlaxWavaVecaForCTC,
						    FlaxWavaVecaForPreTraining,
						    FlaxWavaVecaModel,
						    FlaxWavaVecaPreTrainedModel,
						)


else:
			import sys

			UpperCAmelCase_							:					Optional[Any]		 =   _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





