


import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


A				:       List[str]      =  get_tests_dir('fixtures/test_sentencepiece.model')






@require_sentencepiece
class      UpperCamelCase(	_a  ,				unittest.TestCase ):
					snake_case_		:					Any    					=						XLMProphetNetTokenizer
					snake_case_		:					Tuple    					=						False
					snake_case_		:					Any    					=						True


					def 				SCREAMING_SNAKE_CASE_	(    self						:		str    )      ->						Dict:




											'''simple docstring'''

											super().setUp()

											# We have a SentencePiece fixture for testing
											__snake_case				=		XLMProphetNetTokenizer(SCREAMING_SNAKE_CASE  ,     keep_accents=SCREAMING_SNAKE_CASE    )
											tokenizer.save_pretrained(self.tmpdirname    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Any    )      ->						Dict:




											'''simple docstring'''

											__snake_case				=		"[PAD]"
											__snake_case				=		0

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE    )  ,     SCREAMING_SNAKE_CASE    )
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE    )  ,     SCREAMING_SNAKE_CASE    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Optional[Any]    )      ->						Optional[Any]:




											'''simple docstring'''

											__snake_case				=		list(self.get_tokenizer().get_vocab().keys()    )

											self.assertEqual(vocab_keys[0]  ,     "[PAD]"    )
											self.assertEqual(vocab_keys[1]  ,     "[CLS]"    )
											self.assertEqual(vocab_keys[-1]  ,     "j"    )
											self.assertEqual(len(SCREAMING_SNAKE_CASE    )  ,     1_0_1_2    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Any    )      ->						List[Any]:




											'''simple docstring'''

											self.assertEqual(self.get_tokenizer().vocab_size  ,     1_0_1_2    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Any    )      ->						Tuple:




											'''simple docstring'''

											__snake_case				=		XLMProphetNetTokenizer(SCREAMING_SNAKE_CASE  ,     keep_accents=SCREAMING_SNAKE_CASE    )

											__snake_case				=		tokenizer.tokenize("This is a test"    )
											self.assertListEqual(SCREAMING_SNAKE_CASE  ,     ["▁This", "▁is", "▁a", "▁t", "est"]    )

											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE    )  ,     [value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]  ,     )

											__snake_case				=		tokenizer.tokenize("I was born in 92000, and this is falsé."    )
											self.assertListEqual(
											    SCREAMING_SNAKE_CASE  ,     [
											        SPIECE_UNDERLINE + "I",
											        SPIECE_UNDERLINE + "was",
											        SPIECE_UNDERLINE + "b",
											        "or",
											        "n",
											        SPIECE_UNDERLINE + "in",
											        SPIECE_UNDERLINE + "",
											        "9",
											        "2",
											        "0",
											        "0",
											        "0",
											        ",",
											        SPIECE_UNDERLINE + "and",
											        SPIECE_UNDERLINE + "this",
											        SPIECE_UNDERLINE + "is",
											        SPIECE_UNDERLINE + "f",
											        "al",
											        "s",
											        "é",
											        ".",
											    ]  ,     )
											__snake_case				=		tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE    )
											self.assertListEqual(
											    SCREAMING_SNAKE_CASE  ,     [
											        value + tokenizer.fairseq_offset
											        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, -9, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, -9, 4]
											    ]  ,     )

											__snake_case				=		tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE    )
											self.assertListEqual(
											    SCREAMING_SNAKE_CASE  ,     [
											        SPIECE_UNDERLINE + "I",
											        SPIECE_UNDERLINE + "was",
											        SPIECE_UNDERLINE + "b",
											        "or",
											        "n",
											        SPIECE_UNDERLINE + "in",
											        SPIECE_UNDERLINE + "",
											        "[UNK]",
											        "2",
											        "0",
											        "0",
											        "0",
											        ",",
											        SPIECE_UNDERLINE + "and",
											        SPIECE_UNDERLINE + "this",
											        SPIECE_UNDERLINE + "is",
											        SPIECE_UNDERLINE + "f",
											        "al",
											        "s",
											        "[UNK]",
											        ".",
											    ]  ,     )


					@cached_property
					def 				SCREAMING_SNAKE_CASE_	(    self						:		List[str]    )      ->						Union[str, Any]:




											'''simple docstring'''

											return XLMProphetNetTokenizer.from_pretrained("microsoft/xprophetnet-large-wiki100-cased"    )


					@slow
					def 				SCREAMING_SNAKE_CASE_	(    self						:		Union[str, Any]    )      ->						List[str]:




											'''simple docstring'''

											__snake_case				=		"Hello World!"
											__snake_case				=		[3_5_3_8_9, 6_6_7_2, 4_9, 2]
											self.assertListEqual(SCREAMING_SNAKE_CASE  ,     self.big_tokenizer.encode(SCREAMING_SNAKE_CASE    )    )






					@slow
					def 				SCREAMING_SNAKE_CASE_	(    self						:		Any    )      ->						List[Any]:




											'''simple docstring'''

											__snake_case				=		{"input_ids": [[1_1_0_7_3, 8_2_7_8_3, 1_8, 2_6, 8_2_7_8_3, 5_4_9, 5_1_5_4_0, 2_4_8, 1_7_2_0_9, 1_3_0_1, 2_1_7, 2_0, 2_1_5_1_8_6, 1_3_2_5, 1_4_7, 1_7_2_0_9, 1_3_0_1, 2_1_7, 2_0, 5_6_3_7_0, 5_3, 1_2_2_0_2_0, 2_0, 1_6_4_7_7, 2_7, 8_7_3_5_5, 4_5_4_8, 2_0, 4_7_2_8, 7_8_3_9_2, 1_7, 1_5_9_9_6_9, 1_8, 2_6, 2_4_4_9_1, 6_2_9, 1_5, 5_3_8, 2_2_7_0_4, 5_4_3_9, 1_5, 2_7_8_8, 2_4_4_9_1, 9_8_8_5, 1_5, 4_3_5_3_4, 6_0_5, 1_5, 8_1_4, 1_8_4_0_3, 3_3_2_0_0, 2_9, 1_5, 4_3_5_3_4, 2_4_4_5_8, 1_2_4_1_0, 1_1_1, 2_4_9_6_6, 8_3_6_6_9, 9_6_3_7, 1_4_4_0_6_8, 2_6, 8_5_0, 2_2_3_4_6, 2_7, 1_4_7, 2_4_9_6_6, 8_3_6_6_9, 8_3_4_9_0, 2_6, 3_9_1_1_3, 7_3_5, 2_7, 6_8_9, 6_5_6, 2_8_0_0, 1_3_3_9, 4_6_0_0, 5_3, 1_2_2_0_2_0, 1_1_5_7_8_5, 3_4, 8_1_6, 1_3_3_9, 4_6_8_8_7, 1_8, 1_4_7, 5_3_9_0_5, 1_9_5_1, 4_2_2_3_8, 4_1_1_7_0, 1_7_7_3_2, 8_3_4, 4_3_6, 1_5, 2_7_5_2_3, 9_8_7_3_3, 2_1_7, 1_4_7, 5_5_4_2, 4_9_8_1, 9_3_0, 1_7_3_4_7, 1_6, 2], [2_0_0_9_1, 6_2_9, 9_4, 8_2_7_8_6, 5_8, 4_9_0, 2_0, 1_5_2_8, 8_4, 5_3_9_0_5, 3_4_4, 8_0_5_9_2, 1_1_0_1_2_8, 1_8_8_2_2, 5_2_6_7, 1_3_0_6, 6_2, 1_5_2_5_3_7, 3_0_8, 7_9_9_7, 4_0_1, 1_2_4_4_2_7, 5_4_9, 3_5_4_4_2, 2_2_5, 1_0_9, 1_5_0_5_5, 2_5_7_4_8, 1_4_7, 7_1_1_9, 4_3_7_1_2, 3_4, 7_6_7, 1_3_5_3_6_6, 1_8, 1_6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_9_2, 6_3_7_8_4, 1_1_9_4_6_6, 1_7, 1_4_7_8_0_8, 8_8_2_1_4, 1_8, 6_5_6, 8_1, 3_2, 3_2_9_6, 1_0_2_8_0, 1_6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=SCREAMING_SNAKE_CASE  ,     model_name="microsoft/xprophetnet-large-wiki100-cased"  ,     revision="1acad1643ddd54a44df6a1b797ada8373685d90e"  ,     )









from datasets.utils.patching import _PatchedModuleObj, patch_submodule

from . import _test_patching





def 						_lowerCAmelCase							(     )    ->						Tuple:


						'''simple docstring'''






						import os as original_os
						from os import path as original_path
						from os import rename as original_rename
						from os.path import dirname as original_dirname
						from os.path import join as original_join

						assert _test_patching.os is original_os
						assert _test_patching.path is original_path
						assert _test_patching.join is original_join

						assert _test_patching.renamed_os is original_os
						assert _test_patching.renamed_path is original_path
						assert _test_patching.renamed_join is original_join

						__snake_case				=		"__test_patch_submodule_mock__"
						with patch_submodule(_test_patching   ,						"os.path.join"   ,						_lowerCAmelCase			):
												# Every way to access os.path.join must be patched, and the rest must stay untouched

												# check os.path.join
												assert isinstance(_test_patching.os   ,						_PatchedModuleObj			)
												assert isinstance(_test_patching.os.path   ,						_PatchedModuleObj			)
												assert _test_patching.os.path.join is mock

												# check path.join
												assert isinstance(_test_patching.path   ,						_PatchedModuleObj			)
												assert _test_patching.path.join is mock

												# check join
												assert _test_patching.join is mock

												# check that the other attributes are untouched
												assert _test_patching.os.rename is original_rename
												assert _test_patching.path.dirname is original_dirname
												assert _test_patching.os.path.dirname is original_dirname

												# Even renamed modules or objects must be patched

												# check renamed_os.path.join
												assert isinstance(_test_patching.renamed_os   ,						_PatchedModuleObj			)
												assert isinstance(_test_patching.renamed_os.path   ,						_PatchedModuleObj			)
												assert _test_patching.renamed_os.path.join is mock

												# check renamed_path.join
												assert isinstance(_test_patching.renamed_path   ,						_PatchedModuleObj			)
												assert _test_patching.renamed_path.join is mock

												# check renamed_join
												assert _test_patching.renamed_join is mock

												# check that the other attributes are untouched
												assert _test_patching.renamed_os.rename is original_rename
												assert _test_patching.renamed_path.dirname is original_dirname
												assert _test_patching.renamed_os.path.dirname is original_dirname

						# check that everthing is back to normal when the patch is over

						assert _test_patching.os is original_os
						assert _test_patching.path is original_path
						assert _test_patching.join is original_join

						assert _test_patching.renamed_os is original_os
						assert _test_patching.renamed_path is original_path
						assert _test_patching.renamed_join is original_join





def 						_lowerCAmelCase							(     )    ->						int:


						'''simple docstring'''






						assert _test_patching.open is open

						__snake_case				=		"__test_patch_submodule_builtin_mock__"
						# _test_patching has "open" in its globals
						assert _test_patching.open is open
						with patch_submodule(_test_patching   ,						"open"   ,						_lowerCAmelCase			):
												assert _test_patching.open is mock

						# check that everthing is back to normal when the patch is over

						assert _test_patching.open is open





def 						_lowerCAmelCase							(     )    ->						Optional[Any]:


						'''simple docstring'''






						__snake_case				=		"__test_patch_submodule_missing_mock__"
						with patch_submodule(_test_patching   ,						"pandas.read_csv"   ,						_lowerCAmelCase			):
												pass





def 						_lowerCAmelCase							(     )    ->						Optional[Any]:


						'''simple docstring'''






						__snake_case				=		"__test_patch_submodule_missing_builtin_mock__"
						# _test_patching doesn't have "len" in its globals
						assert getattr(_test_patching   ,						"len"   ,						_lowerCAmelCase			) is None
						with patch_submodule(_test_patching   ,						"len"   ,						_lowerCAmelCase			):
												assert _test_patching.len is mock
						assert _test_patching.len is len





def 						_lowerCAmelCase							(     )    ->						Optional[Any]:


						'''simple docstring'''






						__snake_case				=		"__test_patch_submodule_start_and_stop_mock__"
						__snake_case				=		patch_submodule(_test_patching   ,						"open"   ,						_lowerCAmelCase			)
						assert _test_patching.open is open
						patch.start()
						assert _test_patching.open is mock
						patch.stop()
						assert _test_patching.open is open





def 						_lowerCAmelCase							(     )    ->						Any:


						'''simple docstring'''






						from os import rename as original_rename
						from os.path import dirname as original_dirname
						from os.path import join as original_join

						__snake_case				=		"__test_patch_submodule_successive_join__"
						__snake_case				=		"__test_patch_submodule_successive_dirname__"
						__snake_case				=		"__test_patch_submodule_successive_rename__"
						assert _test_patching.os.path.join is original_join
						assert _test_patching.os.path.dirname is original_dirname
						assert _test_patching.os.rename is original_rename

						with patch_submodule(_test_patching   ,						"os.path.join"   ,						_lowerCAmelCase			):
												with patch_submodule(_test_patching   ,						"os.rename"   ,						_lowerCAmelCase			):
																		with patch_submodule(_test_patching   ,						"os.path.dirname"   ,						_lowerCAmelCase			):
																								assert _test_patching.os.path.join is mock_join
																								assert _test_patching.os.path.dirname is mock_dirname
																								assert _test_patching.os.rename is mock_rename

    # try another order
						with patch_submodule(_test_patching   ,						"os.rename"   ,						_lowerCAmelCase			):
												with patch_submodule(_test_patching   ,						"os.path.join"   ,						_lowerCAmelCase			):
																		with patch_submodule(_test_patching   ,						"os.path.dirname"   ,						_lowerCAmelCase			):
																								assert _test_patching.os.path.join is mock_join
																								assert _test_patching.os.path.dirname is mock_dirname
																								assert _test_patching.os.rename is mock_rename

						assert _test_patching.os.path.join is original_join
						assert _test_patching.os.path.dirname is original_dirname
						assert _test_patching.os.rename is original_rename





def 						_lowerCAmelCase							(     )    ->						str:


						'''simple docstring'''






						__snake_case				=		"__test_patch_submodule_doesnt_exist_mock__"
						with patch_submodule(_test_patching   ,						"__module_that_doesn_exist__.__attribute_that_doesn_exist__"   ,						_lowerCAmelCase			):
												pass
						with patch_submodule(_test_patching   ,						"os.__attribute_that_doesn_exist__"   ,						_lowerCAmelCase			):
												pass





