


'''simple docstring'''
import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case				 =	get_tests_dir('''fixtures/test_sentencepiece.model''')
@require_sentencepiece
class   lowercase      (				A__      ,						unittest.TestCase   ):

		"""simple docstring"""

		_a       					=							XLMProphetNetTokenizer
		_a       					=							False
		_a       					=							True




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							super().setUp()

							# We have a SentencePiece fixture for testing
							UpperCamelCase__  :Optional[Any]				   =					XLMProphetNetTokenizer(UpperCamelCase_       ,  keep_accents=UpperCamelCase_				)
							tokenizer.save_pretrained(self.tmpdirname				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :List[Any]				   =					'''[PAD]'''
							UpperCamelCase__  :List[Any]				   =					0

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase_				)       ,  UpperCamelCase_				)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase_				)       ,  UpperCamelCase_				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Union[str, Any]				   =					list(self.get_tokenizer().get_vocab().keys()				)

							self.assertEqual(vocab_keys[0]       ,  '''[PAD]'''				)
							self.assertEqual(vocab_keys[1]       ,  '''[CLS]'''				)
							self.assertEqual(vocab_keys[-1]       ,  '''j'''				)
							self.assertEqual(len(UpperCamelCase_				)       ,  1012				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							self.assertEqual(self.get_tokenizer().vocab_size       ,  1012				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :List[Any]				   =					XLMProphetNetTokenizer(UpperCamelCase_       ,  keep_accents=UpperCamelCase_				)

							UpperCamelCase__  :Union[str, Any]				   =					tokenizer.tokenize('''This is a test'''				)
							self.assertListEqual(UpperCamelCase_       ,  ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(UpperCamelCase_				)       ,  [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,  )

							UpperCamelCase__  :List[Any]				   =					tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
							self.assertListEqual(
							    UpperCamelCase_       ,  [
							        SPIECE_UNDERLINE + '''I''',
							        SPIECE_UNDERLINE + '''was''',
							        SPIECE_UNDERLINE + '''b''',
							        '''or''',
							        '''n''',
							        SPIECE_UNDERLINE + '''in''',
							        SPIECE_UNDERLINE + '''''',
							        '''9''',
							        '''2''',
							        '''0''',
							        '''0''',
							        '''0''',
							        ''',''',
							        SPIECE_UNDERLINE + '''and''',
							        SPIECE_UNDERLINE + '''this''',
							        SPIECE_UNDERLINE + '''is''',
							        SPIECE_UNDERLINE + '''f''',
							        '''al''',
							        '''s''',
							        '''é''',
							        '''.''',
							    ]       ,  )
							UpperCamelCase__  :Dict				   =					tokenizer.convert_tokens_to_ids(UpperCamelCase_				)
							self.assertListEqual(
							    UpperCamelCase_       ,  [
							        value + tokenizer.fairseq_offset
							        for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
							    ]       ,  )

							UpperCamelCase__  :Any				   =					tokenizer.convert_ids_to_tokens(UpperCamelCase_				)
							self.assertListEqual(
							    UpperCamelCase_       ,  [
							        SPIECE_UNDERLINE + '''I''',
							        SPIECE_UNDERLINE + '''was''',
							        SPIECE_UNDERLINE + '''b''',
							        '''or''',
							        '''n''',
							        SPIECE_UNDERLINE + '''in''',
							        SPIECE_UNDERLINE + '''''',
							        '''[UNK]''',
							        '''2''',
							        '''0''',
							        '''0''',
							        '''0''',
							        ''',''',
							        SPIECE_UNDERLINE + '''and''',
							        SPIECE_UNDERLINE + '''this''',
							        SPIECE_UNDERLINE + '''is''',
							        SPIECE_UNDERLINE + '''f''',
							        '''al''',
							        '''s''',
							        '''[UNK]''',
							        '''.''',
							    ]       ,  )




		@cached_property
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							return XLMProphetNetTokenizer.from_pretrained('''microsoft/xprophetnet-large-wiki100-cased'''				)




		@slow
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Optional[int]				   =					'''Hello World!'''
							UpperCamelCase__  :Dict				   =					[35389, 6672, 49, 2]
							self.assertListEqual(UpperCamelCase_       ,  self.big_tokenizer.encode(UpperCamelCase_				)				)




		@slow
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :List[str]				   =					{'''input_ids''': [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=UpperCamelCase_       ,  model_name='''microsoft/xprophetnet-large-wiki100-cased'''       ,  revision='''1acad1643ddd54a44df6a1b797ada8373685d90e'''       ,  )



'''simple docstring'''
import importlib
import os
import sys


# This is required to make the module import works (when the python process is running from the root of the repo)
sys.path.append('''.''')


def      a (     __a  )    ->      List[str]:



					'''simple docstring'''






					UpperCamelCase__  :int				   =					test_file.split(os.path.sep  )
					if components[0:2] != ["tests", "models"]:
										raise ValueError(
										    '''`test_file` should start with `tests/models/` (with `/` being the OS specific path separator). Got '''
										    f'''{test_file} instead.'''  )
					UpperCamelCase__  :List[Any]				   =					components[-1]
					if not test_fn.endswith('''py'''  ):
										raise ValueError(f'''`test_file` should be a python file. Got {test_fn} instead.'''  )
					if not test_fn.startswith('''test_modeling_'''  ):
										raise ValueError(
										    f'''`test_file` should point to a file name of the form `test_modeling_*.py`. Got {test_fn} instead.'''  )

					UpperCamelCase__  :Optional[Any]				   =					components[:-1] + [test_fn.replace('''.py'''					,   ''''''  )]
					UpperCamelCase__  :str				   =					'''.'''.join(__a  )

					return test_module_path


def      a (     __a  )    ->      Tuple:



					'''simple docstring'''






					UpperCamelCase__  :Dict				   =					get_module_path(__a  )
					UpperCamelCase__  :List[str]				   =					importlib.import_module(__a  )

					return test_module


def      a (     __a  )    ->      int:



					'''simple docstring'''






					UpperCamelCase__  :Dict				   =					[]
					UpperCamelCase__  :Optional[Any]				   =					get_test_module(__a  )
					for attr in dir(__a  ):
										if attr.endswith('''ModelTester'''  ):
															tester_classes.append(getattr(__a					,   __a  )  )

    # sort with class names
					return sorted(__a					,   key=lambda __a  : x.__name__  )


def      a (     __a  )    ->      List[Any]:



					'''simple docstring'''






					UpperCamelCase__  :List[Any]				   =					[]
					UpperCamelCase__  :Dict				   =					get_test_module(__a  )
					for attr in dir(__a  ):
										UpperCamelCase__  :Dict				   =					getattr(__a					,   __a  )
										# (TF/Flax)ModelTesterMixin is also an attribute in specific model test module. Let's exclude them by checking
										# `all_model_classes` is not empty (which also excludes other special classes).
										UpperCamelCase__  :Optional[int]				   =					getattr(__a					,   '''all_model_classes'''					,   []  )
										if len(__a  ) > 0:
															test_classes.append(__a  )

    # sort with class names
					return sorted(__a					,   key=lambda __a  : x.__name__  )


def      a (     __a  )    ->      Union[str, Any]:



					'''simple docstring'''






					UpperCamelCase__  :int				   =					get_test_classes(__a  )
					UpperCamelCase__  :Any				   =					set()
					for test_class in test_classes:
										model_classes.update(test_class.all_model_classes  )

					# sort with class names
					return sorted(__a					,   key=lambda __a  : x.__name__  )


def      a (     __a  )    ->      Dict:



					'''simple docstring'''






					UpperCamelCase__  :Optional[Any]				   =					test_class()
					if hasattr(__a					,   '''setUp'''  ):
										test.setUp()

					UpperCamelCase__  :Union[str, Any]				   =					None
					if hasattr(__a					,   '''model_tester'''  ):
										# `(TF/Flax)ModelTesterMixin` has this attribute default to `None`. Let's skip this case.
										if test.model_tester is not None:
															UpperCamelCase__  :List[str]				   =					test.model_tester.__class__

					return model_tester


def      a (     __a					,   __a  )    ->      Any:



					'''simple docstring'''






					UpperCamelCase__  :str				   =					get_test_classes(__a  )

					UpperCamelCase__  :List[str]				   =					[]
					for test_class in test_classes:
										if model_class in test_class.all_model_classes:
															target_test_classes.append(__a  )

    # sort with class names
					return sorted(__a					,   key=lambda __a  : x.__name__  )


def      a (     __a					,   __a  )    ->      Union[str, Any]:



					'''simple docstring'''






					UpperCamelCase__  :Optional[int]				   =					get_test_classes_for_model(__a					,   __a  )

					UpperCamelCase__  :List[str]				   =					[]
					for test_class in test_classes:
										UpperCamelCase__  :int				   =					get_model_tester_from_test_class(__a  )
										if tester_class is not None:
															tester_classes.append(__a  )

    # sort with class names
					return sorted(__a					,   key=lambda __a  : x.__name__  )


def      a (     __a  )    ->      int:



					'''simple docstring'''






					UpperCamelCase__  :Any				   =					get_test_classes(__a  )
					UpperCamelCase__  :Dict				   =					{test_class: get_model_tester_from_test_class(__a  ) for test_class in test_classes}
					return test_tester_mapping


def      a (     __a  )    ->      List[str]:



					'''simple docstring'''






					UpperCamelCase__  :Any				   =					get_model_classes(__a  )
					UpperCamelCase__  :Tuple				   =					{
					    model_class: get_test_classes_for_model(__a					,   __a  ) for model_class in model_classes
					}
					return model_test_mapping


def      a (     __a  )    ->      Tuple:



					'''simple docstring'''






					UpperCamelCase__  :List[str]				   =					get_model_classes(__a  )
					UpperCamelCase__  :Union[str, Any]				   =					{
					    model_class: get_tester_classes_for_model(__a					,   __a  ) for model_class in model_classes
					}
					return model_to_tester_mapping







def      a (     __a  )    ->      List[Any]:



					'''simple docstring'''






					if isinstance(__a					,   __a  ):
										return o
					elif isinstance(__a					,   __a  ):
										return o.__name__
					elif isinstance(__a					,   (list, tuple)  ):
										return [to_json(__a  ) for x in o]
					elif isinstance(__a					,   __a  ):
										return {to_json(__a  ): to_json(__a  ) for k, v in o.items()}
					else:
										return o