



"""simple docstring"""


import unittest

from transformers import TrOCRConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers.models.trocr.modeling_trocr import TrOCRDecoder, TrOCRForCausalLM
@require_torch
class       _a			:
			"""simple docstring"""



			def __init__(		self  :     Tuple					, __UpperCamelCase  :     List[str]					, __UpperCamelCase  :     int=9_9					, __UpperCamelCase  :     Tuple=1_3					, __UpperCamelCase  :     str=1_6					, __UpperCamelCase  :     int=7					, __UpperCamelCase  :     Tuple=True					, __UpperCamelCase  :     Dict=True					, __UpperCamelCase  :     Optional[int]=True					, __UpperCamelCase  :     List[Any]=False					, __UpperCamelCase  :     Tuple=True					, __UpperCamelCase  :     Union[str, Any]=2					, __UpperCamelCase  :     List[Any]=3_2					, __UpperCamelCase  :     Any=4					, __UpperCamelCase  :     int=4					, __UpperCamelCase  :     str=3_0					, __UpperCamelCase  :     int=0					, __UpperCamelCase  :     Optional[Any]=1					, __UpperCamelCase  :     Optional[int]=2					, __UpperCamelCase  :     Any=None					, )->Any:
							_UpperCAmelCase      		=     parent
							_UpperCAmelCase      		=     batch_size
							_UpperCAmelCase      		=     decoder_seq_length
							# For common tests
							_UpperCAmelCase      		=     self.decoder_seq_length
							_UpperCAmelCase      		=     is_training
							_UpperCAmelCase      		=     use_attention_mask
							_UpperCAmelCase      		=     use_labels

							_UpperCAmelCase      		=     vocab_size
							_UpperCAmelCase      		=     d_model
							_UpperCAmelCase      		=     d_model
							_UpperCAmelCase      		=     decoder_layers
							_UpperCAmelCase      		=     decoder_layers
							_UpperCAmelCase      		=     decoder_ffn_dim
							_UpperCAmelCase      		=     decoder_attention_heads
							_UpperCAmelCase      		=     decoder_attention_heads
							_UpperCAmelCase      		=     eos_token_id
							_UpperCAmelCase      		=     bos_token_id
							_UpperCAmelCase      		=     pad_token_id
							_UpperCAmelCase      		=     decoder_start_token_id
							_UpperCAmelCase      		=     use_cache
							_UpperCAmelCase      		=     max_position_embeddings

							_UpperCAmelCase      		=     None
							_UpperCAmelCase      		=     decoder_seq_length
							_UpperCAmelCase      		=     2
							_UpperCAmelCase      		=     1



			def 			lowercase__    (		self  :     Optional[Any] )->Dict:
							_UpperCAmelCase      		=     ids_tensor([self.batch_size, self.decoder_seq_length]					, self.vocab_size )

							_UpperCAmelCase      		=     None
							if self.use_attention_mask:
											_UpperCAmelCase      		=     ids_tensor([self.batch_size, self.decoder_seq_length]					, vocab_size=2 )

							_UpperCAmelCase      		=     None
							if self.use_labels:
											_UpperCAmelCase      		=     ids_tensor([self.batch_size, self.decoder_seq_length]					, self.vocab_size )

							_UpperCAmelCase      		=     TrOCRConfig(
							    vocab_size=self.vocab_size					, d_model=self.d_model					, decoder_layers=self.decoder_layers					, decoder_ffn_dim=self.decoder_ffn_dim					, decoder_attention_heads=self.decoder_attention_heads					, eos_token_id=self.eos_token_id					, bos_token_id=self.bos_token_id					, use_cache=self.use_cache					, pad_token_id=self.pad_token_id					, decoder_start_token_id=self.decoder_start_token_id					, max_position_embeddings=self.max_position_embeddings					, )

							return (config, input_ids, attention_mask, lm_labels)



			def 			lowercase__    (		self  :     int					, __UpperCamelCase  :     List[str]					, __UpperCamelCase  :     List[str]					, __UpperCamelCase  :     int					, __UpperCamelCase  :     str					, )->List[Any]:
							_UpperCAmelCase      		=     True
							_UpperCAmelCase      		=     TrOCRDecoder(config=_a ).to(_a ).eval()
							_UpperCAmelCase      		=     input_ids[:2]

							input_ids[input_ids == 0] += 1
							# first forward pass
							_UpperCAmelCase      		=     model(_a					, use_cache=_a )
							_UpperCAmelCase      		=     model(_a )
							_UpperCAmelCase      		=     model(_a					, use_cache=_a )

							self.parent.assertTrue(len(_a ) == len(_a ) )
							self.parent.assertTrue(len(_a ) == len(_a ) + 1 )

							_UpperCAmelCase      		=     outputs['''past_key_values''']

							# create hypothetical next token and extent to next_input_ids
							_UpperCAmelCase      		=     ids_tensor((2, 1)					, config.vocab_size - 1 ) + 1

							# append to next input_ids and
							_UpperCAmelCase      		=     torch.cat([input_ids, next_tokens]					, dim=-1 )

							_UpperCAmelCase      		=     model(_a )['''last_hidden_state''']
							_UpperCAmelCase      		=     model(_a					, past_key_values=_a )['''last_hidden_state''']

							# select random slice
							_UpperCAmelCase      		=     ids_tensor((1,)					, output_from_past.shape[-1] ).item()
							_UpperCAmelCase      		=     output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
							_UpperCAmelCase      		=     output_from_past[:, 0, random_slice_idx].detach()

							# test that outputs are equal for slice
							assert torch.allclose(_a					, _a					, atol=1e-3 )



			def 			lowercase__    (		self  :     Optional[Any] )->Optional[Any]:
							_UpperCAmelCase      		=     self.prepare_config_and_inputs()
							_UpperCAmelCase      		=     config_and_inputs

							_UpperCAmelCase      		=     {'''input_ids''': input_ids, '''attention_mask''': attention_mask}
							return config, inputs_dict



@require_torch
class       _a			(       __lowercase      ,				__lowercase      ,				__lowercase      ,				unittest.TestCase):
			"""simple docstring"""
			UpperCamelCase__         =						(TrOCRDecoder, TrOCRForCausalLM) if is_torch_available() else ()
			UpperCamelCase__         =						(TrOCRForCausalLM,) if is_torch_available() else ()
			UpperCamelCase__         =						{"text-generation": TrOCRForCausalLM} if is_torch_available() else {}
			UpperCamelCase__         =						True
			UpperCamelCase__         =						False



			def 			lowercase__    (		self  :     str )->Tuple:
							_UpperCAmelCase      		=     TrOCRStandaloneDecoderModelTester(self					, is_training=_a )
							_UpperCAmelCase      		=     ConfigTester(self					, config_class=_a )



			def 			lowercase__    (		self  :     Any )->Optional[Any]:
							pass



			def 			lowercase__    (		self  :     int )->Optional[Any]:
							pass



			def 			lowercase__    (		self  :     Any )->List[Any]:
							pass



			def 			lowercase__    (		self  :     Optional[Any] )->Union[str, Any]:
							self.config_tester.run_common_tests()



			def 			lowercase__    (		self  :     Optional[int] )->Union[str, Any]:
							_UpperCAmelCase      		=     self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_decoder_model_past(*_a )



			def 			lowercase__    (		self  :     Union[str, Any] )->List[str]:
							return



			@unittest.skip('''The model doesn\'t support left padding''' )  # and it's not used enough to be worth fixing :)
			def 			lowercase__    (		self  :     List[Any] )->str:
							pass






import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


a__												=						get_tests_dir('''fixtures/test_sentencepiece.model''')

@require_sentencepiece
class 							UpperCAmelCase_						(     __lowercase							,			unittest.TestCase  ):
							"""simple docstring"""







							UpperCAmelCase__			:		List[str]             =       XLMProphetNetTokenizer
							UpperCAmelCase__			:		Optional[int]             =       False
							UpperCAmelCase__			:		List[Any]             =       True


							def 							__lowercase							(     self			)       ->       int:
												super().setUp()

												# We have a SentencePiece fixture for testing
												_a      :       List[Any]   =						XLMProphetNetTokenizer(_a			,				keep_accents=_a			)
												tokenizer.save_pretrained(self.tmpdirname			)


							def 							__lowercase							(     self			)       ->       Any:
												_a      :       Tuple   =						'''[PAD]'''
												_a      :       int   =						0

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a			)			,				_a			)
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a			)			,				_a			)


							def 							__lowercase							(     self			)       ->       str:
												_a      :       Any   =						list(self.get_tokenizer().get_vocab().keys()			)

												self.assertEqual(vocab_keys[0]			,				'''[PAD]'''			)
												self.assertEqual(vocab_keys[1]			,				'''[CLS]'''			)
												self.assertEqual(vocab_keys[-1]			,				'''j'''			)
												self.assertEqual(len(_a			)			,				1_0_1_2			)


							def 							__lowercase							(     self			)       ->       Union[str, Any]:
												self.assertEqual(self.get_tokenizer().vocab_size			,				1_0_1_2			)


							def 							__lowercase							(     self			)       ->       str:
												_a      :       Tuple   =						XLMProphetNetTokenizer(_a			,				keep_accents=_a			)

												_a      :       Union[str, Any]   =						tokenizer.tokenize('''This is a test'''			)
												self.assertListEqual(_a			,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']			)

												self.assertListEqual(
												    tokenizer.convert_tokens_to_ids(_a			)			,				[value + tokenizer.fairseq_offset for value in [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]]			,				)

												_a      :       Optional[int]   =						tokenizer.tokenize('''I was born in 92000, and this is falsé.'''			)
												self.assertListEqual(
												    _a			,				[
												        SPIECE_UNDERLINE + '''I''',
												        SPIECE_UNDERLINE + '''was''',
												        SPIECE_UNDERLINE + '''b''',
												        '''or''',
												        '''n''',
												        SPIECE_UNDERLINE + '''in''',
												        SPIECE_UNDERLINE + '''''',
												        '''9''',
												        '''2''',
												        '''0''',
												        '''0''',
												        '''0''',
												        ''',''',
												        SPIECE_UNDERLINE + '''and''',
												        SPIECE_UNDERLINE + '''this''',
												        SPIECE_UNDERLINE + '''is''',
												        SPIECE_UNDERLINE + '''f''',
												        '''al''',
												        '''s''',
												        '''é''',
												        '''.''',
												    ]			,				)
												_a      :       List[Any]   =						tokenizer.convert_tokens_to_ids(_a			)
												self.assertListEqual(
												    _a			,				[
												        value + tokenizer.fairseq_offset
												        for value in [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, -9, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, -9, 4]
												    ]			,				)

												_a      :       List[str]   =						tokenizer.convert_ids_to_tokens(_a			)
												self.assertListEqual(
												    _a			,				[
												        SPIECE_UNDERLINE + '''I''',
												        SPIECE_UNDERLINE + '''was''',
												        SPIECE_UNDERLINE + '''b''',
												        '''or''',
												        '''n''',
												        SPIECE_UNDERLINE + '''in''',
												        SPIECE_UNDERLINE + '''''',
												        '''[UNK]''',
												        '''2''',
												        '''0''',
												        '''0''',
												        '''0''',
												        ''',''',
												        SPIECE_UNDERLINE + '''and''',
												        SPIECE_UNDERLINE + '''this''',
												        SPIECE_UNDERLINE + '''is''',
												        SPIECE_UNDERLINE + '''f''',
												        '''al''',
												        '''s''',
												        '''[UNK]''',
												        '''.''',
												    ]			,				)


							@cached_property
							def 							__lowercase							(     self			)       ->       List[str]:
												return XLMProphetNetTokenizer.from_pretrained('''microsoft/xprophetnet-large-wiki100-cased'''			)


							@slow
							def 							__lowercase							(     self			)       ->       Tuple:
												_a      :       str   =						'''Hello World!'''
												_a      :       Tuple   =						[3_5_3_8_9, 6_6_7_2, 4_9, 2]
												self.assertListEqual(_a			,				self.big_tokenizer.encode(_a			)			)




							@slow
							def 							__lowercase							(     self			)       ->       str:
												# fmt: off
												_a      :       str   =						{'''input_ids''': [[1_1_0_7_3, 8_2_7_8_3, 1_8, 2_6, 8_2_7_8_3, 5_4_9, 5_1_5_4_0, 2_4_8, 1_7_2_0_9, 1_3_0_1, 2_1_7, 2_0, 2_1_5_1_8_6, 1_3_2_5, 1_4_7, 1_7_2_0_9, 1_3_0_1, 2_1_7, 2_0, 5_6_3_7_0, 5_3, 1_2_2_0_2_0, 2_0, 1_6_4_7_7, 2_7, 8_7_3_5_5, 4_5_4_8, 2_0, 4_7_2_8, 7_8_3_9_2, 1_7, 1_5_9_9_6_9, 1_8, 2_6, 2_4_4_9_1, 6_2_9, 1_5, 5_3_8, 2_2_7_0_4, 5_4_3_9, 1_5, 2_7_8_8, 2_4_4_9_1, 9_8_8_5, 1_5, 4_3_5_3_4, 6_0_5, 1_5, 8_1_4, 1_8_4_0_3, 3_3_2_0_0, 2_9, 1_5, 4_3_5_3_4, 2_4_4_5_8, 1_2_4_1_0, 1_1_1, 2_4_9_6_6, 8_3_6_6_9, 9_6_3_7, 1_4_4_0_6_8, 2_6, 8_5_0, 2_2_3_4_6, 2_7, 1_4_7, 2_4_9_6_6, 8_3_6_6_9, 8_3_4_9_0, 2_6, 3_9_1_1_3, 7_3_5, 2_7, 6_8_9, 6_5_6, 2_8_0_0, 1_3_3_9, 4_6_0_0, 5_3, 1_2_2_0_2_0, 1_1_5_7_8_5, 3_4, 8_1_6, 1_3_3_9, 4_6_8_8_7, 1_8, 1_4_7, 5_3_9_0_5, 1_9_5_1, 4_2_2_3_8, 4_1_1_7_0, 1_7_7_3_2, 8_3_4, 4_3_6, 1_5, 2_7_5_2_3, 9_8_7_3_3, 2_1_7, 1_4_7, 5_5_4_2, 4_9_8_1, 9_3_0, 1_7_3_4_7, 1_6, 2], [2_0_0_9_1, 6_2_9, 9_4, 8_2_7_8_6, 5_8, 4_9_0, 2_0, 1_5_2_8, 8_4, 5_3_9_0_5, 3_4_4, 8_0_5_9_2, 1_1_0_1_2_8, 1_8_8_2_2, 5_2_6_7, 1_3_0_6, 6_2, 1_5_2_5_3_7, 3_0_8, 7_9_9_7, 4_0_1, 1_2_4_4_2_7, 5_4_9, 3_5_4_4_2, 2_2_5, 1_0_9, 1_5_0_5_5, 2_5_7_4_8, 1_4_7, 7_1_1_9, 4_3_7_1_2, 3_4, 7_6_7, 1_3_5_3_6_6, 1_8, 1_6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_9_2, 6_3_7_8_4, 1_1_9_4_6_6, 1_7, 1_4_7_8_0_8, 8_8_2_1_4, 1_8, 6_5_6, 8_1, 3_2, 3_2_9_6, 1_0_2_8_0, 1_6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
												# fmt: on

												self.tokenizer_integration_test_util(
												    expected_encoding=_a			,				model_name='''microsoft/xprophetnet-large-wiki100-cased'''			,				revision='''1acad1643ddd54a44df6a1b797ada8373685d90e'''			,				)



