





import unittest

from transformers import TrOCRConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers.models.trocr.modeling_trocr import TrOCRDecoder, TrOCRForCausalLM
@require_torch
class      UpperCamelCase_						:




					def __init__(						self			,  lowerCAmelCase_			,  lowerCAmelCase_=99			,  lowerCAmelCase_=13			,  lowerCAmelCase_=16			,  lowerCAmelCase_=7			,  lowerCAmelCase_=True			,  lowerCAmelCase_=True			,  lowerCAmelCase_=True			,  lowerCAmelCase_=False			,  lowerCAmelCase_=True			,  lowerCAmelCase_=2			,  lowerCAmelCase_=32			,  lowerCAmelCase_=4			,  lowerCAmelCase_=4			,  lowerCAmelCase_=30			,  lowerCAmelCase_=0			,  lowerCAmelCase_=1			,  lowerCAmelCase_=2			,  lowerCAmelCase_=None			,  )						->							Dict:
										_snake_case													=			parent
										_snake_case													=			batch_size
										_snake_case													=			decoder_seq_length
										# For common tests
										_snake_case													=			self.decoder_seq_length
										_snake_case													=			is_training
										_snake_case													=			use_attention_mask
										_snake_case													=			use_labels

										_snake_case													=			vocab_size
										_snake_case													=			d_model
										_snake_case													=			d_model
										_snake_case													=			decoder_layers
										_snake_case													=			decoder_layers
										_snake_case													=			decoder_ffn_dim
										_snake_case													=			decoder_attention_heads
										_snake_case													=			decoder_attention_heads
										_snake_case													=			eos_token_id
										_snake_case													=			bos_token_id
										_snake_case													=			pad_token_id
										_snake_case													=			decoder_start_token_id
										_snake_case													=			use_cache
										_snake_case													=			max_position_embeddings

										_snake_case													=			None
										_snake_case													=			decoder_seq_length
										_snake_case													=			2
										_snake_case													=			1




					def 		lowerCAmelCase     (						self					)						->							Optional[int]:
										_snake_case													=			ids_tensor([self.batch_size, self.decoder_seq_length]			,  self.vocab_size					)

										_snake_case													=			None
										if self.use_attention_mask:
															_snake_case													=			ids_tensor([self.batch_size, self.decoder_seq_length]			,  vocab_size=2					)

										_snake_case													=			None
										if self.use_labels:
															_snake_case													=			ids_tensor([self.batch_size, self.decoder_seq_length]			,  self.vocab_size					)

										_snake_case													=			TrOCRConfig(
										    vocab_size=self.vocab_size			,  d_model=self.d_model			,  decoder_layers=self.decoder_layers			,  decoder_ffn_dim=self.decoder_ffn_dim			,  decoder_attention_heads=self.decoder_attention_heads			,  eos_token_id=self.eos_token_id			,  bos_token_id=self.bos_token_id			,  use_cache=self.use_cache			,  pad_token_id=self.pad_token_id			,  decoder_start_token_id=self.decoder_start_token_id			,  max_position_embeddings=self.max_position_embeddings			,  )

										return (config, input_ids, attention_mask, lm_labels)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_			,  )						->							List[str]:
										_snake_case													=			True
										_snake_case													=			TrOCRDecoder(config=lowerCAmelCase_					).to(lowerCAmelCase_					).eval()
										_snake_case													=			input_ids[:2]

										input_ids[input_ids == 0] += 1
										# first forward pass
										_snake_case													=			model(lowerCAmelCase_			,  use_cache=lowerCAmelCase_					)
										_snake_case													=			model(lowerCAmelCase_					)
										_snake_case													=			model(lowerCAmelCase_			,  use_cache=lowerCAmelCase_					)

										self.parent.assertTrue(len(lowerCAmelCase_					) == len(lowerCAmelCase_					)					)
										self.parent.assertTrue(len(lowerCAmelCase_					) == len(lowerCAmelCase_					) + 1					)

										_snake_case													=			outputs['past_key_values']

										# create hypothetical next token and extent to next_input_ids
										_snake_case													=			ids_tensor((2, 1)			,  config.vocab_size - 1					) + 1

										# append to next input_ids and
										_snake_case													=			torch.cat([input_ids, next_tokens]			,  dim=-1					)

										_snake_case													=			model(lowerCAmelCase_					)['last_hidden_state']
										_snake_case													=			model(lowerCAmelCase_			,  past_key_values=lowerCAmelCase_					)['last_hidden_state']

										# select random slice
										_snake_case													=			ids_tensor((1,)			,  output_from_past.shape[-1]					).item()
										_snake_case													=			output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
										_snake_case													=			output_from_past[:, 0, random_slice_idx].detach()

										# test that outputs are equal for slice
										assert torch.allclose(lowerCAmelCase_			,  lowerCAmelCase_			,  atol=1E-3					)






					def 		lowerCAmelCase     (						self					)						->							Any:
										_snake_case													=			self.prepare_config_and_inputs()
										_snake_case							,       _snake_case							,       _snake_case							,       _snake_case													=			config_and_inputs

										_snake_case													=			{'input_ids': input_ids, 'attention_mask': attention_mask}
										return config, inputs_dict





@require_torch
class      UpperCamelCase_						(					_lowerCamelCase							,					_lowerCamelCase							,					_lowerCamelCase							,					unittest.TestCase   ):
					lowerCAmelCase_							=		(TrOCRDecoder, TrOCRForCausalLM) if is_torch_available() else ()
					lowerCAmelCase_							=		(TrOCRForCausalLM,) if is_torch_available() else ()
					lowerCAmelCase_							=		{'''text-generation''': TrOCRForCausalLM} if is_torch_available() else {}
					lowerCAmelCase_							=		True
					lowerCAmelCase_							=		False




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			TrOCRStandaloneDecoderModelTester(self			,  is_training=lowerCAmelCase_					)
										_snake_case													=			ConfigTester(self			,  config_class=lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self					)						->							List[str]:
										pass




					def 		lowerCAmelCase     (						self					)						->							str:
										pass




					def 		lowerCAmelCase     (						self					)						->							int:
										pass




					def 		lowerCAmelCase     (						self					)						->							List[str]:
										self.config_tester.run_common_tests()




					def 		lowerCAmelCase     (						self					)						->							List[str]:
										_snake_case													=			self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_decoder_model_past(*lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self					)						->							List[Any]:
										return






					@unittest.skip('The model doesn\'t support left padding'					)  # and it's not used enough to be worth fixing :)
					def 		lowerCAmelCase     (						self					)						->							Optional[int]:
										pass










import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class      UpperCamelCase_						(					_lowerCamelCase							,					unittest.TestCase   ):
					lowerCAmelCase_							=		LEDTokenizer
					lowerCAmelCase_							=		LEDTokenizerFast
					lowerCAmelCase_							=		True




					def 		lowerCAmelCase     (						self					)						->							List[str]:
										super().setUp()
										_snake_case													=			[
										    'l',
										    'o',
										    'w',
										    'e',
										    'r',
										    's',
										    't',
										    'i',
										    'd',
										    'n',
										    '\u0120',
										    '\u0120l',
										    '\u0120n',
										    '\u0120lo',
										    '\u0120low',
										    'er',
										    '\u0120lowest',
										    '\u0120newer',
										    '\u0120wider',
										    '<unk>',
										]
										_snake_case													=			dict(zip(lowerCAmelCase_			,  range(len(lowerCAmelCase_					)					)					)					)
										_snake_case													=			['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
										_snake_case													=			{'unk_token': '<unk>'}

										_snake_case													=			os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES['vocab_file']					)
										_snake_case													=			os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES['merges_file']					)
										with open(self.vocab_file			,  'w'			,  encoding='utf-8'					) as fp:
															fp.write(json.dumps(lowerCAmelCase_					) + '\n'					)
										with open(self.merges_file			,  'w'			,  encoding='utf-8'					) as fp:
															fp.write('\n'.join(lowerCAmelCase_					)					)




					def 		lowerCAmelCase     (						self			,  **lowerCAmelCase_					)						->							List[str]:
										kwargs.update(self.special_tokens_map					)
										return self.tokenizer_class.from_pretrained(self.tmpdirname			,  **lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self			,  **lowerCAmelCase_					)						->							str:
										kwargs.update(self.special_tokens_map					)
										return self.rust_tokenizer_class.from_pretrained(self.tmpdirname			,  **lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							Union[str, Any]:
										return "lower newer", "lower newer"




					@cached_property
					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										return LEDTokenizer.from_pretrained('allenai/led-base-16384'					)




					@cached_property
					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										return LEDTokenizerFast.from_pretrained('allenai/led-base-16384'					)




					@require_torch
					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			['A long paragraph for summarization.', 'Another paragraph for summarization.']
										_snake_case													=			[0, 250, 251, 1_7818, 13, 3_9186, 1938, 4, 2]

										for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
															_snake_case													=			tokenizer(lowerCAmelCase_			,  max_length=len(lowerCAmelCase_					)			,  padding=lowerCAmelCase_			,  return_tensors='pt'					)
															self.assertIsInstance(lowerCAmelCase_			,  lowerCAmelCase_					)

															self.assertEqual((2, 9)			,  batch.input_ids.shape					)
															self.assertEqual((2, 9)			,  batch.attention_mask.shape					)
															_snake_case													=			batch.input_ids.tolist()[0]
															self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)




					@require_torch
					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										_snake_case													=			['A long paragraph for summarization.', 'Another paragraph for summarization.']
										for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
															_snake_case													=			tokenizer(lowerCAmelCase_			,  padding=lowerCAmelCase_			,  return_tensors='pt'					)
															self.assertIn('input_ids'			,  lowerCAmelCase_					)
															self.assertIn('attention_mask'			,  lowerCAmelCase_					)
															self.assertNotIn('labels'			,  lowerCAmelCase_					)
															self.assertNotIn('decoder_attention_mask'			,  lowerCAmelCase_					)




					@require_torch
					def 		lowerCAmelCase     (						self					)						->							Optional[int]:
										_snake_case													=			[
										    'Summary of the text.',
										    'Another summary.',
										]
										for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
															_snake_case													=			tokenizer(text_target=lowerCAmelCase_			,  max_length=32			,  padding='max_length'			,  return_tensors='pt'					)
															self.assertEqual(32			,  targets['input_ids'].shape[1]					)




					@require_torch
					def 		lowerCAmelCase     (						self					)						->							List[str]:
										for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
															_snake_case													=			tokenizer(
															    ['I am a small frog' * 1024, 'I am a small frog']			,  padding=lowerCAmelCase_			,  truncation=lowerCAmelCase_			,  return_tensors='pt'					)
															self.assertIsInstance(lowerCAmelCase_			,  lowerCAmelCase_					)
															self.assertEqual(batch.input_ids.shape			,  (2, 5122)					)




					@require_torch
					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			['A long paragraph for summarization.']
										_snake_case													=			[
										    'Summary of the text.',
										]
										for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
															_snake_case													=			tokenizer(lowerCAmelCase_			,  return_tensors='pt'					)
															_snake_case													=			tokenizer(text_target=lowerCAmelCase_			,  return_tensors='pt'					)
															_snake_case													=			inputs['input_ids']
															_snake_case													=			targets['input_ids']
															self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()					)
															self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()					)
															self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()					)
															self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()					)




					@require_torch
					def 		lowerCAmelCase     (						self					)						->							List[str]:
										for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
															_snake_case													=			['Summary of the text.', 'Another summary.']
															_snake_case													=			[[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

															_snake_case													=			tokenizer(lowerCAmelCase_			,  padding=lowerCAmelCase_					)
															_snake_case													=			[[0] * len(lowerCAmelCase_					) for x in encoded_output['input_ids']]
															_snake_case													=			tokenizer.pad(lowerCAmelCase_					)
															self.assertSequenceEqual(outputs['global_attention_mask']			,  lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										pass






					def 		lowerCAmelCase     (						self					)						->							str:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)
																				_snake_case													=			self.tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)
																				_snake_case													=			'A, <mask> AllenNLP sentence.'
																				_snake_case													=			tokenizer_r.encode_plus(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_			,  return_token_type_ids=lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.encode_plus(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_			,  return_token_type_ids=lowerCAmelCase_					)
																				self.assertEqual(sum(tokens_r['token_type_ids']					)			,  sum(tokens_p['token_type_ids']					)					)
																				self.assertEqual(
																				    sum(tokens_r['attention_mask']					) / len(tokens_r['attention_mask']					)			,  sum(tokens_p['attention_mask']					) / len(tokens_p['attention_mask']					)			,  )

																				_snake_case													=			tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']					)
																				_snake_case													=			tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']					)
																				self.assertSequenceEqual(tokens_p['input_ids']			,  [0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]					)
																				self.assertSequenceEqual(tokens_r['input_ids']			,  [0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]					)

																				self.assertSequenceEqual(
																				    lowerCAmelCase_			,  ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']					)
																				self.assertSequenceEqual(
																				    lowerCAmelCase_			,  ['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']					)



