


import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin

class 						__a (							a_       , unittest.TestCase     ):
		_a      :    Dict           =      CanineTokenizer
		_a      :    Any           =      False
		def       UpperCAmelCase__							(     self		)					->			Optional[Any]:




									"""simple docstring"""





									super().setUp()
									_UpperCAmelCase											=     CanineTokenizer()
									tokenizer.save_pretrained(self.tmpdirname		)
		@cached_property
		def       UpperCAmelCase__							(     self		)					->			Union[str, Any]:




									"""simple docstring"""





									return CanineTokenizer.from_pretrained('google/canine-s'		)
		def       UpperCAmelCase__							(     self     ,       **_SCREAMING_SNAKE_CASE		)					->			CanineTokenizer:




									"""simple docstring"""





									_UpperCAmelCase											=     self.tokenizer_class.from_pretrained(self.tmpdirname     ,       **_lowercase		)
									_UpperCAmelCase											=     1024
									return tokenizer
		@require_torch
		def       UpperCAmelCase__							(     self		)					->			Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase											=     self.canine_tokenizer
									_UpperCAmelCase											=     ["""Life is like a box of chocolates.""", """You never know what you're gonna get."""]
									# fmt: off
									_UpperCAmelCase											=     [57344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 57345, 0, 0, 0, 0]
									# fmt: on
									_UpperCAmelCase											=     tokenizer(_lowercase     ,       padding=_lowercase     ,       return_tensors='pt'		)
									self.assertIsInstance(_lowercase     ,       _lowercase		)

									_UpperCAmelCase											=     list(batch.input_ids.numpy()[0]		)

									self.assertListEqual(_lowercase     ,       _lowercase		)

									self.assertEqual((2, 39)     ,       batch.input_ids.shape		)
									self.assertEqual((2, 39)     ,       batch.attention_mask.shape		)
		@require_torch
		def       UpperCAmelCase__							(     self		)					->			List[str]:




									"""simple docstring"""





									_UpperCAmelCase											=     self.canine_tokenizer
									_UpperCAmelCase											=     ["""Once there was a man.""", """He wrote a test in HuggingFace Tranformers."""]
									_UpperCAmelCase											=     tokenizer(_lowercase     ,       padding=_lowercase     ,       return_tensors='pt'		)
									# check if input_ids, attention_mask and token_type_ids are returned
									self.assertIn('input_ids'     ,       _lowercase		)
									self.assertIn('attention_mask'     ,       _lowercase		)
									self.assertIn('token_type_ids'     ,       _lowercase		)
		@require_torch
		def       UpperCAmelCase__							(     self		)					->			Dict:




									"""simple docstring"""





									_UpperCAmelCase											=     self.canine_tokenizer
									_UpperCAmelCase											=     [
									    """What's the weater?""",
									    """It's about 25 degrees.""",
									]
									_UpperCAmelCase											=     tokenizer(
									    text_target=_lowercase     ,       max_length=32     ,       padding='max_length'     ,       truncation=_lowercase     ,       return_tensors='pt'		)
									self.assertEqual(32     ,       targets['input_ids'].shape[1]		)
		def       UpperCAmelCase__							(     self		)					->			Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase											=     self.get_tokenizers()
									for tokenizer in tokenizers:
																with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
																							self.assertNotEqual(tokenizer.model_max_length     ,       42		)

        # Now let's start the test
									_UpperCAmelCase											=     self.get_tokenizers()
									for tokenizer in tokenizers:
																with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
																							# Isolate this from the other tests because we save additional tokens/etc
																							_UpperCAmelCase											=     tempfile.mkdtemp()

																							_UpperCAmelCase											=     """ He is very happy, UNwant\u00E9d,running"""
																							_UpperCAmelCase											=     tokenizer.encode(_lowercase     ,       add_special_tokens=_lowercase		)
																							tokenizer.save_pretrained(_lowercase		)

																							_UpperCAmelCase											=     tokenizer.__class__.from_pretrained(_lowercase		)
																							_UpperCAmelCase											=     after_tokenizer.encode(_lowercase     ,       add_special_tokens=_lowercase		)
																							self.assertListEqual(_lowercase     ,       _lowercase		)

																							shutil.rmtree(_lowercase		)

									_UpperCAmelCase											=     self.get_tokenizers(model_max_length=42		)
									for tokenizer in tokenizers:
																with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
																							# Isolate this from the other tests because we save additional tokens/etc
																							_UpperCAmelCase											=     tempfile.mkdtemp()

																							_UpperCAmelCase											=     """ He is very happy, UNwant\u00E9d,running"""

																							_UpperCAmelCase											=     tokenizer.additional_special_tokens

																							# We can add a new special token for Canine as follows:
																							_UpperCAmelCase											=     chr(0XE_007		)
																							additional_special_tokens.append(_lowercase		)
																							tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}		)
																							_UpperCAmelCase											=     tokenizer.encode(_lowercase     ,       add_special_tokens=_lowercase		)
																							tokenizer.save_pretrained(_lowercase		)

																							_UpperCAmelCase											=     tokenizer.__class__.from_pretrained(_lowercase		)
																							_UpperCAmelCase											=     after_tokenizer.encode(_lowercase     ,       add_special_tokens=_lowercase		)
																							self.assertListEqual(_lowercase     ,       _lowercase		)
																							self.assertIn(_lowercase     ,       after_tokenizer.additional_special_tokens		)
																							self.assertEqual(after_tokenizer.model_max_length     ,       42		)

																							_UpperCAmelCase											=     tokenizer.__class__.from_pretrained(_lowercase     ,       model_max_length=43		)
																							self.assertEqual(tokenizer.model_max_length     ,       43		)

																							shutil.rmtree(_lowercase		)
		def       UpperCAmelCase__							(     self		)					->			Any:




									"""simple docstring"""





									_UpperCAmelCase											=     self.get_tokenizers(do_lower_case=_lowercase		)
									for tokenizer in tokenizers:
																with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
																							_UpperCAmelCase											=     self.get_clean_sequence(_lowercase		)

																							# a special token for Canine can be defined as follows:
																							_UpperCAmelCase											=     0XE_005
																							_UpperCAmelCase											=     chr(_lowercase		)

																							tokenizer.add_special_tokens({'cls_token': special_token}		)
																							_UpperCAmelCase											=     tokenizer.encode(_lowercase     ,       add_special_tokens=_lowercase		)
																							self.assertEqual(len(_lowercase		)     ,       1		)

																							_UpperCAmelCase											=     tokenizer.decode(ids + encoded_special_token     ,       clean_up_tokenization_spaces=_lowercase		)
																							_UpperCAmelCase											=     tokenizer.encode(_lowercase     ,       add_special_tokens=_lowercase		)

																							_UpperCAmelCase											=     tokenizer.encode(_lowercase     ,       add_special_tokens=_lowercase		)
																							_UpperCAmelCase											=     tokenizer.encode(_lowercase     ,       add_special_tokens=_lowercase		)
																							self.assertEqual(_lowercase     ,       input_encoded + special_token_id		)

																							_UpperCAmelCase											=     tokenizer.decode(_lowercase     ,       skip_special_tokens=_lowercase		)
																							self.assertTrue(special_token not in decoded		)
		def       UpperCAmelCase__							(     self		)					->			str:




									"""simple docstring"""





									_UpperCAmelCase											=     self.get_tokenizers(do_lower_case=_lowercase		)
									for tokenizer in tokenizers:
																with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
																							_UpperCAmelCase											=     chr(0XE_005		)
																							_UpperCAmelCase											=     chr(0XE_006		)

																							# `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
																							tokenizer.add_tokens([SPECIAL_TOKEN_1]     ,       special_tokens=_lowercase		)
																							# `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
																							# which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
																							tokenizer.add_special_tokens({'additional_special_tokens': [SPECIAL_TOKEN_2]}		)

																							_UpperCAmelCase											=     tokenizer.tokenize(_lowercase		)
																							_UpperCAmelCase											=     tokenizer.tokenize(_lowercase		)

																							self.assertEqual(len(_lowercase		)     ,       1		)
																							self.assertEqual(len(_lowercase		)     ,       1		)
																							self.assertEqual(token_a[0]     ,       _lowercase		)
																							self.assertEqual(token_a[0]     ,       _lowercase		)
		@require_tokenizers
		def       UpperCAmelCase__							(     self		)					->			List[str]:




									"""simple docstring"""





									_UpperCAmelCase											=     self.get_tokenizers(do_lower_case=_lowercase		)
									for tokenizer in tokenizers:
																with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
																							# a special token for Canine can be defined as follows:
																							_UpperCAmelCase											=     0XE_006
																							_UpperCAmelCase											=     chr(_lowercase		)

																							_UpperCAmelCase											=     AddedToken(_lowercase     ,       lstrip=_lowercase		)
																							tokenizer.add_special_tokens({'additional_special_tokens': [new_token]}		)

																							with tempfile.TemporaryDirectory() as tmp_dir_name:
																														tokenizer.save_pretrained(_lowercase		)
																														tokenizer.from_pretrained(_lowercase		)
		def       UpperCAmelCase__							(     self		)					->			Dict:




									"""simple docstring"""





									_UpperCAmelCase											=     []
									if self.test_slow_tokenizer:
																tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())		)

									if self.test_rust_tokenizer:
																tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())		)

									for tokenizer_class, tokenizer_utils in tokenizer_list:
																with tempfile.TemporaryDirectory() as tmp_dir:
																							tokenizer_utils.save_pretrained(_lowercase		)

																							with open(os.path.join(_lowercase     ,       'special_tokens_map.json'		)     ,       encoding='utf-8'		) as json_file:
																														_UpperCAmelCase											=     json.load(_lowercase		)

																							with open(os.path.join(_lowercase     ,       'tokenizer_config.json'		)     ,       encoding='utf-8'		) as json_file:
																														_UpperCAmelCase											=     json.load(_lowercase		)

																							# a special token for Canine can be defined as follows:
																							_UpperCAmelCase											=     0XE_006
																							_UpperCAmelCase											=     chr(_lowercase		)

																							_UpperCAmelCase											=     [new_token_a]
																							_UpperCAmelCase											=     [new_token_a]

																							with open(os.path.join(_lowercase     ,       'special_tokens_map.json'		)     ,       'w'     ,       encoding='utf-8'		) as outfile:
																														json.dump(_lowercase     ,       _lowercase		)
																							with open(os.path.join(_lowercase     ,       'tokenizer_config.json'		)     ,       'w'     ,       encoding='utf-8'		) as outfile:
																														json.dump(_lowercase     ,       _lowercase		)

																							# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																							# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																							# "special_tokens_map.json" files
																							_UpperCAmelCase											=     tokenizer_class.from_pretrained(_lowercase     ,       extra_ids=0		)
																							self.assertIn(_lowercase     ,       tokenizer_without_change_in_init.additional_special_tokens		)
																							# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																							self.assertEqual(
																							    [new_token_a]     ,       tokenizer_without_change_in_init.convert_ids_to_tokens(
																							        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]		)		)     ,       )

																							_UpperCAmelCase											=     0XE_007
																							_UpperCAmelCase											=     chr(_lowercase		)
																							# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																							_UpperCAmelCase											=     [AddedToken(_lowercase     ,       lstrip=_lowercase		)]
																							_UpperCAmelCase											=     tokenizer_class.from_pretrained(
																							    _lowercase     ,       additional_special_tokens=_lowercase     ,       extra_ids=0		)

																							self.assertIn(_lowercase     ,       tokenizer.additional_special_tokens		)
																							# self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
																							self.assertEqual(
																							    [new_token_a]     ,       tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]		)		)		)
		@require_tokenizers
		def       UpperCAmelCase__							(     self		)					->			List[str]:




									"""simple docstring"""





									_UpperCAmelCase											=     self.get_tokenizers(do_lower_case=_lowercase		)
									for tokenizer in tokenizers:
																with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
																							_UpperCAmelCase											=     """hello world"""
																							if self.space_between_special_tokens:
																														_UpperCAmelCase											=     """[CLS] hello world [SEP]"""
																							else:
																														_UpperCAmelCase											=     input
																							_UpperCAmelCase											=     tokenizer.encode(_lowercase     ,       add_special_tokens=_lowercase		)
																							_UpperCAmelCase											=     tokenizer.decode(_lowercase     ,       spaces_between_special_tokens=self.space_between_special_tokens		)
																							self.assertIn(_lowercase     ,       [output, output.lower()]		)
		def       UpperCAmelCase__							(     self		)					->			Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase											=     self.get_tokenizers()
									for tokenizer in tokenizers:
																with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
																							_UpperCAmelCase											=     [
																							    """bos_token""",
																							    """eos_token""",
																							    """unk_token""",
																							    """sep_token""",
																							    """pad_token""",
																							    """cls_token""",
																							    """mask_token""",
																							]

									_UpperCAmelCase											=     """a"""
									_UpperCAmelCase											=     ord(_lowercase		)

									for attr in attributes_list:
																setattr(_lowercase     ,       attr + '_id'     ,       _lowercase		)
																self.assertEqual(getattr(_lowercase     ,       _lowercase		)     ,       _lowercase		)
																self.assertEqual(getattr(_lowercase     ,       attr + '_id'		)     ,       _lowercase		)

																setattr(_lowercase     ,       attr + '_id'     ,       _lowercase		)
																self.assertEqual(getattr(_lowercase     ,       _lowercase		)     ,       _lowercase		)
																self.assertEqual(getattr(_lowercase     ,       attr + '_id'		)     ,       _lowercase		)

									setattr(_lowercase     ,       'additional_special_tokens_ids'     ,       []		)
									self.assertListEqual(getattr(_lowercase     ,       'additional_special_tokens'		)     ,       []		)
									self.assertListEqual(getattr(_lowercase     ,       'additional_special_tokens_ids'		)     ,       []		)

									_UpperCAmelCase											=     0XE_006
									_UpperCAmelCase											=     chr(_lowercase		)
									setattr(_lowercase     ,       'additional_special_tokens_ids'     ,       [additional_special_token_id]		)
									self.assertListEqual(getattr(_lowercase     ,       'additional_special_tokens'		)     ,       [additional_special_token]		)
									self.assertListEqual(getattr(_lowercase     ,       'additional_special_tokens_ids'		)     ,       [additional_special_token_id]		)
		def       UpperCAmelCase__							(     self		)					->			Any:




									"""simple docstring"""





									pass
		def       UpperCAmelCase__							(     self		)					->			int:




									"""simple docstring"""





									pass
		def       UpperCAmelCase__							(     self		)					->			int:




									"""simple docstring"""





									pass
		def       UpperCAmelCase__							(     self		)					->			int:




									"""simple docstring"""





									pass
		def       UpperCAmelCase__							(     self		)					->			Dict:




									"""simple docstring"""





									pass
		def       UpperCAmelCase__							(     self		)					->			Optional[int]:




									"""simple docstring"""





									pass
		def       UpperCAmelCase__							(     self		)					->			List[Any]:




									"""simple docstring"""





									pass






		def       UpperCAmelCase__							(     self		)					->			Union[str, Any]:




									"""simple docstring"""





									pass



import unittest

from transformers import TrOCRConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers.models.trocr.modeling_trocr import TrOCRDecoder, TrOCRForCausalLM



@require_torch
class        A__:
						"""simple docstring"""






						def __init__(    self  ,      _lowercase  ,      _lowercase=99  ,      _lowercase=13  ,      _lowercase=16  ,      _lowercase=7  ,      _lowercase=True  ,      _lowercase=True  ,      _lowercase=True  ,      _lowercase=False  ,      _lowercase=True  ,      _lowercase=2  ,      _lowercase=32  ,      _lowercase=4  ,      _lowercase=4  ,      _lowercase=30  ,      _lowercase=0  ,      _lowercase=1  ,      _lowercase=2  ,      _lowercase=None  ,      )     -> List[str]:
									a_						:						Union[str, Any] 							= parent
									a_						:						Dict 							= batch_size
									a_						:						Any 							= decoder_seq_length
									# For common tests
									a_						:						Any 							= self.decoder_seq_length
									a_						:						int 							= is_training
									a_						:						List[Any] 							= use_attention_mask
									a_						:						List[str] 							= use_labels

									a_						:						Optional[Any] 							= vocab_size
									a_						:						Dict 							= d_model
									a_						:						Any 							= d_model
									a_						:						List[Any] 							= decoder_layers
									a_						:						List[str] 							= decoder_layers
									a_						:						Union[str, Any] 							= decoder_ffn_dim
									a_						:						Optional[Any] 							= decoder_attention_heads
									a_						:						Optional[Any] 							= decoder_attention_heads
									a_						:						Tuple 							= eos_token_id
									a_						:						Union[str, Any] 							= bos_token_id
									a_						:						Dict 							= pad_token_id
									a_						:						Union[str, Any] 							= decoder_start_token_id
									a_						:						List[Any] 							= use_cache
									a_						:						List[str] 							= max_position_embeddings

									a_						:						int 							= None
									a_						:						Any 							= decoder_seq_length
									a_						:						Union[str, Any] 							= 2
									a_						:						Optional[int] 							= 1





						def 			UpperCamelCase__					(    self    )     -> int:
									a_						:						Optional[int] 							= ids_tensor([self.batch_size, self.decoder_seq_length]  ,      self.vocab_size    )

									a_						:						Optional[int] 							= None
									if self.use_attention_mask:
												a_						:						List[Any] 							= ids_tensor([self.batch_size, self.decoder_seq_length]  ,      vocab_size=2    )

									a_						:						List[Any] 							= None
									if self.use_labels:
												a_						:						Union[str, Any] 							= ids_tensor([self.batch_size, self.decoder_seq_length]  ,      self.vocab_size    )

									a_						:						Any 							= TrOCRConfig(
									    vocab_size=self.vocab_size  ,      d_model=self.d_model  ,      decoder_layers=self.decoder_layers  ,      decoder_ffn_dim=self.decoder_ffn_dim  ,      decoder_attention_heads=self.decoder_attention_heads  ,      eos_token_id=self.eos_token_id  ,      bos_token_id=self.bos_token_id  ,      use_cache=self.use_cache  ,      pad_token_id=self.pad_token_id  ,      decoder_start_token_id=self.decoder_start_token_id  ,      max_position_embeddings=self.max_position_embeddings  ,      )

									return (config, input_ids, attention_mask, lm_labels)





						def 			UpperCamelCase__					(    self  ,      _lowercase  ,      _lowercase  ,      _lowercase  ,      _lowercase  ,      )     -> List[Any]:
									a_						:						Optional[int] 							= True
									a_						:						List[str] 							= TrOCRDecoder(config=_lowercase    ).to(_lowercase    ).eval()
									a_						:						str 							= input_ids[:2]

									input_ids[input_ids == 0] += 1
									# first forward pass
									a_						:						Dict 							= model(_lowercase  ,      use_cache=_lowercase    )
									a_						:						Union[str, Any] 							= model(_lowercase    )
									a_						:						Union[str, Any] 							= model(_lowercase  ,      use_cache=_lowercase    )

									self.parent.assertTrue(len(_lowercase    ) == len(_lowercase    )    )
									self.parent.assertTrue(len(_lowercase    ) == len(_lowercase    ) + 1    )

									a_						:						Dict 							= outputs["""past_key_values"""]

									# create hypothetical next token and extent to next_input_ids
									a_						:						List[str] 							= ids_tensor((2, 1)  ,      config.vocab_size - 1    ) + 1

									# append to next input_ids and
									a_						:						int 							= torch.cat([input_ids, next_tokens]  ,      dim=-1    )

									a_						:						Dict 							= model(_lowercase    )["""last_hidden_state"""]
									a_						:						List[str] 							= model(_lowercase  ,      past_key_values=_lowercase    )["""last_hidden_state"""]

									# select random slice
									a_						:						Optional[Any] 							= ids_tensor((1,)  ,      output_from_past.shape[-1]    ).item()
									a_						:						Union[str, Any] 							= output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
									a_						:						Any 							= output_from_past[:, 0, random_slice_idx].detach()

									# test that outputs are equal for slice
									assert torch.allclose(_lowercase  ,      _lowercase  ,      atol=1e-3    )





						def 			UpperCamelCase__					(    self    )     -> Optional[Any]:
									a_						:						List[Any] 							= self.prepare_config_and_inputs()
									a_							,					a_							,					a_							,					a_						:						List[Any] 							= config_and_inputs

									a_						:						str 							= {"""input_ids""": input_ids, """attention_mask""": attention_mask}
									return config, inputs_dict







@require_torch
class        A__(a_,  a_,  a_,  unittest.TestCase							):
						"""simple docstring"""






						_A    :				Optional[Any]       					=			(TrOCRDecoder, TrOCRForCausalLM) if is_torch_available() else ()
						_A    :				Optional[int]       					=			(TrOCRForCausalLM,) if is_torch_available() else ()
						_A    :				Tuple       					=			{'''text-generation''': TrOCRForCausalLM} if is_torch_available() else {}
						_A    :				List[str]       					=			True
						_A    :				List[str]       					=			False





						def 			UpperCamelCase__					(    self    )     -> List[Any]:
									a_						:						str 							= TrOCRStandaloneDecoderModelTester(self  ,      is_training=_lowercase    )
									a_						:						str 							= ConfigTester(self  ,      config_class=_lowercase    )





						def 			UpperCamelCase__					(    self    )     -> Tuple:
									pass





						def 			UpperCamelCase__					(    self    )     -> int:
									pass





						def 			UpperCamelCase__					(    self    )     -> str:
									pass





						def 			UpperCamelCase__					(    self    )     -> Dict:
									self.config_tester.run_common_tests()





						def 			UpperCamelCase__					(    self    )     -> List[str]:
									a_						:						int 							= self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_decoder_model_past(*_lowercase    )





						def 			UpperCamelCase__					(    self    )     -> Tuple:
									return





						@unittest.skip("""The model doesn't support left padding"""    )  # and it's not used enough to be worth fixing :)
						def 			UpperCamelCase__					(    self    )     -> Optional[Any]:
									pass


