





import unittest

from transformers import TrOCRConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers.models.trocr.modeling_trocr import TrOCRDecoder, TrOCRForCausalLM







@require_torch
class       __A							:




      """simple docstring"""







      def __init__(		self   ,				a__   ,				a__=99   ,				a__=13   ,				a__=16   ,				a__=7   ,				a__=True   ,				a__=True   ,				a__=True   ,				a__=False   ,				a__=True   ,				a__=2   ,				a__=32   ,				a__=4   ,				a__=4   ,				a__=30   ,				a__=0   ,				a__=1   ,				a__=2   ,				a__=None   ,				):



             """simple docstring"""





             _lowerCamelCase				:  int						=		parent
             _lowerCamelCase				:  Optional[Any]						=		batch_size
             _lowerCamelCase				:  Optional[Any]						=		decoder_seq_length
             # For common tests
             _lowerCamelCase				:  List[Any]						=		self.decoder_seq_length
             _lowerCamelCase				:  List[Any]						=		is_training
             _lowerCamelCase				:  Any						=		use_attention_mask
             _lowerCamelCase				:  Optional[Any]						=		use_labels

             _lowerCamelCase				:  Optional[int]						=		vocab_size
             _lowerCamelCase				:  List[Any]						=		d_model
             _lowerCamelCase				:  Optional[int]						=		d_model
             _lowerCamelCase				:  Tuple						=		decoder_layers
             _lowerCamelCase				:  List[Any]						=		decoder_layers
             _lowerCamelCase				:  Optional[int]						=		decoder_ffn_dim
             _lowerCamelCase				:  Optional[Any]						=		decoder_attention_heads
             _lowerCamelCase				:  Any						=		decoder_attention_heads
             _lowerCamelCase				:  Optional[int]						=		eos_token_id
             _lowerCamelCase				:  Tuple						=		bos_token_id
             _lowerCamelCase				:  Dict						=		pad_token_id
             _lowerCamelCase				:  Dict						=		decoder_start_token_id
             _lowerCamelCase				:  str						=		use_cache
             _lowerCamelCase				:  Union[str, Any]						=		max_position_embeddings

             _lowerCamelCase				:  Optional[int]						=		None
             _lowerCamelCase				:  str						=		decoder_seq_length
             _lowerCamelCase				:  Tuple						=		2
             _lowerCamelCase				:  Dict						=		1


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  str						=		ids_tensor([self.batch_size, self.decoder_seq_length]   ,				self.vocab_size)

             _lowerCamelCase				:  Dict						=		None
             if self.use_attention_mask:
                    _lowerCamelCase				:  Union[str, Any]						=		ids_tensor([self.batch_size, self.decoder_seq_length]   ,				vocab_size=2)

             _lowerCamelCase				:  List[Any]						=		None
             if self.use_labels:
                    _lowerCamelCase				:  List[Any]						=		ids_tensor([self.batch_size, self.decoder_seq_length]   ,				self.vocab_size)

             _lowerCamelCase				:  Dict						=		TrOCRConfig(
                 vocab_size=self.vocab_size   ,				d_model=self.d_model   ,				decoder_layers=self.decoder_layers   ,				decoder_ffn_dim=self.decoder_ffn_dim   ,				decoder_attention_heads=self.decoder_attention_heads   ,				eos_token_id=self.eos_token_id   ,				bos_token_id=self.bos_token_id   ,				use_cache=self.use_cache   ,				pad_token_id=self.pad_token_id   ,				decoder_start_token_id=self.decoder_start_token_id   ,				max_position_embeddings=self.max_position_embeddings   ,				)

             return (config, input_ids, attention_mask, lm_labels)


      def 					__snake_case      (		self   ,				a__   ,				a__   ,				a__   ,				a__   ,				):



             """simple docstring"""





             _lowerCamelCase				:  Any						=		True
             _lowerCamelCase				:  Dict						=		TrOCRDecoder(config=a__).to(a__).eval()
             _lowerCamelCase				:  List[Any]						=		input_ids[:2]

             input_ids[input_ids == 0] += 1
             # first forward pass
             _lowerCamelCase				:  List[str]						=		model(a__   ,				use_cache=a__)
             _lowerCamelCase				:  List[Any]						=		model(a__)
             _lowerCamelCase				:  List[str]						=		model(a__   ,				use_cache=a__)

             self.parent.assertTrue(len(a__) == len(a__))
             self.parent.assertTrue(len(a__) == len(a__) + 1)

             _lowerCamelCase				:  Union[str, Any]						=		outputs['''past_key_values''']

             # create hypothetical next token and extent to next_input_ids
             _lowerCamelCase				:  int						=		ids_tensor((2, 1)   ,				config.vocab_size - 1) + 1

             # append to next input_ids and
             _lowerCamelCase				:  List[Any]						=		torch.cat([input_ids, next_tokens]   ,				dim=-1)

             _lowerCamelCase				:  List[str]						=		model(a__)['''last_hidden_state''']
             _lowerCamelCase				:  Dict						=		model(a__   ,				past_key_values=a__)['''last_hidden_state''']

             # select random slice
             _lowerCamelCase				:  Any						=		ids_tensor((1,)   ,				output_from_past.shape[-1]).item()
             _lowerCamelCase				:  Optional[Any]						=		output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
             _lowerCamelCase				:  List[Any]						=		output_from_past[:, 0, random_slice_idx].detach()

             # test that outputs are equal for slice
             assert torch.allclose(a__   ,				a__   ,				atol=1e-3)



      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Tuple						=		self.prepare_config_and_inputs()
             _lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase,	_lowerCamelCase				:  Dict						=		config_and_inputs

             _lowerCamelCase				:  List[Any]						=		{'''input_ids''': input_ids, '''attention_mask''': attention_mask}
             return config, inputs_dict







@require_torch
class       __A							(   lowerCamelCase__			,lowerCamelCase__			,lowerCamelCase__			,unittest.TestCase	):




      """simple docstring"""







      UpperCAmelCase__							=   (TrOCRDecoder, TrOCRForCausalLM) if is_torch_available() else ()
      UpperCAmelCase__							=   (TrOCRForCausalLM,) if is_torch_available() else ()
      UpperCAmelCase__							=   {"""text-generation""": TrOCRForCausalLM} if is_torch_available() else {}
      UpperCAmelCase__							=   True
      UpperCAmelCase__							=   False


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  List[str]						=		TrOCRStandaloneDecoderModelTester(self   ,				is_training=a__)
             _lowerCamelCase				:  Optional[Any]						=		ConfigTester(self   ,				config_class=a__)


      def 					__snake_case      (		self):



             """simple docstring"""





             pass


      def 					__snake_case      (		self):



             """simple docstring"""





             pass


      def 					__snake_case      (		self):



             """simple docstring"""





             pass


      def 					__snake_case      (		self):



             """simple docstring"""





             self.config_tester.run_common_tests()


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  List[Any]						=		self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_decoder_model_past(*a__)


      def 					__snake_case      (		self):



             """simple docstring"""





             return



      @unittest.skip('''The model doesn\'t support left padding''')  # and it's not used enough to be worth fixing :)
      def 					__snake_case      (		self):



             """simple docstring"""





             pass








import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class       __A							(   lowerCamelCase__			,unittest.TestCase	):




      """simple docstring"""







      UpperCAmelCase__							=   OpenAIGPTTokenizer
      UpperCAmelCase__							=   OpenAIGPTTokenizerFast
      UpperCAmelCase__							=   True
      UpperCAmelCase__							=   False


      def 					__snake_case      (		self):



             """simple docstring"""





             super().setUp()

             # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
             _lowerCamelCase				:  Tuple						=		[
                 '''l''',
                 '''o''',
                 '''w''',
                 '''e''',
                 '''r''',
                 '''s''',
                 '''t''',
                 '''i''',
                 '''d''',
                 '''n''',
                 '''w</w>''',
                 '''r</w>''',
                 '''t</w>''',
                 '''lo''',
                 '''low''',
                 '''er</w>''',
                 '''low</w>''',
                 '''lowest</w>''',
                 '''newer</w>''',
                 '''wider</w>''',
                 '''<unk>''',
             ]
             _lowerCamelCase				:  Any						=		dict(zip(a__   ,				range(len(a__))))
             _lowerCamelCase				:  List[str]						=		['''#version: 0.2''', '''l o''', '''lo w''', '''e r</w>''', '''''']

             _lowerCamelCase				:  str						=		os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''vocab_file'''])
             _lowerCamelCase				:  int						=		os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''merges_file'''])
             with open(self.vocab_file   ,				'''w''') as fp:
                    fp.write(json.dumps(a__))
             with open(self.merges_file   ,				'''w''') as fp:
                    fp.write('''\n'''.join(a__))


      def 					__snake_case      (		self   ,				a__):



             """simple docstring"""





             return "lower newer", "lower newer"


      def 					__snake_case      (		self):



             """simple docstring"""





             _lowerCamelCase				:  Optional[Any]						=		OpenAIGPTTokenizer(self.vocab_file   ,				self.merges_file)

             _lowerCamelCase				:  Dict						=		'''lower'''
             _lowerCamelCase				:  Optional[Any]						=		['''low''', '''er</w>''']
             _lowerCamelCase				:  List[Any]						=		tokenizer.tokenize(a__)
             self.assertListEqual(a__   ,				a__)

             _lowerCamelCase				:  Any						=		tokens + ['''<unk>''']
             _lowerCamelCase				:  Optional[int]						=		[14, 15, 20]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(a__)   ,				a__)


      def 					__snake_case      (		self   ,				a__=15):



             """simple docstring"""





             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                           _lowerCamelCase				:  Dict						=		self.rust_tokenizer_class.from_pretrained(a__   ,				**a__)

                           # Simple input
                           _lowerCamelCase				:  Optional[int]						=		'''This is a simple input'''
                           _lowerCamelCase				:  Optional[Any]						=		['''This is a simple input 1''', '''This is a simple input 2''']
                           _lowerCamelCase				:  Optional[int]						=		('''This is a simple input''', '''This is a pair''')
                           _lowerCamelCase				:  List[Any]						=		[
                               ('''This is a simple input 1''', '''This is a simple input 2'''),
                               ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                           ]

                           # Simple input tests
                           self.assertRaises(a__   ,				tokenizer_r.encode   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Simple input
                           self.assertRaises(a__   ,				tokenizer_r.encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Simple input
                           self.assertRaises(
                               a__   ,				tokenizer_r.batch_encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length'''   ,				)

                           # Pair input
                           self.assertRaises(a__   ,				tokenizer_r.encode   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Pair input
                           self.assertRaises(a__   ,				tokenizer_r.encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length''')

                           # Pair input
                           self.assertRaises(
                               a__   ,				tokenizer_r.batch_encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length'''   ,				)



      def 					__snake_case      (		self):



             """simple docstring"""





             pass







@require_ftfy
@require_spacy
@require_tokenizers
class       __A							(   lowerCamelCase__	):




      """simple docstring"""







      pass

