


import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 		__magic_name__						(snake_case_   ,unittest.TestCase				):





     '''simple docstring'''



     __lowercase				:							Optional[Any]	   =       OpenAIGPTTokenizer
     __lowercase				:							int	   =       OpenAIGPTTokenizerFast
     __lowercase				:							str	   =       True
     __lowercase				:							Dict	   =       False

     def 	SCREAMING_SNAKE_CASE__						(					self:List[str]   ):
       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       snake_case__						= [
           '''l''',
           '''o''',
           '''w''',
           '''e''',
           '''r''',
           '''s''',
           '''t''',
           '''i''',
           '''d''',
           '''n''',
           '''w</w>''',
           '''r</w>''',
           '''t</w>''',
           '''lo''',
           '''low''',
           '''er</w>''',
           '''low</w>''',
           '''lowest</w>''',
           '''newer</w>''',
           '''wider</w>''',
           '''<unk>''',
       ]
       snake_case__						= dict(zip(_a     ,		range(len(_a   )   )   )   )
       snake_case__						= ['''#version: 0.2''', '''l o''', '''lo w''', '''e r</w>''', '''''']

       snake_case__						= os.path.join(self.tmpdirname     ,		VOCAB_FILES_NAMES['''vocab_file''']   )
       snake_case__						= os.path.join(self.tmpdirname     ,		VOCAB_FILES_NAMES['''merges_file''']   )
       with open(self.vocab_file     ,		'''w'''   ) as fp:
         fp.write(json.dumps(_a   )   )
       with open(self.merges_file     ,		'''w'''   ) as fp:
         fp.write('''\n'''.join(_a   )   )

     def 	SCREAMING_SNAKE_CASE__						(					self:int     ,		_a:Dict   ):
       return "lower newer", "lower newer"

     def 	SCREAMING_SNAKE_CASE__						(					self:Optional[Any]   ):
       snake_case__						= OpenAIGPTTokenizer(self.vocab_file     ,		self.merges_file   )

       snake_case__						= '''lower'''
       snake_case__						= ['''low''', '''er</w>''']
       snake_case__						= tokenizer.tokenize(_a   )
       self.assertListEqual(_a     ,		_a   )

       snake_case__						= tokens + ['''<unk>''']
       snake_case__						= [14, 15, 20]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_a   )     ,		_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Optional[int]     ,		_a:Dict=15   ):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""   ):
           snake_case__						= self.rust_tokenizer_class.from_pretrained(_a     ,		**_a   )

           # Simple input
           snake_case__						= '''This is a simple input'''
           snake_case__						= ['''This is a simple input 1''', '''This is a simple input 2''']
           snake_case__						= ('''This is a simple input''', '''This is a pair''')
           snake_case__						= [
               ('''This is a simple input 1''', '''This is a simple input 2'''),
               ('''This is a simple pair 1''', '''This is a simple pair 2'''),
           ]

           # Simple input tests
           self.assertRaises(_a     ,		tokenizer_r.encode     ,		_a     ,		max_length=_a     ,		padding='''max_length'''   )

           # Simple input
           self.assertRaises(_a     ,		tokenizer_r.encode_plus     ,		_a     ,		max_length=_a     ,		padding='''max_length'''   )

           # Simple input
           self.assertRaises(
               _a     ,		tokenizer_r.batch_encode_plus     ,		_a     ,		max_length=_a     ,		padding='''max_length'''     ,		)

           # Pair input
           self.assertRaises(_a     ,		tokenizer_r.encode     ,		_a     ,		max_length=_a     ,		padding='''max_length'''   )

           # Pair input
           self.assertRaises(_a     ,		tokenizer_r.encode_plus     ,		_a     ,		max_length=_a     ,		padding='''max_length'''   )

           # Pair input
           self.assertRaises(
               _a     ,		tokenizer_r.batch_encode_plus     ,		_a     ,		max_length=_a     ,		padding='''max_length'''     ,		)







     def 	SCREAMING_SNAKE_CASE__						(					self:Any   ):
       pass




@require_ftfy
@require_spacy
@require_tokenizers
class 		__magic_name__						(snake_case_				):





     '''simple docstring'''



     pass





import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    AutoConfig,
    AutoImageProcessor,
    CLIPConfig,
    CLIPImageProcessor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_image_processing import CustomImageProcessor  # noqa E402
class 		__magic_name__						(unittest.TestCase				):





     '''simple docstring'''



     def 	SCREAMING_SNAKE_CASE__						(					self:List[Any]   ):
       snake_case__						= 0

     def 	SCREAMING_SNAKE_CASE__						(					self:Optional[Any]   ):
       snake_case__						= AutoImageProcessor.from_pretrained('''openai/clip-vit-base-patch32'''   )
       self.assertIsInstance(_a     ,		_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:str   ):
       with tempfile.TemporaryDirectory() as tmpdirname:
         snake_case__						= Path(_a   ) / '''preprocessor_config.json'''
         snake_case__						= Path(_a   ) / '''config.json'''
         json.dump(
             {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}     ,		open(_a     ,		'''w'''   )     ,		)
         json.dump({'''model_type''': '''clip'''}     ,		open(_a     ,		'''w'''   )   )

         snake_case__						= AutoImageProcessor.from_pretrained(_a   )
         self.assertIsInstance(_a     ,		_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Dict   ):
       # Ensure we can load the image processor from the feature extractor config
       with tempfile.TemporaryDirectory() as tmpdirname:
         snake_case__						= Path(_a   ) / '''preprocessor_config.json'''
         snake_case__						= Path(_a   ) / '''config.json'''
         json.dump(
             {'''feature_extractor_type''': '''CLIPFeatureExtractor''', '''processor_class''': '''CLIPProcessor'''}     ,		open(_a     ,		'''w'''   )     ,		)
         json.dump({'''model_type''': '''clip'''}     ,		open(_a     ,		'''w'''   )   )

         snake_case__						= AutoImageProcessor.from_pretrained(_a   )
         self.assertIsInstance(_a     ,		_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Optional[int]   ):
       with tempfile.TemporaryDirectory() as tmpdirname:
         snake_case__						= CLIPConfig()

         # Create a dummy config file with image_proceesor_type
         snake_case__						= Path(_a   ) / '''preprocessor_config.json'''
         snake_case__						= Path(_a   ) / '''config.json'''
         json.dump(
             {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}     ,		open(_a     ,		'''w'''   )     ,		)
         json.dump({'''model_type''': '''clip'''}     ,		open(_a     ,		'''w'''   )   )

         # remove image_processor_type to make sure config.json alone is enough to load image processor locally
         snake_case__						= AutoImageProcessor.from_pretrained(_a   ).to_dict()

         config_dict.pop('''image_processor_type'''   )
         snake_case__						= CLIPImageProcessor(**_a   )

         # save in new folder
         model_config.save_pretrained(_a   )
         config.save_pretrained(_a   )

         snake_case__						= AutoImageProcessor.from_pretrained(_a   )

         # make sure private variable is not incorrectly saved
         snake_case__						= json.loads(config.to_json_string()   )
         self.assertTrue('''_processor_class''' not in dict_as_saved   )

       self.assertIsInstance(_a     ,		_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:List[str]   ):
       with tempfile.TemporaryDirectory() as tmpdirname:
         snake_case__						= Path(_a   ) / '''preprocessor_config.json'''
         json.dump(
             {'''image_processor_type''': '''CLIPImageProcessor''', '''processor_class''': '''CLIPProcessor'''}     ,		open(_a     ,		'''w'''   )     ,		)

         snake_case__						= AutoImageProcessor.from_pretrained(_a   )
         self.assertIsInstance(_a     ,		_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Dict   ):
       with self.assertRaisesRegex(
           _a     ,		'''clip-base is not a local folder and is not a valid model identifier'''   ):
         snake_case__						= AutoImageProcessor.from_pretrained('''clip-base'''   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Union[str, Any]   ):
       with self.assertRaisesRegex(
           _a     ,		r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''   ):
         snake_case__						= AutoImageProcessor.from_pretrained(_a     ,		revision='''aaaaaa'''   )

     def 	SCREAMING_SNAKE_CASE__						(					self:List[Any]   ):
       with self.assertRaisesRegex(
           _a     ,		'''hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json.'''     ,		):
         snake_case__						= AutoImageProcessor.from_pretrained('''hf-internal-testing/config-no-model'''   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Dict   ):
       # If remote code is not set, we will time out when asking whether to load the model.
       with self.assertRaises(_a   ):
         snake_case__						= AutoImageProcessor.from_pretrained('''hf-internal-testing/test_dynamic_image_processor'''   )
       # If remote code is disabled, we can't load this config.
       with self.assertRaises(_a   ):
         snake_case__						= AutoImageProcessor.from_pretrained(
             '''hf-internal-testing/test_dynamic_image_processor'''     ,		trust_remote_code=_a   )

       snake_case__						= AutoImageProcessor.from_pretrained(
           '''hf-internal-testing/test_dynamic_image_processor'''     ,		trust_remote_code=_a   )
       self.assertEqual(image_processor.__class__.__name__     ,		'''NewImageProcessor'''   )

       # Test image processor can be reloaded.
       with tempfile.TemporaryDirectory() as tmp_dir:
         image_processor.save_pretrained(_a   )
         snake_case__						= AutoImageProcessor.from_pretrained(_a     ,		trust_remote_code=_a   )
       self.assertEqual(reloaded_image_processor.__class__.__name__     ,		'''NewImageProcessor'''   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Dict   ):
       try:
         AutoConfig.register('''custom'''     ,		_a   )
         AutoImageProcessor.register(_a     ,		_a   )
         # Trying to register something existing in the Transformers library will raise an error
         with self.assertRaises(_a   ):
           AutoImageProcessor.register(_a     ,		_a   )

         with tempfile.TemporaryDirectory() as tmpdirname:
           snake_case__						= Path(_a   ) / '''preprocessor_config.json'''
           snake_case__						= Path(_a   ) / '''config.json'''
           json.dump(
               {'''feature_extractor_type''': '''CLIPFeatureExtractor''', '''processor_class''': '''CLIPProcessor'''}     ,		open(_a     ,		'''w'''   )     ,		)
           json.dump({'''model_type''': '''clip'''}     ,		open(_a     ,		'''w'''   )   )

           snake_case__						= CustomImageProcessor.from_pretrained(_a   )

         # Now that the config is registered, it can be used as any other config with the auto-API
         with tempfile.TemporaryDirectory() as tmp_dir:
           image_processor.save_pretrained(_a   )
           snake_case__						= AutoImageProcessor.from_pretrained(_a   )
           self.assertIsInstance(_a     ,		_a   )

       finally:
         if "custom" in CONFIG_MAPPING._extra_content:
           del CONFIG_MAPPING._extra_content["custom"]
         if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
           del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]







     def 	SCREAMING_SNAKE_CASE__						(					self:Optional[int]   ):
       class 		__magic_name__						(snake_case_				):





            '''simple docstring'''



            __lowercase				:							List[str]	   =       True

       try:
         AutoConfig.register('''custom'''     ,		_a   )
         AutoImageProcessor.register(_a     ,		_a   )
         # If remote code is not set, the default is to use local
         snake_case__						= AutoImageProcessor.from_pretrained('''hf-internal-testing/test_dynamic_image_processor'''   )
         self.assertEqual(image_processor.__class__.__name__     ,		'''NewImageProcessor'''   )
         self.assertTrue(image_processor.is_local   )

         # If remote code is disabled, we load the local one.
         snake_case__						= AutoImageProcessor.from_pretrained(
             '''hf-internal-testing/test_dynamic_image_processor'''     ,		trust_remote_code=_a   )
         self.assertEqual(image_processor.__class__.__name__     ,		'''NewImageProcessor'''   )
         self.assertTrue(image_processor.is_local   )

         # If remote is enabled, we load from the Hub
         snake_case__						= AutoImageProcessor.from_pretrained(
             '''hf-internal-testing/test_dynamic_image_processor'''     ,		trust_remote_code=_a   )
         self.assertEqual(image_processor.__class__.__name__     ,		'''NewImageProcessor'''   )
         self.assertTrue(not hasattr(_a     ,		'''is_local'''   )   )

       finally:
         if "custom" in CONFIG_MAPPING._extra_content:
           del CONFIG_MAPPING._extra_content["custom"]
         if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
           del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]

