

"""simple docstring"""




import os
import unittest

from transformers import LxmertTokenizer, LxmertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class   __lowercase      (     __lowerCamelCase						,      unittest.TestCase       ):
  snake_case_												=							LxmertTokenizer
  snake_case_												=							LxmertTokenizerFast
  snake_case_												=							True
  snake_case_												=							True

  def  __lowercase							(	self		:      Optional[Any]     ):






   '''simple docstring'''
   super().setUp()

   UpperCAmelCase__       :	int        =       [
       """[UNK]""",
       """[CLS]""",
       """[SEP]""",
       """want""",
       """##want""",
       """##ed""",
       """wa""",
       """un""",
       """runn""",
       """##ing""",
       """,""",
       """low""",
       """lowest""",
   ]
   UpperCAmelCase__       :	Any        =       os.path.join(self.tmpdirname       ,VOCAB_FILES_NAMES["""vocab_file"""]     )
   with open(self.vocab_file       ,"""w"""       ,encoding="""utf-8"""     ) as vocab_writer:
    vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]     )     )

  def  __lowercase							(	self		:      Union[str, Any]       ,A		:      Any     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Tuple        =       """UNwant\u00E9d,running"""
   UpperCAmelCase__       :	int        =       """unwanted, running"""
   return input_text, output_text

  def  __lowercase							(	self		:      Optional[int]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Any        =       self.tokenizer_class(self.vocab_file     )

   UpperCAmelCase__       :	List[Any]        =       tokenizer.tokenize("""UNwant\u00E9d,running"""     )
   self.assertListEqual(A       ,["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]     )
   self.assertListEqual(tokenizer.convert_tokens_to_ids(A     )       ,[7, 4, 5, 10, 8, 9]     )




  def  __lowercase							(	self		:      Any     ):






   '''simple docstring'''
   if not self.test_rust_tokenizer:
    return

   UpperCAmelCase__       :	int        =       self.get_tokenizer()
   UpperCAmelCase__       :	Optional[int]        =       self.get_rust_tokenizer()

   UpperCAmelCase__       :	Union[str, Any]        =       """I was born in 92000, and this is falsé."""

   UpperCAmelCase__       :	List[Any]        =       tokenizer.tokenize(A     )
   UpperCAmelCase__       :	Optional[Any]        =       rust_tokenizer.tokenize(A     )
   self.assertListEqual(A       ,A     )

   UpperCAmelCase__       :	Dict        =       tokenizer.encode(A       ,add_special_tokens=A     )
   UpperCAmelCase__       :	int        =       rust_tokenizer.encode(A       ,add_special_tokens=A     )
   self.assertListEqual(A       ,A     )

   UpperCAmelCase__       :	Any        =       self.get_rust_tokenizer()
   UpperCAmelCase__       :	int        =       tokenizer.encode(A     )
   UpperCAmelCase__       :	Any        =       rust_tokenizer.encode(A     )
   self.assertListEqual(A       ,A     )













import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    AutoConfig,
    AutoImageProcessor,
    CLIPConfig,
    CLIPImageProcessor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_image_processing import CustomImageProcessor  # noqa E402






class 					SCREAMING_SNAKE_CASE_  (       unittest.TestCase ):
       """simple docstring"""


       def 				UpperCamelCase__						(				self	:Any):



          """simple docstring"""

          _lowercase              =0





       def 				UpperCamelCase__						(				self	:Optional[int]):



          """simple docstring"""

          _lowercase              =AutoImageProcessor.from_pretrained('openai/clip-vit-base-patch32')
          self.assertIsInstance(snake_case,      snake_case)





       def 				UpperCamelCase__						(				self	:Union[str, Any]):



          """simple docstring"""

          with tempfile.TemporaryDirectory() as tmpdirname:
             _lowercase              =Path(snake_case) / 'preprocessor_config.json'
             _lowercase              =Path(snake_case) / 'config.json'
             json.dump(
                 {'image_processor_type': 'CLIPImageProcessor', 'processor_class': 'CLIPProcessor'},      open(snake_case,      'w'),      )
             json.dump({'model_type': 'clip'},      open(snake_case,      'w'))

             _lowercase              =AutoImageProcessor.from_pretrained(snake_case)
             self.assertIsInstance(snake_case,      snake_case)





       def 				UpperCamelCase__						(				self	:Union[str, Any]):



          """simple docstring"""

          with tempfile.TemporaryDirectory() as tmpdirname:
             _lowercase              =Path(snake_case) / 'preprocessor_config.json'
             _lowercase              =Path(snake_case) / 'config.json'
             json.dump(
                 {'feature_extractor_type': 'CLIPFeatureExtractor', 'processor_class': 'CLIPProcessor'},      open(snake_case,      'w'),      )
             json.dump({'model_type': 'clip'},      open(snake_case,      'w'))

             _lowercase              =AutoImageProcessor.from_pretrained(snake_case)
             self.assertIsInstance(snake_case,      snake_case)





       def 				UpperCamelCase__						(				self	:Union[str, Any]):



          """simple docstring"""

          with tempfile.TemporaryDirectory() as tmpdirname:
             _lowercase              =CLIPConfig()

             # Create a dummy config file with image_proceesor_type
             _lowercase              =Path(snake_case) / 'preprocessor_config.json'
             _lowercase              =Path(snake_case) / 'config.json'
             json.dump(
                 {'image_processor_type': 'CLIPImageProcessor', 'processor_class': 'CLIPProcessor'},      open(snake_case,      'w'),      )
             json.dump({'model_type': 'clip'},      open(snake_case,      'w'))

             # remove image_processor_type to make sure config.json alone is enough to load image processor locally
             _lowercase              =AutoImageProcessor.from_pretrained(snake_case).to_dict()

             config_dict.pop('image_processor_type')
             _lowercase              =CLIPImageProcessor(**snake_case)

             # save in new folder
             model_config.save_pretrained(snake_case)
             config.save_pretrained(snake_case)

             _lowercase              =AutoImageProcessor.from_pretrained(snake_case)

             # make sure private variable is not incorrectly saved
             _lowercase              =json.loads(config.to_json_string())
             self.assertTrue('_processor_class' not in dict_as_saved)

          self.assertIsInstance(snake_case,      snake_case)





       def 				UpperCamelCase__						(				self	:Optional[int]):



          """simple docstring"""

          with tempfile.TemporaryDirectory() as tmpdirname:
             _lowercase              =Path(snake_case) / 'preprocessor_config.json'
             json.dump(
                 {'image_processor_type': 'CLIPImageProcessor', 'processor_class': 'CLIPProcessor'},      open(snake_case,      'w'),      )

             _lowercase              =AutoImageProcessor.from_pretrained(snake_case)
             self.assertIsInstance(snake_case,      snake_case)





       def 				UpperCamelCase__						(				self	:int):



          """simple docstring"""

          with self.assertRaisesRegex(
              snake_case,      'clip-base is not a local folder and is not a valid model identifier'):
             _lowercase              =AutoImageProcessor.from_pretrained('clip-base')





       def 				UpperCamelCase__						(				self	:Dict):



          """simple docstring"""

          with self.assertRaisesRegex(
              snake_case,      r'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'):
             _lowercase              =AutoImageProcessor.from_pretrained(snake_case,      revision='aaaaaa')





       def 				UpperCamelCase__						(				self	:List[Any]):



          """simple docstring"""

          with self.assertRaisesRegex(
              snake_case,      'hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json.',      ):
             _lowercase              =AutoImageProcessor.from_pretrained('hf-internal-testing/config-no-model')





       def 				UpperCamelCase__						(				self	:Optional[Any]):



          """simple docstring"""

          with self.assertRaises(snake_case):
             _lowercase              =AutoImageProcessor.from_pretrained('hf-internal-testing/test_dynamic_image_processor')
          # If remote code is disabled, we can't load this config.
          with self.assertRaises(snake_case):
             _lowercase              =AutoImageProcessor.from_pretrained(
                 'hf-internal-testing/test_dynamic_image_processor',      trust_remote_code=snake_case)

          _lowercase              =AutoImageProcessor.from_pretrained(
              'hf-internal-testing/test_dynamic_image_processor',      trust_remote_code=snake_case)
          self.assertEqual(image_processor.__class__.__name__,      'NewImageProcessor')

          # Test image processor can be reloaded.
          with tempfile.TemporaryDirectory() as tmp_dir:
             image_processor.save_pretrained(snake_case)
             _lowercase              =AutoImageProcessor.from_pretrained(snake_case,      trust_remote_code=snake_case)
          self.assertEqual(reloaded_image_processor.__class__.__name__,      'NewImageProcessor')





       def 				UpperCamelCase__						(				self	:Optional[Any]):



          """simple docstring"""

          try:
             AutoConfig.register('custom',      snake_case)
             AutoImageProcessor.register(snake_case,      snake_case)
             # Trying to register something existing in the Transformers library will raise an error
             with self.assertRaises(snake_case):
                AutoImageProcessor.register(snake_case,      snake_case)

             with tempfile.TemporaryDirectory() as tmpdirname:
                _lowercase              =Path(snake_case) / 'preprocessor_config.json'
                _lowercase              =Path(snake_case) / 'config.json'
                json.dump(
                    {'feature_extractor_type': 'CLIPFeatureExtractor', 'processor_class': 'CLIPProcessor'},      open(snake_case,      'w'),      )
                json.dump({'model_type': 'clip'},      open(snake_case,      'w'))

                _lowercase              =CustomImageProcessor.from_pretrained(snake_case)

             # Now that the config is registered, it can be used as any other config with the auto-API
             with tempfile.TemporaryDirectory() as tmp_dir:
                image_processor.save_pretrained(snake_case)
                _lowercase              =AutoImageProcessor.from_pretrained(snake_case)
                self.assertIsInstance(snake_case,      snake_case)

          finally:
             if "custom" in CONFIG_MAPPING._extra_content:
                del CONFIG_MAPPING._extra_content["custom"]
             if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
                del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]





       def 				UpperCamelCase__						(				self	:str):



          """simple docstring"""

          class 					SCREAMING_SNAKE_CASE_  (       _a ):
                 """simple docstring"""


                 __lowerCAmelCase    :		Union[str, Any]					=True

          try:
             AutoConfig.register('custom',      snake_case)
             AutoImageProcessor.register(snake_case,      snake_case)
             # If remote code is not set, the default is to use local
             _lowercase              =AutoImageProcessor.from_pretrained('hf-internal-testing/test_dynamic_image_processor')
             self.assertEqual(image_processor.__class__.__name__,      'NewImageProcessor')
             self.assertTrue(image_processor.is_local)

             # If remote code is disabled, we load the local one.
             _lowercase              =AutoImageProcessor.from_pretrained(
                 'hf-internal-testing/test_dynamic_image_processor',      trust_remote_code=snake_case)
             self.assertEqual(image_processor.__class__.__name__,      'NewImageProcessor')
             self.assertTrue(image_processor.is_local)

             # If remote is enabled, we load from the Hub
             _lowercase              =AutoImageProcessor.from_pretrained(
                 'hf-internal-testing/test_dynamic_image_processor',      trust_remote_code=snake_case)
             self.assertEqual(image_processor.__class__.__name__,      'NewImageProcessor')
             self.assertTrue(not hasattr(snake_case,      'is_local'))

          finally:
             if "custom" in CONFIG_MAPPING._extra_content:
                del CONFIG_MAPPING._extra_content["custom"]
             if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
                del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]


