






import json
import os
import shutil
import tempfile
import unittest

import numpy as np

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES, BertTokenizer
from transformers.testing_utils import require_tokenizers, require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
  from PIL import Image

  from transformers import VisionTextDualEncoderProcessor, ViTImageProcessor
@require_tokenizers
@require_vision
class 							A_    (						unittest.TestCase			):






 '''simple docstring'''




 def  snake_case__				(	self) ->		Optional[int]:

     """simple docstring"""



     _UpperCAmelCase				:      str			  =				tempfile.mkdtemp()

     # fmt: off
     _UpperCAmelCase				:      Tuple			  =				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''', ''',''', '''low''', '''lowest''']
     # fmt: on
     _UpperCAmelCase				:      Union[str, Any]			  =				os.path.join(self.tmpdirname							,  VOCAB_FILES_NAMES['''vocab_file'''])
     with open(self.vocab_file							,  '''w'''							,  encoding='''utf-8''') as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]))

     _UpperCAmelCase				:      Union[str, Any]			  =				{
         '''do_resize''': True,
         '''size''': {'''height''': 18, '''width''': 18},
         '''do_normalize''': True,
         '''image_mean''': [0.5, 0.5, 0.5],
         '''image_std''': [0.5, 0.5, 0.5],
     }
     _UpperCAmelCase				:      Dict			  =				os.path.join(self.tmpdirname							,  _A)
     with open(self.image_processor_file							,  '''w'''							,  encoding='''utf-8''') as fp:
         json.dump(_A							,  _A)


 def  snake_case__				(	self							,  **_A) ->		Any:

     """simple docstring"""



     return BertTokenizer.from_pretrained(self.tmpdirname							,  **_A)


 def  snake_case__				(	self							,  **_A) ->		Tuple:

     """simple docstring"""



     return ViTImageProcessor.from_pretrained(self.tmpdirname							,  **_A)


 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     shutil.rmtree(self.tmpdirname)


 def  snake_case__				(	self) ->		List[str]:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[Any]			  =				[np.random.randint(255							,  size=(3, 30, 400)							,  dtype=np.uinta)]

     _UpperCAmelCase				:      List[str]			  =				[Image.fromarray(np.moveaxis(_A							,  0							,  -1)) for x in image_inputs]

     return image_inputs


 def  snake_case__				(	self) ->		List[Any]:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[Any]			  =				self.get_tokenizer()
     _UpperCAmelCase				:      Optional[int]			  =				self.get_image_processor()

     _UpperCAmelCase				:      Optional[int]			  =				VisionTextDualEncoderProcessor(tokenizer=_A							,  image_processor=_A)

     processor.save_pretrained(self.tmpdirname)
     _UpperCAmelCase				:      int			  =				VisionTextDualEncoderProcessor.from_pretrained(self.tmpdirname)

     self.assertEqual(processor.tokenizer.get_vocab()							,  tokenizer.get_vocab())
     self.assertIsInstance(processor.tokenizer							,  (BertTokenizer, BertTokenizerFast))

     self.assertEqual(processor.image_processor.to_json_string()							,  image_processor.to_json_string())
     self.assertIsInstance(processor.image_processor							,  _A)


 def  snake_case__				(	self) ->		Tuple:

     """simple docstring"""



     _UpperCAmelCase				:      Any			  =				VisionTextDualEncoderProcessor(
         tokenizer=self.get_tokenizer()							,  image_processor=self.get_image_processor())
     processor.save_pretrained(self.tmpdirname)

     _UpperCAmelCase				:      List[Any]			  =				self.get_tokenizer(bos_token='''(BOS)'''							,  eos_token='''(EOS)''')
     _UpperCAmelCase				:      int			  =				self.get_image_processor(do_normalize=_A							,  padding_value=1.0)

     _UpperCAmelCase				:      Tuple			  =				VisionTextDualEncoderProcessor.from_pretrained(
         self.tmpdirname							,  bos_token='''(BOS)'''							,  eos_token='''(EOS)'''							,  do_normalize=_A							,  padding_value=1.0)

     self.assertEqual(processor.tokenizer.get_vocab()							,  tokenizer_add_kwargs.get_vocab())
     self.assertIsInstance(processor.tokenizer							,  (BertTokenizer, BertTokenizerFast))

     self.assertEqual(processor.image_processor.to_json_string()							,  image_processor_add_kwargs.to_json_string())
     self.assertIsInstance(processor.image_processor							,  _A)


 def  snake_case__				(	self) ->		Tuple:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[int]			  =				self.get_image_processor()
     _UpperCAmelCase				:      List[str]			  =				self.get_tokenizer()

     _UpperCAmelCase				:      str			  =				VisionTextDualEncoderProcessor(tokenizer=_A							,  image_processor=_A)

     _UpperCAmelCase				:      Tuple			  =				self.prepare_image_inputs()

     _UpperCAmelCase				:      Optional[Any]			  =				image_processor(_A							,  return_tensors='''np''')
     _UpperCAmelCase				:      str			  =				processor(images=_A							,  return_tensors='''np''')

     for key in input_feat_extract.keys():
         self.assertAlmostEqual(input_feat_extract[key].sum()							,  input_processor[key].sum()							,  delta=1e-2)


 def  snake_case__				(	self) ->		Any:

     """simple docstring"""



     _UpperCAmelCase				:      Union[str, Any]			  =				self.get_image_processor()
     _UpperCAmelCase				:      str			  =				self.get_tokenizer()

     _UpperCAmelCase				:      List[str]			  =				VisionTextDualEncoderProcessor(tokenizer=_A							,  image_processor=_A)

     _UpperCAmelCase				:      Dict			  =				'''lower newer'''

     _UpperCAmelCase				:      List[Any]			  =				processor(text=_A)

     _UpperCAmelCase				:      Any			  =				tokenizer(_A)

     for key in encoded_tok.keys():
         self.assertListEqual(encoded_tok[key]							,  encoded_processor[key])


 def  snake_case__				(	self) ->		Union[str, Any]:

     """simple docstring"""



     _UpperCAmelCase				:      Tuple			  =				self.get_image_processor()
     _UpperCAmelCase				:      List[Any]			  =				self.get_tokenizer()

     _UpperCAmelCase				:      List[str]			  =				VisionTextDualEncoderProcessor(tokenizer=_A							,  image_processor=_A)

     _UpperCAmelCase				:      int			  =				'''lower newer'''
     _UpperCAmelCase				:      Tuple			  =				self.prepare_image_inputs()

     _UpperCAmelCase				:      List[Any]			  =				processor(text=_A							,  images=_A)

     self.assertListEqual(list(inputs.keys())							,  ['''input_ids''', '''token_type_ids''', '''attention_mask''', '''pixel_values'''])

     # test if it raises when no input is passed
     with self.assertRaises(_A):
         processor()


 def  snake_case__				(	self) ->		Union[str, Any]:

     """simple docstring"""



     _UpperCAmelCase				:      Tuple			  =				self.get_image_processor()
     _UpperCAmelCase				:      Optional[Any]			  =				self.get_tokenizer()

     _UpperCAmelCase				:      Any			  =				VisionTextDualEncoderProcessor(tokenizer=_A							,  image_processor=_A)

     _UpperCAmelCase				:      int			  =				[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

     _UpperCAmelCase				:      Union[str, Any]			  =				processor.batch_decode(_A)
     _UpperCAmelCase				:      Optional[Any]			  =				tokenizer.batch_decode(_A)

     self.assertListEqual(_A							,  _A)


 def  snake_case__				(	self) ->		List[Any]:

     """simple docstring"""



     _UpperCAmelCase				:      List[Any]			  =				self.get_image_processor()
     _UpperCAmelCase				:      Union[str, Any]			  =				self.get_tokenizer()

     _UpperCAmelCase				:      Tuple			  =				VisionTextDualEncoderProcessor(tokenizer=_A							,  image_processor=_A)

     _UpperCAmelCase				:      int			  =				'''lower newer'''
     _UpperCAmelCase				:      Optional[int]			  =				self.prepare_image_inputs()

     _UpperCAmelCase				:      Any			  =				processor(text=_A							,  images=_A)

     self.assertListEqual(list(inputs.keys())							,  processor.model_input_names)












import importlib
import json
import os
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
import transformers.models.auto
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.bert.configuration_bert import BertConfig
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402


SCREAMING_SNAKE_CASE 							=						get_tests_dir('fixtures/dummy-config.json')
class 							A_    (						unittest.TestCase			):






 '''simple docstring'''




 def  snake_case__				(	self) ->		Optional[int]:

     """simple docstring"""



     _UpperCAmelCase				:      Union[str, Any]			  =				0


 def  snake_case__				(	self) ->		Union[str, Any]:

     """simple docstring"""



     self.assertIsNotNone(transformers.models.auto.__spec__)
     self.assertIsNotNone(importlib.util.find_spec('''transformers.models.auto'''))


 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     _UpperCAmelCase				:      List[str]			  =				AutoConfig.from_pretrained('''bert-base-uncased''')
     self.assertIsInstance(_A							,  _A)


 def  snake_case__				(	self) ->		Dict:

     """simple docstring"""



     _UpperCAmelCase				:      Tuple			  =				AutoConfig.from_pretrained(_A)
     self.assertIsInstance(_A							,  _A)


 def  snake_case__				(	self) ->		Optional[int]:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[Any]			  =				AutoConfig.from_pretrained(_A)
     self.assertIsInstance(_A							,  _A)


 def  snake_case__				(	self) ->		Any:

     """simple docstring"""



     _UpperCAmelCase				:      str			  =				AutoConfig.for_model('''roberta''')
     self.assertIsInstance(_A							,  _A)


 def  snake_case__				(	self) ->		int:

     """simple docstring"""



     with tempfile.TemporaryDirectory() as tmp_dir:
         # This model name contains bert and roberta, but roberta ends up being picked.
         _UpperCAmelCase				:      str			  =				os.path.join(_A							,  '''fake-roberta''')
         os.makedirs(_A							,  exist_ok=_A)
         with open(os.path.join(_A							,  '''config.json''')							,  '''w''') as f:
             f.write(json.dumps({}))
         _UpperCAmelCase				:      Any			  =				AutoConfig.from_pretrained(_A)
         self.assertEqual(type(_A)							,  _A)


 def  snake_case__				(	self) ->		List[Any]:

     """simple docstring"""



     try:
         AutoConfig.register('''custom'''							,  _A)
         # Wrong model type will raise an error
         with self.assertRaises(_A):
             AutoConfig.register('''model'''							,  _A)
         # Trying to register something existing in the Transformers library will raise an error
         with self.assertRaises(_A):
             AutoConfig.register('''bert'''							,  _A)

         # Now that the config is registered, it can be used as any other config with the auto-API
         _UpperCAmelCase				:      Tuple			  =				CustomConfig()
         with tempfile.TemporaryDirectory() as tmp_dir:
             config.save_pretrained(_A)
             _UpperCAmelCase				:      Tuple			  =				AutoConfig.from_pretrained(_A)
             self.assertIsInstance(_A							,  _A)

     finally:
         if "custom" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["custom"]


 def  snake_case__				(	self) ->		Union[str, Any]:

     """simple docstring"""



     with self.assertRaisesRegex(
         _A							,  '''bert-base is not a local folder and is not a valid model identifier'''):
         _UpperCAmelCase				:      Tuple			  =				AutoConfig.from_pretrained('''bert-base''')


 def  snake_case__				(	self) ->		int:

     """simple docstring"""



     with self.assertRaisesRegex(
         _A							,  R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''):
         _UpperCAmelCase				:      List[Any]			  =				AutoConfig.from_pretrained(_A							,  revision='''aaaaaa''')


 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     with self.assertRaisesRegex(
         _A							,  '''hf-internal-testing/no-config-test-repo does not appear to have a file named config.json.'''							,  ):
         _UpperCAmelCase				:      List[str]			  =				AutoConfig.from_pretrained('''hf-internal-testing/no-config-test-repo''')


 def  snake_case__				(	self) ->		List[str]:

     """simple docstring"""



     with self.assertRaises(_A):
         _UpperCAmelCase				:      Dict			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model''')
     # If remote code is disabled, we can't load this config.
     with self.assertRaises(_A):
         _UpperCAmelCase				:      Optional[int]			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''							,  trust_remote_code=_A)

     _UpperCAmelCase				:      int			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''							,  trust_remote_code=_A)
     self.assertEqual(config.__class__.__name__							,  '''NewModelConfig''')

     # Test config can be reloaded.
     with tempfile.TemporaryDirectory() as tmp_dir:
         config.save_pretrained(_A)
         _UpperCAmelCase				:      Any			  =				AutoConfig.from_pretrained(_A							,  trust_remote_code=_A)
     self.assertEqual(reloaded_config.__class__.__name__							,  '''NewModelConfig''')


 def  snake_case__				(	self) ->		Tuple:

     """simple docstring"""



     class 							A_    (						__lowercase			):






      '''simple docstring'''




      _SCREAMING_SNAKE_CASE							:			Union[str, Any] =					"new-model"

     try:
         AutoConfig.register('''new-model'''							,  _A)
         # If remote code is not set, the default is to use local
         _UpperCAmelCase				:      List[str]			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model''')
         self.assertEqual(config.__class__.__name__							,  '''NewModelConfigLocal''')

         # If remote code is disabled, we load the local one.
         _UpperCAmelCase				:      List[Any]			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''							,  trust_remote_code=_A)
         self.assertEqual(config.__class__.__name__							,  '''NewModelConfigLocal''')

         # If remote is enabled, we load from the Hub
         _UpperCAmelCase				:      Dict			  =				AutoConfig.from_pretrained('''hf-internal-testing/test_dynamic_model'''							,  trust_remote_code=_A)
         self.assertEqual(config.__class__.__name__							,  '''NewModelConfig''')

     finally:
         if "new-model" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["new-model"]




