






import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
							from PIL import Image

							from transformers import CLIPImageProcessor, CLIPProcessor

@require_vision
class 				lowerCAmelCase_	(		unittest.TestCase			):


				def 						__a					(					self      ):
											_lowercase							:   str 							=   tempfile.mkdtemp()

											# fmt: off
											_lowercase							:   str 							=   ['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
											# fmt: on
											_lowercase							:   Any 							=   dict(zip(_lowerCAmelCase ,						range(len(_lowerCAmelCase      )      )      )      )
											_lowercase							:   Any 							=   ['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>', '']
											_lowercase							:   List[str] 							=   {'unk_token': '<unk>'}

											_lowercase							:   Union[str, Any] 							=   os.path.join(self.tmpdirname ,						VOCAB_FILES_NAMES['vocab_file']      )
											_lowercase							:   Optional[Any] 							=   os.path.join(self.tmpdirname ,						VOCAB_FILES_NAMES['merges_file']      )
											with open(self.vocab_file ,						'w' ,						encoding='utf-8'      ) as fp:
																		fp.write(json.dumps(_lowerCAmelCase      ) + '\n'      )
											with open(self.merges_file ,						'w' ,						encoding='utf-8'      ) as fp:
																		fp.write('\n'.join(_lowerCAmelCase      )      )

											_lowercase							:   int 							=   {
											    'do_resize': True,
											    'size': 2_0,
											    'do_center_crop': True,
											    'crop_size': 1_8,
											    'do_normalize': True,
											    'image_mean': [0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73],
											    'image_std': [0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11],
											}
											_lowercase							:   str 							=   os.path.join(self.tmpdirname ,						_lowerCAmelCase      )
											with open(self.image_processor_file ,						'w' ,						encoding='utf-8'      ) as fp:
																		json.dump(_lowerCAmelCase ,						_lowerCAmelCase      )


				def 						__a					(					self ,						**_lowerCAmelCase      ):
											return CLIPTokenizer.from_pretrained(self.tmpdirname ,						**_lowerCAmelCase      )


				def 						__a					(					self ,						**_lowerCAmelCase      ):
											return CLIPTokenizerFast.from_pretrained(self.tmpdirname ,						**_lowerCAmelCase      )


				def 						__a					(					self ,						**_lowerCAmelCase      ):
											return CLIPImageProcessor.from_pretrained(self.tmpdirname ,						**_lowerCAmelCase      )


				def 						__a					(					self      ):
											shutil.rmtree(self.tmpdirname      )


				def 						__a					(					self      ):
											_lowercase							:   Dict 							=   [np.random.randint(2_5_5 ,						size=(3, 3_0, 4_0_0) ,						dtype=np.uinta      )]

											_lowercase							:   int 							=   [Image.fromarray(np.moveaxis(_lowerCAmelCase ,						0 ,						-1      )      ) for x in image_inputs]

											return image_inputs


				def 						__a					(					self      ):
											_lowercase							:   Dict 							=   self.get_tokenizer()
											_lowercase							:   Any 							=   self.get_rust_tokenizer()
											_lowercase							:   List[str] 							=   self.get_image_processor()

											_lowercase							:   List[Any] 							=   CLIPProcessor(tokenizer=_lowerCAmelCase ,						image_processor=_lowerCAmelCase      )
											processor_slow.save_pretrained(self.tmpdirname      )
											_lowercase							:   Union[str, Any] 							=   CLIPProcessor.from_pretrained(self.tmpdirname ,						use_fast=_lowerCAmelCase      )

											_lowercase							:   Dict 							=   CLIPProcessor(tokenizer=_lowerCAmelCase ,						image_processor=_lowerCAmelCase      )
											processor_fast.save_pretrained(self.tmpdirname      )
											_lowercase							:   int 							=   CLIPProcessor.from_pretrained(self.tmpdirname      )

											self.assertEqual(processor_slow.tokenizer.get_vocab() ,						tokenizer_slow.get_vocab()      )
											self.assertEqual(processor_fast.tokenizer.get_vocab() ,						tokenizer_fast.get_vocab()      )
											self.assertEqual(tokenizer_slow.get_vocab() ,						tokenizer_fast.get_vocab()      )
											self.assertIsInstance(processor_slow.tokenizer ,						_lowerCAmelCase      )
											self.assertIsInstance(processor_fast.tokenizer ,						_lowerCAmelCase      )

											self.assertEqual(processor_slow.image_processor.to_json_string() ,						image_processor.to_json_string()      )
											self.assertEqual(processor_fast.image_processor.to_json_string() ,						image_processor.to_json_string()      )
											self.assertIsInstance(processor_slow.image_processor ,						_lowerCAmelCase      )
											self.assertIsInstance(processor_fast.image_processor ,						_lowerCAmelCase      )


				def 						__a					(					self      ):
											_lowercase							:   Union[str, Any] 							=   CLIPProcessor(tokenizer=self.get_tokenizer() ,						image_processor=self.get_image_processor()      )
											processor.save_pretrained(self.tmpdirname      )

											_lowercase							:   Optional[Any] 							=   self.get_tokenizer(bos_token='(BOS)' ,						eos_token='(EOS)'      )
											_lowercase							:   List[str] 							=   self.get_image_processor(do_normalize=_lowerCAmelCase ,						padding_value=1.0      )

											_lowercase							:   List[Any] 							=   CLIPProcessor.from_pretrained(
											    self.tmpdirname ,						bos_token='(BOS)' ,						eos_token='(EOS)' ,						do_normalize=_lowerCAmelCase ,						padding_value=1.0      )

											self.assertEqual(processor.tokenizer.get_vocab() ,						tokenizer_add_kwargs.get_vocab()      )
											self.assertIsInstance(processor.tokenizer ,						_lowerCAmelCase      )

											self.assertEqual(processor.image_processor.to_json_string() ,						image_processor_add_kwargs.to_json_string()      )
											self.assertIsInstance(processor.image_processor ,						_lowerCAmelCase      )


				def 						__a					(					self      ):
											_lowercase							:   Dict 							=   self.get_image_processor()
											_lowercase							:   Union[str, Any] 							=   self.get_tokenizer()

											_lowercase							:   Any 							=   CLIPProcessor(tokenizer=_lowerCAmelCase ,						image_processor=_lowerCAmelCase      )

											_lowercase							:   Any 							=   self.prepare_image_inputs()

											_lowercase							:   int 							=   image_processor(_lowerCAmelCase ,						return_tensors='np'      )
											_lowercase							:   List[Any] 							=   processor(images=_lowerCAmelCase ,						return_tensors='np'      )

											for key in input_image_proc.keys():
																		self.assertAlmostEqual(input_image_proc[key].sum() ,						input_processor[key].sum() ,						delta=1E-2      )


				def 						__a					(					self      ):
											_lowercase							:   Any 							=   self.get_image_processor()
											_lowercase							:   Any 							=   self.get_tokenizer()

											_lowercase							:   str 							=   CLIPProcessor(tokenizer=_lowerCAmelCase ,						image_processor=_lowerCAmelCase      )

											_lowercase							:   Dict 							=   'lower newer'

											_lowercase							:   Any 							=   processor(text=_lowerCAmelCase      )

											_lowercase							:   Tuple 							=   tokenizer(_lowerCAmelCase      )

											for key in encoded_tok.keys():
																		self.assertListEqual(encoded_tok[key] ,						encoded_processor[key]      )


				def 						__a					(					self      ):
											_lowercase							:   int 							=   self.get_image_processor()
											_lowercase							:   Union[str, Any] 							=   self.get_tokenizer()

											_lowercase							:   int 							=   CLIPProcessor(tokenizer=_lowerCAmelCase ,						image_processor=_lowerCAmelCase      )

											_lowercase							:   int 							=   'lower newer'
											_lowercase							:   Optional[Any] 							=   self.prepare_image_inputs()

											_lowercase							:   List[str] 							=   processor(text=_lowerCAmelCase ,						images=_lowerCAmelCase      )

											self.assertListEqual(list(inputs.keys()      ) ,						['input_ids', 'attention_mask', 'pixel_values']      )

											# test if it raises when no input is passed
											with pytest.raises(_lowerCAmelCase      ):
																		processor()


				def 						__a					(					self      ):
											_lowercase							:   Optional[Any] 							=   self.get_image_processor()
											_lowercase							:   Tuple 							=   self.get_tokenizer()

											_lowercase							:   List[Any] 							=   CLIPProcessor(tokenizer=_lowerCAmelCase ,						image_processor=_lowerCAmelCase      )

											_lowercase							:   Dict 							=   [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

											_lowercase							:   int 							=   processor.batch_decode(_lowerCAmelCase      )
											_lowercase							:   Dict 							=   tokenizer.batch_decode(_lowerCAmelCase      )

											self.assertListEqual(_lowerCAmelCase ,						_lowerCAmelCase      )


				def 						__a					(					self      ):
											_lowercase							:   List[str] 							=   self.get_image_processor()
											_lowercase							:   Optional[Any] 							=   self.get_tokenizer()

											_lowercase							:   Any 							=   CLIPProcessor(tokenizer=_lowerCAmelCase ,						image_processor=_lowerCAmelCase      )

											_lowercase							:   List[Any] 							=   'lower newer'
											_lowercase							:   List[str] 							=   self.prepare_image_inputs()

											_lowercase							:   Dict 							=   processor(text=_lowerCAmelCase ,						images=_lowerCAmelCase      )

											self.assertListEqual(list(inputs.keys()      ) ,						processor.model_input_names      )






"""simple docstring"""





from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import numpy
				import tensorflow as tf

				from transformers import (
				    TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    BertConfig,
				    DPRConfig,
				    TFDPRContextEncoder,
				    TFDPRQuestionEncoder,
				    TFDPRReader,
				)






class        _UpperCAmelCase:



		def __init__(   self							,						__a							,						__a=13							,						__a=7							,						__a=True							,						__a=True							,						__a=True							,						__a=True							,						__a=99							,						__a=32							,						__a=2							,						__a=4							,						__a=37							,						__a="gelu"							,						__a=0.1							,						__a=0.1							,						__a=5_12							,						__a=16							,						__a=2							,						__a=0.02							,						__a=3							,						__a=4							,						__a=None							,						__a=0							,						)			->			Any:






							'''simple docstring'''
							_UpperCamelCase									=    parent
							_UpperCamelCase									=    batch_size
							_UpperCamelCase									=    seq_length
							_UpperCamelCase									=    is_training
							_UpperCamelCase									=    use_input_mask
							_UpperCamelCase									=    use_token_type_ids
							_UpperCamelCase									=    use_labels
							_UpperCamelCase									=    vocab_size
							_UpperCamelCase									=    hidden_size
							_UpperCamelCase									=    num_hidden_layers
							_UpperCamelCase									=    num_attention_heads
							_UpperCamelCase									=    intermediate_size
							_UpperCamelCase									=    hidden_act
							_UpperCamelCase									=    hidden_dropout_prob
							_UpperCamelCase									=    attention_probs_dropout_prob
							_UpperCamelCase									=    max_position_embeddings
							_UpperCamelCase									=    type_vocab_size
							_UpperCamelCase									=    type_sequence_label_size
							_UpperCamelCase									=    initializer_range
							_UpperCamelCase									=    num_labels
							_UpperCamelCase									=    num_choices
							_UpperCamelCase									=    scope
							_UpperCamelCase									=    projection_dim



		def 		UpperCAmelCase				(   self)			->			List[Any]:






							'''simple docstring'''
							_UpperCamelCase									=    ids_tensor([self.batch_size, self.seq_length]							,						self.vocab_size)

							_UpperCamelCase									=    None
							if self.use_input_mask:
												# follow test_modeling_tf_ctrl.py
												_UpperCamelCase									=    random_attention_mask([self.batch_size, self.seq_length])

							_UpperCamelCase									=    None
							if self.use_token_type_ids:
												_UpperCamelCase									=    ids_tensor([self.batch_size, self.seq_length]							,						self.type_vocab_size)

							_UpperCamelCase									=    None
							_UpperCamelCase									=    None
							_UpperCamelCase									=    None
							if self.use_labels:
												_UpperCamelCase									=    ids_tensor([self.batch_size]							,						self.type_sequence_label_size)
												_UpperCamelCase									=    ids_tensor([self.batch_size, self.seq_length]							,						self.num_labels)
												_UpperCamelCase									=    ids_tensor([self.batch_size]							,						self.num_choices)

							_UpperCamelCase									=    BertConfig(
							    vocab_size=self.vocab_size							,						hidden_size=self.hidden_size							,						num_hidden_layers=self.num_hidden_layers							,						num_attention_heads=self.num_attention_heads							,						intermediate_size=self.intermediate_size							,						hidden_act=self.hidden_act							,						hidden_dropout_prob=self.hidden_dropout_prob							,						attention_probs_dropout_prob=self.attention_probs_dropout_prob							,						max_position_embeddings=self.max_position_embeddings							,						type_vocab_size=self.type_vocab_size							,						is_decoder=__a							,						initializer_range=self.initializer_range							,						)
							_UpperCamelCase									=    DPRConfig(projection_dim=self.projection_dim							,						**config.to_dict())

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a							,						__a							,						__a							,						__a							,						__a)			->			Optional[int]:






							'''simple docstring'''
							_UpperCamelCase									=    TFDPRContextEncoder(config=__a)
							_UpperCamelCase									=    model(__a							,						attention_mask=__a							,						token_type_ids=__a)
							_UpperCamelCase									=    model(__a							,						token_type_ids=__a)
							_UpperCamelCase									=    model(__a)
							self.parent.assertEqual(result.pooler_output.shape							,						(self.batch_size, self.projection_dim or self.hidden_size))



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a							,						__a							,						__a							,						__a							,						__a)			->			Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase									=    TFDPRQuestionEncoder(config=__a)
							_UpperCamelCase									=    model(__a							,						attention_mask=__a							,						token_type_ids=__a)
							_UpperCamelCase									=    model(__a							,						token_type_ids=__a)
							_UpperCamelCase									=    model(__a)
							self.parent.assertEqual(result.pooler_output.shape							,						(self.batch_size, self.projection_dim or self.hidden_size))



		def 		UpperCAmelCase				(   self							,						__a							,						__a							,						__a							,						__a							,						__a							,						__a							,						__a)			->			Dict:






							'''simple docstring'''
							_UpperCamelCase									=    TFDPRReader(config=__a)
							_UpperCamelCase									=    model(__a							,						attention_mask=__a)

							self.parent.assertEqual(result.start_logits.shape							,						(self.batch_size, self.seq_length))
							self.parent.assertEqual(result.end_logits.shape							,						(self.batch_size, self.seq_length))
							self.parent.assertEqual(result.relevance_logits.shape							,						(self.batch_size,))




		def 		UpperCAmelCase				(   self)			->			Any:






							'''simple docstring'''
							_UpperCamelCase									=    self.prepare_config_and_inputs()
							(
							    (
							    _UpperCamelCase
							)					,    (
							    _UpperCamelCase
							)					,    (
							    _UpperCamelCase
							)					,    (
							    _UpperCamelCase
							)					,    (
							    _UpperCamelCase
							)					,    (
							    _UpperCamelCase
							)					,    (
							    _UpperCamelCase
							)					,    
							)									=    config_and_inputs
							_UpperCamelCase									=    {'''input_ids''': input_ids}
							return config, inputs_dict






@require_tf
class        _UpperCAmelCase(       lowerCamelCase ,	lowerCamelCase ,	unittest.TestCase      ):
		lowercase__         = (
		    (
		        TFDPRContextEncoder,
		        TFDPRQuestionEncoder,
		        TFDPRReader,
		    )
		    if is_tf_available()
		    else ()
		)
		lowercase__         = {'feature-extraction': TFDPRQuestionEncoder} if is_tf_available() else {}

		lowercase__         = False
		lowercase__         = False
		lowercase__         = False
		lowercase__         = False
		lowercase__         = False



		def 		UpperCAmelCase				(   self)			->			Tuple:






							'''simple docstring'''
							_UpperCamelCase									=    TFDPRModelTester(self)
							_UpperCamelCase									=    ConfigTester(self							,						config_class=__a							,						hidden_size=37)



		def 		UpperCAmelCase				(   self)			->			Optional[int]:






							'''simple docstring'''
							self.config_tester.run_common_tests()



		def 		UpperCAmelCase				(   self)			->			Tuple:






							'''simple docstring'''
							_UpperCamelCase									=    self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_dpr_context_encoder(*__a)



		def 		UpperCAmelCase				(   self)			->			Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase									=    self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_dpr_question_encoder(*__a)



		def 		UpperCAmelCase				(   self)			->			Dict:






							'''simple docstring'''
							_UpperCamelCase									=    self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_dpr_reader(*__a)




		@slow
		def 		UpperCAmelCase				(   self)			->			str:






							'''simple docstring'''
							for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												_UpperCamelCase									=    TFDPRContextEncoder.from_pretrained(__a)
												self.assertIsNotNone(__a)

							for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												_UpperCamelCase									=    TFDPRContextEncoder.from_pretrained(__a)
												self.assertIsNotNone(__a)

							for model_name in TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												_UpperCamelCase									=    TFDPRQuestionEncoder.from_pretrained(__a)
												self.assertIsNotNone(__a)

							for model_name in TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												_UpperCamelCase									=    TFDPRReader.from_pretrained(__a)
												self.assertIsNotNone(__a)






@require_tf
class        _UpperCAmelCase(       unittest.TestCase      ):



		@slow
		def 		UpperCAmelCase				(   self)			->			Tuple:






							'''simple docstring'''
							_UpperCamelCase									=    TFDPRQuestionEncoder.from_pretrained('''facebook/dpr-question_encoder-single-nq-base''')

							_UpperCamelCase									=    tf.constant(
							    [[1_01, 75_92, 10_10, 20_03, 20_26, 38_99, 1_01_40, 10_29, 1_02]])  # [CLS] hello, is my dog cute? [SEP]
							_UpperCamelCase									=    model(__a)[0]  # embedding shape = (1, 768)
							# compare the actual values for a slice.
							_UpperCamelCase									=    tf.constant(
							    [
							        [
							            0.0323_6253,
							            0.1275_3335,
							            0.1681_8509,
							            0.0027_9786,
							            0.389_6933,
							            0.2426_4945,
							            0.217_8971,
							            -0.0233_5227,
							            -0.0848_1959,
							            -0.1432_4117,
							        ]
							    ])
							self.assertTrue(numpy.allclose(output[:, :10].numpy()							,						expected_slice.numpy()							,						atol=1e-4))






