

from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers import (
					    TFRoFormerForCausalLM,
					    TFRoFormerForMaskedLM,
					    TFRoFormerForMultipleChoice,
					    TFRoFormerForQuestionAnswering,
					    TFRoFormerForSequenceClassification,
					    TFRoFormerForTokenClassification,
					    TFRoFormerModel,
					)
					from transformers.models.roformer.modeling_tf_roformer import (
					    TFRoFormerSelfAttention,
					    TFRoFormerSinusoidalPositionalEmbedding,
					)







class 				lowercase       :




				def __init__(      self   ,     snake_case   ,     snake_case=13   ,     snake_case=7   ,     snake_case=True   ,     snake_case=True   ,     snake_case=True   ,     snake_case=True   ,     snake_case=99   ,     snake_case=32   ,     snake_case=2   ,     snake_case=4   ,     snake_case=37   ,     snake_case="gelu"   ,     snake_case=0.1   ,     snake_case=0.1   ,     snake_case=512   ,     snake_case=16   ,     snake_case=2   ,     snake_case=0.02   ,     snake_case=3   ,     snake_case=4   ,     snake_case=None   ,     ):
											snake_case_           =					parent
											snake_case_           =					13
											snake_case_           =					7
											snake_case_           =					True
											snake_case_           =					True
											snake_case_           =					True
											snake_case_           =					True
											snake_case_           =					99
											snake_case_           =					32
											snake_case_           =					2
											snake_case_           =					4
											snake_case_           =					37
											snake_case_           =					'gelu'
											snake_case_           =					0.1
											snake_case_           =					0.1
											snake_case_           =					512
											snake_case_           =					16
											snake_case_           =					2
											snake_case_           =					0.02
											snake_case_           =					3
											snake_case_           =					4
											snake_case_           =					None




				def 					a							(      self ):
											snake_case_           =					ids_tensor([self.batch_size, self.seq_length]   ,     self.vocab_size )

											snake_case_           =					None
											if self.use_input_mask:
																		snake_case_           =					random_attention_mask([self.batch_size, self.seq_length] )

											snake_case_           =					None
											if self.use_token_type_ids:
																		snake_case_           =					ids_tensor([self.batch_size, self.seq_length]   ,     self.type_vocab_size )

											snake_case_           =					None
											snake_case_           =					None
											snake_case_           =					None
											if self.use_labels:
																		snake_case_           =					ids_tensor([self.batch_size]   ,     self.type_sequence_label_size )
																		snake_case_           =					ids_tensor([self.batch_size, self.seq_length]   ,     self.num_labels )
																		snake_case_           =					ids_tensor([self.batch_size]   ,     self.num_choices )

											snake_case_           =					RoFormerConfig(
											    vocab_size=self.vocab_size   ,     hidden_size=self.hidden_size   ,     num_hidden_layers=self.num_hidden_layers   ,     num_attention_heads=self.num_attention_heads   ,     intermediate_size=self.intermediate_size   ,     hidden_act=self.hidden_act   ,     hidden_dropout_prob=self.hidden_dropout_prob   ,     attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,     max_position_embeddings=self.max_position_embeddings   ,     type_vocab_size=self.type_vocab_size   ,     initializer_range=self.initializer_range   ,     return_dict=snake_case   ,     )

											return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					TFRoFormerModel(config=snake_case )
											snake_case_           =					{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}

											snake_case_           =					[input_ids, input_mask]
											snake_case_           =					model(snake_case )

											snake_case_           =					model(snake_case )

											self.parent.assertEqual(result.last_hidden_state.shape   ,     (self.batch_size, self.seq_length, self.hidden_size) )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					True
											snake_case_           =					TFRoFormerForCausalLM(config=snake_case )
											snake_case_           =					{
											    'input_ids': input_ids,
											    'attention_mask': input_mask,
											    'token_type_ids': token_type_ids,
											}
											snake_case_           =					model(snake_case )['logits']
											self.parent.assertListEqual(
											    list(prediction_scores.numpy().shape )   ,     [self.batch_size, self.seq_length, self.vocab_size] )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					TFRoFormerForMaskedLM(config=snake_case )
											snake_case_           =					{
											    'input_ids': input_ids,
											    'attention_mask': input_mask,
											    'token_type_ids': token_type_ids,
											}
											snake_case_           =					model(snake_case )
											self.parent.assertEqual(result.logits.shape   ,     (self.batch_size, self.seq_length, self.vocab_size) )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					self.num_labels
											snake_case_           =					TFRoFormerForSequenceClassification(config=snake_case )
											snake_case_           =					{
											    'input_ids': input_ids,
											    'attention_mask': input_mask,
											    'token_type_ids': token_type_ids,
											}

											snake_case_           =					model(snake_case )
											self.parent.assertEqual(result.logits.shape   ,     (self.batch_size, self.num_labels) )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					self.num_choices
											snake_case_           =					TFRoFormerForMultipleChoice(config=snake_case )
											snake_case_           =					tf.tile(tf.expand_dims(snake_case   ,     1 )   ,     (1, self.num_choices, 1) )
											snake_case_           =					tf.tile(tf.expand_dims(snake_case   ,     1 )   ,     (1, self.num_choices, 1) )
											snake_case_           =					tf.tile(tf.expand_dims(snake_case   ,     1 )   ,     (1, self.num_choices, 1) )
											snake_case_           =					{
											    'input_ids': multiple_choice_inputs_ids,
											    'attention_mask': multiple_choice_input_mask,
											    'token_type_ids': multiple_choice_token_type_ids,
											}
											snake_case_           =					model(snake_case )
											self.parent.assertEqual(result.logits.shape   ,     (self.batch_size, self.num_choices) )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					self.num_labels
											snake_case_           =					TFRoFormerForTokenClassification(config=snake_case )
											snake_case_           =					{
											    'input_ids': input_ids,
											    'attention_mask': input_mask,
											    'token_type_ids': token_type_ids,
											}
											snake_case_           =					model(snake_case )
											self.parent.assertEqual(result.logits.shape   ,     (self.batch_size, self.seq_length, self.num_labels) )




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											snake_case_           =					TFRoFormerForQuestionAnswering(config=snake_case )
											snake_case_           =					{
											    'input_ids': input_ids,
											    'attention_mask': input_mask,
											    'token_type_ids': token_type_ids,
											}

											snake_case_           =					model(snake_case )
											self.parent.assertEqual(result.start_logits.shape   ,     (self.batch_size, self.seq_length) )
											self.parent.assertEqual(result.end_logits.shape   ,     (self.batch_size, self.seq_length) )




				def 					a							(      self ):
											snake_case_           =					self.prepare_config_and_inputs()
											(
											    (
											    snake_case_
											)       ,					(
											    snake_case_
											)       ,					(
											    snake_case_
											)       ,					(
											    snake_case_
											)       ,					(
											    snake_case_
											)       ,					(
											    snake_case_
											)       ,					(
											    snake_case_
											)       ,					
											)           =					config_and_inputs
											snake_case_           =					{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
											return config, inputs_dict







@require_tf
class 				lowercase       ( lowercase_			,						lowercase_			,						unittest.TestCase    ):
				__SCREAMING_SNAKE_CASE      :				Union[str, Any]     =    (
				    (
				        TFRoFormerModel,
				        TFRoFormerForCausalLM,
				        TFRoFormerForMaskedLM,
				        TFRoFormerForQuestionAnswering,
				        TFRoFormerForSequenceClassification,
				        TFRoFormerForTokenClassification,
				        TFRoFormerForMultipleChoice,
				    )
				    if is_tf_available()
				    else ()
				)
				__SCREAMING_SNAKE_CASE      :				List[Any]     =    (
				    {
				        '''feature-extraction''': TFRoFormerModel,
				        '''fill-mask''': TFRoFormerForMaskedLM,
				        '''question-answering''': TFRoFormerForQuestionAnswering,
				        '''text-classification''': TFRoFormerForSequenceClassification,
				        '''text-generation''': TFRoFormerForCausalLM,
				        '''token-classification''': TFRoFormerForTokenClassification,
				        '''zero-shot''': TFRoFormerForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)

				__SCREAMING_SNAKE_CASE      :				Tuple     =    False
				__SCREAMING_SNAKE_CASE      :				Union[str, Any]     =    False




				def 					a							(      self   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case   ,     snake_case ):
											if pipeline_test_casse_name == "TextGenerationPipelineTests":
																		return True

											return False




				def 					a							(      self ):
											snake_case_           =					TFRoFormerModelTester(self )
											snake_case_           =					ConfigTester(self   ,     config_class=snake_case   ,     hidden_size=37 )




				def 					a							(      self ):
											self.config_tester.run_common_tests()




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_lm(*snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_lm_head(*snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_multiple_choice(*snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*snake_case )




				def 					a							(      self ):
											snake_case_           =					self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*snake_case )




				@slow
				def 					a							(      self ):
											snake_case_           =					TFRoFormerModel.from_pretrained('junnyu/roformer_chinese_base' )
											self.assertIsNotNone(snake_case )







@require_tf
class 				lowercase       ( unittest.TestCase    ):




				@slow
				def 					a							(      self ):
											snake_case_           =					TFRoFormerForMaskedLM.from_pretrained('junnyu/roformer_chinese_base' )
											snake_case_           =					tf.constant([[0, 1, 2, 3, 4, 5]] )
											snake_case_           =					model(snake_case )[0]

											# TODO Replace vocab size
											snake_case_           =					5_0000

											snake_case_           =					[1, 6, vocab_size]
											self.assertEqual(output.shape   ,     snake_case )

											print(output[:, :3, :3] )

											# TODO Replace values below with what was printed above.
											snake_case_           =					tf.constant(
											    [
											        [
											            [-0.12_05_33_41, -1.0_26_49_01, 0.29_22_19_46],
											            [-1.5_13_37_83, 0.19_74_33, 0.15_19_06_07],
											            [-5.0_13_54_03, -3.90_02_56, -0.84_03_87_64],
											        ]
											    ] )
											tf.debugging.assert_near(output[:, :3, :3]   ,     snake_case   ,     atol=1e-4 )







@require_tf
class 				lowercase       ( unittest.TestCase    ):
				__SCREAMING_SNAKE_CASE      :				Dict     =    1e-4




				def 					a							(      self ):
											snake_case_           =					tf.constant([[4, 10]] )
											snake_case_           =					TFRoFormerSinusoidalPositionalEmbedding(num_positions=6   ,     embedding_dim=6 )

											snake_case_           =					emba(input_ids.shape )
											snake_case_           =					tf.constant(
											    [[0.00_00, 0.00_00, 0.00_00, 1.00_00, 1.00_00, 1.00_00], [0.84_15, 0.04_64, 0.00_22, 0.54_03, 0.99_89, 1.00_00]] )

											tf.debugging.assert_near(snake_case   ,     snake_case   ,     atol=self.tolerance )




				def 					a							(      self ):
											snake_case_           =					tf.constant(
											    [
											        [0.00_00, 0.00_00, 0.00_00, 0.00_00, 0.00_00],
											        [0.84_15, 0.82_19, 0.80_20, 0.78_19, 0.76_17],
											        [0.90_93, 0.93_64, 0.95_81, 0.97_49, 0.98_70],
											    ] )
											snake_case_           =					TFRoFormerSinusoidalPositionalEmbedding(num_positions=512   ,     embedding_dim=512 )
											emba([2, 16, 512] )
											snake_case_           =					emba.weight[:3, :5]

											tf.debugging.assert_near(snake_case   ,     snake_case   ,     atol=self.tolerance )







@require_tf
class 				lowercase       ( unittest.TestCase    ):
				__SCREAMING_SNAKE_CASE      :				str     =    1e-4




				def 					a							(      self ):
											# 2,12,16,64
											snake_case_           =					tf.reshape(tf.range(2 * 12 * 16 * 64   ,     dtype=tf.floataa )   ,     shape=(2, 12, 16, 64) ) / 100

											snake_case_           =					-tf.reshape(tf.range(2 * 12 * 16 * 64   ,     dtype=tf.floataa )   ,     shape=(2, 12, 16, 64) ) / 100

											snake_case_           =					TFRoFormerSinusoidalPositionalEmbedding(num_positions=32   ,     embedding_dim=64 )
											snake_case_           =					embed_positions([2, 16, 768] )[None, None, :, :]

											snake_case_       ,					snake_case_           =					TFRoFormerSelfAttention.apply_rotary_position_embeddings(
											    snake_case   ,     snake_case   ,     snake_case )

											snake_case_           =					tf.constant(
											    [
											        [0.00_00, 0.01_00, 0.02_00, 0.03_00, 0.04_00, 0.05_00, 0.06_00, 0.07_00],
											        [-0.20_12, 0.88_97, 0.02_63, 0.94_01, 0.20_74, 0.94_63, 0.34_81, 0.93_43],
											        [-1.70_57, 0.62_71, -1.21_45, 1.38_97, -0.63_03, 1.76_47, -0.11_73, 1.89_85],
											        [-2.17_31, -1.63_97, -2.73_58, 0.28_54, -2.18_40, 1.71_83, -1.30_18, 2.48_71],
											        [0.27_17, -3.61_73, -2.92_06, -2.19_88, -3.66_38, 0.38_58, -2.91_55, 2.29_80],
											        [3.98_59, -2.15_80, -0.79_84, -4.49_04, -4.11_81, -2.02_52, -4.47_82, 1.12_53],
											    ] )
											snake_case_           =					tf.constant(
											    [
											        [0.00_00, -0.01_00, -0.02_00, -0.03_00, -0.04_00, -0.05_00, -0.06_00, -0.07_00],
											        [0.20_12, -0.88_97, -0.02_63, -0.94_01, -0.20_74, -0.94_63, -0.34_81, -0.93_43],
											        [1.70_57, -0.62_71, 1.21_45, -1.38_97, 0.63_03, -1.76_47, 0.11_73, -1.89_85],
											        [2.17_31, 1.63_97, 2.73_58, -0.28_54, 2.18_40, -1.71_83, 1.30_18, -2.48_71],
											        [-0.27_17, 3.61_73, 2.92_06, 2.19_88, 3.66_38, -0.38_58, 2.91_55, -2.29_80],
											        [-3.98_59, 2.15_80, 0.79_84, 4.49_04, 4.11_81, 2.02_52, 4.47_82, -1.12_53],
											    ] )

											tf.debugging.assert_near(query_layer[0, 0, :6, :8]   ,     snake_case   ,     atol=self.tolerance )
											tf.debugging.assert_near(key_layer[0, 0, :6, :8]   ,     snake_case   ,     atol=self.tolerance )







import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
from ...utils import logging
from ..auto import CONFIG_MAPPING


_UpperCAmelCase	:     Dict		       = logging.get_logger(__name__)

_UpperCAmelCase	:     Dict		       = {
    """salesforce/blip2-opt-2.7b""": """https://huggingface.co/salesforce/blip2-opt-2.7b/resolve/main/config.json""",
}







class 				lowercase       ( lowercase_    ):
				__SCREAMING_SNAKE_CASE      :				Optional[int]     =    '''blip_2_vision_model'''




				def __init__(      self   ,     snake_case=1408   ,     snake_case=6144   ,     snake_case=39   ,     snake_case=16   ,     snake_case=224   ,     snake_case=14   ,     snake_case="gelu"   ,     snake_case=0.0_00_01   ,     snake_case=0.0   ,     snake_case=1e-1_0   ,     snake_case=True   ,     **snake_case   ,     ):
											super().__init__(**snake_case )

											snake_case_           =					hidden_size
											snake_case_           =					intermediate_size
											snake_case_           =					num_hidden_layers
											snake_case_           =					num_attention_heads
											snake_case_           =					patch_size
											snake_case_           =					image_size
											snake_case_           =					initializer_range
											snake_case_           =					attention_dropout
											snake_case_           =					layer_norm_eps
											snake_case_           =					hidden_act
											snake_case_           =					qkv_bias




				@classmethod
				def 					a							(      cls   ,     snake_case   ,     **snake_case ):
											cls._set_token_in_kwargs(snake_case )

											snake_case_       ,					snake_case_           =					cls.get_config_dict(snake_case   ,     **snake_case )

											# get the vision config dict if we are loading from Blip2Config
											if config_dict.get('model_type' ) == "blip-2":
																		snake_case_           =					config_dict['vision_config']

											if "model_type" in config_dict and hasattr(cls   ,     'model_type' ) and config_dict["model_type"] != cls.model_type:
																		logger.warning(
																		    F'''You are using a model of type {config_dict["model_type"]} to instantiate a model of type '''
																		    F'''{cls.model_type}. This is not supported for all configurations of models and can yield errors.''' )

											return cls.from_dict(snake_case   ,     **snake_case )







class 				lowercase       ( lowercase_    ):
				__SCREAMING_SNAKE_CASE      :				Any     =    '''blip_2_qformer'''




				def __init__(      self   ,     snake_case=3_0522   ,     snake_case=768   ,     snake_case=12   ,     snake_case=12   ,     snake_case=3072   ,     snake_case="gelu"   ,     snake_case=0.1   ,     snake_case=0.1   ,     snake_case=512   ,     snake_case=0.02   ,     snake_case=1e-1_2   ,     snake_case=0   ,     snake_case="absolute"   ,     snake_case=2   ,     snake_case=1408   ,     **snake_case   ,     ):
											super().__init__(pad_token_id=snake_case   ,     **snake_case )

											snake_case_           =					vocab_size
											snake_case_           =					hidden_size
											snake_case_           =					num_hidden_layers
											snake_case_           =					num_attention_heads
											snake_case_           =					hidden_act
											snake_case_           =					intermediate_size
											snake_case_           =					hidden_dropout_prob
											snake_case_           =					attention_probs_dropout_prob
											snake_case_           =					max_position_embeddings
											snake_case_           =					initializer_range
											snake_case_           =					layer_norm_eps
											snake_case_           =					position_embedding_type
											snake_case_           =					cross_attention_frequency
											snake_case_           =					encoder_hidden_size




				@classmethod
				def 					a							(      cls   ,     snake_case   ,     **snake_case ):
											cls._set_token_in_kwargs(snake_case )

											snake_case_       ,					snake_case_           =					cls.get_config_dict(snake_case   ,     **snake_case )

											# get the qformer config dict if we are loading from Blip2Config
											if config_dict.get('model_type' ) == "blip-2":
																		snake_case_           =					config_dict['qformer_config']

											if "model_type" in config_dict and hasattr(cls   ,     'model_type' ) and config_dict["model_type"] != cls.model_type:
																		logger.warning(
																		    F'''You are using a model of type {config_dict["model_type"]} to instantiate a model of type '''
																		    F'''{cls.model_type}. This is not supported for all configurations of models and can yield errors.''' )

											return cls.from_dict(snake_case   ,     **snake_case )







class 				lowercase       ( lowercase_    ):
				__SCREAMING_SNAKE_CASE      :				Optional[int]     =    '''blip-2'''
				__SCREAMING_SNAKE_CASE      :				Dict     =    True




				def __init__(      self   ,     snake_case=None   ,     snake_case=None   ,     snake_case=None   ,     snake_case=32   ,     **snake_case ):
											super().__init__(**snake_case )

											if vision_config is None:
																		snake_case_           =					{}
																		logger.info('vision_config is None. initializing the Blip2VisionConfig with default values.' )

											if qformer_config is None:
																		snake_case_           =					{}
																		logger.info('qformer_config is None. Initializing the Blip2QFormerConfig with default values.' )

											if text_config is None:
																		snake_case_           =					{}
																		logger.info('text_config is None. Initializing the text config with default values (`OPTConfig`).' )

											snake_case_           =					BlipaVisionConfig(**snake_case )
											snake_case_           =					BlipaQFormerConfig(**snake_case )
											snake_case_           =					text_config['model_type'] if 'model_type' in text_config else 'opt'
											snake_case_           =					CONFIG_MAPPING[text_model_type](**snake_case )

											snake_case_           =					self.text_config.tie_word_embeddings
											snake_case_           =					self.text_config.is_encoder_decoder

											snake_case_           =					num_query_tokens
											snake_case_           =					self.vision_config.hidden_size
											snake_case_           =					self.text_config.model_type in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
											snake_case_           =					1.0
											snake_case_           =					0.02




				@classmethod
				def 					a							(      cls   ,     snake_case   ,     snake_case   ,     snake_case   ,     **snake_case   ,     ):
											return cls(
											    vision_config=vision_config.to_dict()   ,     qformer_config=qformer_config.to_dict()   ,     text_config=text_config.to_dict()   ,     **snake_case   ,     )




				def 					a							(      self ):
											snake_case_           =					copy.deepcopy(self.__dict__ )
											snake_case_           =					self.vision_config.to_dict()
											snake_case_           =					self.qformer_config.to_dict()
											snake_case_           =					self.text_config.to_dict()
											snake_case_           =					self.__class__.model_type
											return output




