





from __future__ import annotations

import unittest

from transformers import CTRLConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers.models.ctrl.modeling_tf_ctrl import (
					    TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
					    TFCTRLForSequenceClassification,
					    TFCTRLLMHeadModel,
					    TFCTRLModel,
					)
class 	UpperCamelCase_						(     _lowerCamelCase    ):




					def __init__(						self ,  lowerCAmelCase_ ,  )						->							Tuple:
										_snake_case													= parent
										_snake_case													= 13
										_snake_case													= 7
										_snake_case													= True
										_snake_case													= True
										_snake_case													= True
										_snake_case													= True
										_snake_case													= True
										_snake_case													= 99
										_snake_case													= 32
										_snake_case													= 2
										_snake_case													= 4
										_snake_case													= 37
										_snake_case													= 'gelu'
										_snake_case													= 0.1
										_snake_case													= 0.1
										_snake_case													= 512
										_snake_case													= 16
										_snake_case													= 2
										_snake_case													= 0.02
										_snake_case													= 3
										_snake_case													= 4
										_snake_case													= None
										_snake_case													= self.vocab_size - 1




					def 		lowerCAmelCase	(						self     )						->							int:
										_snake_case													= ids_tensor([self.batch_size, self.seq_length] ,  self.vocab_size     )

										_snake_case													= None
										if self.use_input_mask:
															_snake_case													= random_attention_mask([self.batch_size, self.seq_length]     )

										_snake_case													= None
										if self.use_token_type_ids:
															_snake_case													= ids_tensor([self.batch_size, self.seq_length] ,  self.type_vocab_size     )

										_snake_case													= None
										if self.use_mc_token_ids:
															_snake_case													= ids_tensor([self.batch_size, self.num_choices] ,  self.seq_length     )

										_snake_case													= None
										_snake_case													= None
										_snake_case													= None
										if self.use_labels:
															_snake_case													= ids_tensor([self.batch_size] ,  self.type_sequence_label_size     )
															_snake_case													= ids_tensor([self.batch_size, self.seq_length] ,  self.num_labels     )
															_snake_case													= ids_tensor([self.batch_size] ,  self.num_choices     )

										_snake_case													= CTRLConfig(
										    vocab_size=self.vocab_size ,  n_embd=self.hidden_size ,  n_layer=self.num_hidden_layers ,  n_head=self.num_attention_heads ,  n_positions=self.max_position_embeddings ,  pad_token_id=self.pad_token_id ,  )

										_snake_case													= ids_tensor([self.num_hidden_layers, self.num_attention_heads] ,  2     )

										return (
										    config,
										    input_ids,
										    input_mask,
										    head_mask,
										    token_type_ids,
										    mc_token_ids,
										    sequence_labels,
										    token_labels,
										    choice_labels,
										)




					def 		lowerCAmelCase	(						self ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  *lowerCAmelCase_     )						->							int:
										_snake_case													= TFCTRLModel(config=_lowercase     )
										_snake_case													= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
										_snake_case													= model(_lowercase     )

										_snake_case													= [input_ids, None, input_mask]  # None is the input for 'past'
										_snake_case													= model(_lowercase     )

										_snake_case													= model(_lowercase     )

										self.parent.assertEqual(result.last_hidden_state.shape ,  (self.batch_size, self.seq_length, self.hidden_size)     )




					def 		lowerCAmelCase	(						self ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  *lowerCAmelCase_     )						->							str:
										_snake_case													= TFCTRLLMHeadModel(config=_lowercase     )
										_snake_case													= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
										_snake_case													= model(_lowercase     )
										self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.vocab_size)     )




					def 		lowerCAmelCase	(						self ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  *lowerCAmelCase_     )						->							Dict:
										_snake_case													= self.num_labels
										_snake_case													= ids_tensor([self.batch_size] ,  self.type_sequence_label_size     )
										_snake_case													= {
										    'input_ids': input_ids,
										    'token_type_ids': token_type_ids,
										    'labels': sequence_labels,
										}
										_snake_case													= TFCTRLForSequenceClassification(_lowercase     )
										_snake_case													= model(_lowercase     )
										self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.num_labels)     )






					def 		lowerCAmelCase	(						self     )						->							Dict:
										_snake_case													= self.prepare_config_and_inputs()

										(
										    (
										    _snake_case    
										)							,   (
										    _snake_case    
										)							,   (
										    _snake_case    
										)							,   (
										    _snake_case    
										)							,   (
										    _snake_case    
										)							,   (
										    _snake_case    
										)							,   (
										    _snake_case    
										)							,   (
										    _snake_case    
										)							,   (
										    _snake_case    
										)							,       
										)													= config_and_inputs

										_snake_case													= {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
										return config, inputs_dict





@require_tf
class 	UpperCamelCase_						(     _lowerCamelCase							,     _lowerCamelCase							,     unittest.TestCase    ):
					lowerCAmelCase_							=		(TFCTRLModel, TFCTRLLMHeadModel, TFCTRLForSequenceClassification) if is_tf_available() else ()
					lowerCAmelCase_							=		(TFCTRLLMHeadModel,) if is_tf_available() else ()
					lowerCAmelCase_							=		(
					    {
					        """feature-extraction""": TFCTRLModel,
					        """text-classification""": TFCTRLForSequenceClassification,
					        """text-generation""": TFCTRLLMHeadModel,
					        """zero-shot""": TFCTRLForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)
					lowerCAmelCase_							=		False
					lowerCAmelCase_							=		False




					def 		lowerCAmelCase	(						self ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_ ,  lowerCAmelCase_     )						->							int:
										if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
															# Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
															# `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny
															# config could not be created.
															return True

										return False




					def 		lowerCAmelCase	(						self     )						->							List[Any]:
										_snake_case													= TFCTRLModelTester(self     )
										_snake_case													= ConfigTester(self ,  config_class=_lowercase ,  n_embd=37     )




					def 		lowerCAmelCase	(						self     )						->							Tuple:
										self.config_tester.run_common_tests()




					def 		lowerCAmelCase	(						self     )						->							Optional[Any]:
										_snake_case													= self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_ctrl_model(*_lowercase     )




					def 		lowerCAmelCase	(						self     )						->							List[Any]:
										_snake_case													= self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_ctrl_lm_head(*_lowercase     )




					def 		lowerCAmelCase	(						self     )						->							Any:
										_snake_case													= self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_ctrl_for_sequence_classification(*_lowercase     )




					def 		lowerCAmelCase	(						self     )						->							List[Any]:
										_snake_case							,   _snake_case													= self.model_tester.prepare_config_and_inputs_for_common()
										_snake_case													= [TFCTRLLMHeadModel]
										_snake_case													= [TFCTRLForSequenceClassification]

										for model_class in self.all_model_classes:
															_snake_case													= model_class(_lowercase     )
															model.build()  # may be needed for the get_bias() call below
															assert isinstance(model.get_input_embeddings() ,  tf.keras.layers.Layer     )

															if model_class in list_lm_models:
																				_snake_case													= model.get_output_embeddings()
																				assert isinstance(_lowercase ,  tf.keras.layers.Layer     )
																				_snake_case													= model.get_bias()
																				assert isinstance(_lowercase ,  _lowercase     )
																				for k, v in name.items():
																									assert isinstance(_lowercase ,  tf.Variable     )
															elif model_class in list_other_models_with_output_ebd:
																				_snake_case													= model.get_output_embeddings()
																				assert isinstance(_lowercase ,  tf.keras.layers.Layer     )
																				_snake_case													= model.get_bias()
																				assert name is None
															else:
																				_snake_case													= model.get_output_embeddings()
																				assert x is None
																				_snake_case													= model.get_bias()
																				assert name is None






					@slow
					def 		lowerCAmelCase	(						self     )						->							Any:
										for model_name in TF_CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															_snake_case													= TFCTRLModel.from_pretrained(_lowercase     )
															self.assertIsNotNone(_lowercase     )





@require_tf
class 	UpperCamelCase_						(     unittest.TestCase    ):




					@slow
					def 		lowerCAmelCase	(						self     )						->							str:
										_snake_case													= TFCTRLLMHeadModel.from_pretrained('ctrl'     )
										_snake_case													= tf.convert_to_tensor([[1_1859, 0, 1611, 8]] ,  dtype=tf.intaa     )  # Legal the president is
										_snake_case													= [
										    1_1859,
										    0,
										    1611,
										    8,
										    5,
										    150,
										    2_6449,
										    2,
										    19,
										    348,
										    469,
										    3,
										    2595,
										    48,
										    2_0740,
										    24_6533,
										    24_6533,
										    19,
										    30,
										    5,
										]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a

										_snake_case													= model.generate(_lowercase ,  do_sample=_lowercase     )
										self.assertListEqual(output_ids[0].numpy().tolist() ,  _lowercase     )






import argparse

import torch
from clip import load

from transformers import CLIPConfig, CLIPModel




def        lowerCAmelCase(	SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       )->		List[Any]:



    """simple docstring"""






    UpperCamelCase_   ,    UpperCamelCase_   ,    UpperCamelCase_          =		pt_attn_layer.in_proj_weight.chunk(3       ,		dim=0       )
    UpperCamelCase_   ,    UpperCamelCase_   ,    UpperCamelCase_          =		pt_attn_layer.in_proj_bias.chunk(3       ,		dim=0       )

    UpperCamelCase_          =		pt_attn_layer.out_proj.weight
    UpperCamelCase_          =		pt_attn_layer.out_proj.bias

    UpperCamelCase_          =		q_proj
    UpperCamelCase_          =		q_proj_bias

    UpperCamelCase_          =		k_proj
    UpperCamelCase_          =		k_proj_bias

    UpperCamelCase_          =		v_proj
    UpperCamelCase_          =		v_proj_bias

    UpperCamelCase_          =		out_proj_weights
    UpperCamelCase_          =		out_proj_bias




def        lowerCAmelCase(	SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       )->		Union[str, Any]:



    """simple docstring"""






    copy_linear(hf_mlp.fca       ,		pt_mlp.c_fc       )
    copy_linear(hf_mlp.fca       ,		pt_mlp.c_proj       )




def        lowerCAmelCase(	SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       )->		List[Any]:



    """simple docstring"""






    UpperCamelCase_          =		pt_linear.weight
    UpperCamelCase_          =		pt_linear.bias




def        lowerCAmelCase(	SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       )->		List[Any]:



    """simple docstring"""






    copy_linear(hf_layer.layer_norma       ,		pt_layer.ln_a       )
    copy_linear(hf_layer.layer_norma       ,		pt_layer.ln_a       )

    # copy MLP
    copy_mlp(hf_layer.mlp       ,		pt_layer.mlp       )

    # copy attn
    copy_attn_layer(hf_layer.self_attn       ,		pt_layer.attn       )




def        lowerCAmelCase(	SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       )->		List[Any]:



    """simple docstring"""






    for hf_layer, pt_layer in zip(SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       ):
        copy_layer(SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       )




def        lowerCAmelCase(	SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       )->		Optional[Any]:



    """simple docstring"""






    UpperCamelCase_          =		pt_model.token_embedding.weight
    UpperCamelCase_          =		pt_model.positional_embedding

    # copy layer norm
    copy_linear(hf_encoder.final_layer_norm       ,		pt_model.ln_final       )

    # copy hidden layers
    copy_layers(hf_encoder.encoder.layers       ,		pt_model.transformer.resblocks       )




def        lowerCAmelCase(	SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       )->		Any:



    """simple docstring"""






    UpperCamelCase_          =		pt_model.text_projection.data.T

    # copy text encoder
    copy_encoder(hf_model.text_model       ,		SCREAMING_SNAKE_CASE_       )




def        lowerCAmelCase(	SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       )->		Any:



    """simple docstring"""






    UpperCamelCase_          =		pt_model.visual.proj.data.T

    # copy layer norms
    copy_linear(hf_model.vision_model.pre_layrnorm       ,		pt_model.visual.ln_pre       )
    copy_linear(hf_model.vision_model.post_layernorm       ,		pt_model.visual.ln_post       )

    # copy embeds
    UpperCamelCase_          =		pt_model.visual.conva.weight.data
    UpperCamelCase_          =		pt_model.visual.class_embedding
    UpperCamelCase_          =		pt_model.visual.positional_embedding.data

    # copy encoder
    copy_layers(hf_model.vision_model.encoder.layers       ,		pt_model.visual.transformer.resblocks       )




@torch.no_grad()
def        lowerCAmelCase(	SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_=None       )->		List[Any]:



    """simple docstring"""






    if config_path is not None:
        UpperCamelCase_          =		CLIPConfig.from_pretrained(SCREAMING_SNAKE_CASE_       )
    else:
        UpperCamelCase_          =		CLIPConfig(projection_dim=5_1_2       ,		text_config={}       ,		vision_config={}       )

    UpperCamelCase_          =		CLIPModel(SCREAMING_SNAKE_CASE_       ).eval()

    UpperCamelCase_   ,    UpperCamelCase_          =		load(SCREAMING_SNAKE_CASE_       ,		device="cpu"       ,		jit=SCREAMING_SNAKE_CASE_       )
    UpperCamelCase_          =		pt_model.eval()

    copy_text_model_and_projection(SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       )
    copy_vison_model_and_projection(SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       )
    UpperCamelCase_          =		pt_model.logit_scale

    UpperCamelCase_          =		torch.arange(0       ,		7_7       ).unsqueeze(0       )
    UpperCamelCase_          =		torch.randn(1       ,		3       ,		2_2_4       ,		2_2_4       )

    UpperCamelCase_          =		hf_model(input_ids=SCREAMING_SNAKE_CASE_       ,		pixel_values=SCREAMING_SNAKE_CASE_       ,		return_dict=SCREAMING_SNAKE_CASE_       )
    UpperCamelCase_          =		hf_outputs.logits_per_image
    UpperCamelCase_          =		hf_outputs.logits_per_text
    UpperCamelCase_   ,    UpperCamelCase_          =		pt_model(SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       )

    assert torch.allclose(SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       ,		atol=1E-3       )
    assert torch.allclose(SCREAMING_SNAKE_CASE_       ,		SCREAMING_SNAKE_CASE_       ,		atol=1E-3       )

    hf_model.save_pretrained(SCREAMING_SNAKE_CASE_       )


if __name__ == "__main__":
 SCREAMING_SNAKE_CASE						:Any      =			argparse.ArgumentParser()
 parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
 parser.add_argument("""--checkpoint_path""", default=None, type=str, help="""Path to fairseq checkpoint""")
 parser.add_argument("""--config_path""", default=None, type=str, help="""Path to hf config.json of model to convert""")
 SCREAMING_SNAKE_CASE						:Dict      =			parser.parse_args()

 convert_clip_checkpoint(args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path)
