



'''simple docstring'''





import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        RemBertConfig,
        RemBertForCausalLM,
        RemBertForMaskedLM,
        RemBertForMultipleChoice,
        RemBertForQuestionAnswering,
        RemBertForSequenceClassification,
        RemBertForTokenClassification,
        RemBertModel,
    )
    from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST



class       snake_case__		:
 def __init__(					self		:				List[Any]	,			__a		:				List[Any]	,			__a		:				str=13	,			__a		:				List[Any]=7	,			__a		:				List[Any]=True	,			__a		:				Dict=True	,			__a		:				str=True	,			__a		:				List[str]=True	,			__a		:				Dict=99	,			__a		:				int=32	,			__a		:				Optional[int]=18	,			__a		:				Union[str, Any]=43	,			__a		:				str=5	,			__a		:				Optional[Any]=4	,			__a		:				int=37	,			__a		:				Tuple="gelu"	,			__a		:				List[str]=0.1	,			__a		:				Tuple=0.1	,			__a		:				int=512	,			__a		:				List[str]=16	,			__a		:				List[Any]=2	,			__a		:				Optional[Any]=0.0_2	,			__a		:				Optional[Any]=3	,			__a		:				Tuple=4	,			__a		:				str=None	,			)	->					Optional[Any]:







       '''simple docstring'''
       __snake_case      :					Optional[Any]						=  parent
       __snake_case      :					int						=  batch_size
       __snake_case      :					Any						=  seq_length
       __snake_case      :					Dict						=  is_training
       __snake_case      :					Any						=  use_input_mask
       __snake_case      :					List[str]						=  use_token_type_ids
       __snake_case      :					Tuple						=  use_labels
       __snake_case      :					str						=  vocab_size
       __snake_case      :					Optional[int]						=  hidden_size
       __snake_case      :					Optional[int]						=  input_embedding_size
       __snake_case      :					Optional[int]						=  output_embedding_size
       __snake_case      :					Any						=  num_hidden_layers
       __snake_case      :					List[str]						=  num_attention_heads
       __snake_case      :					Tuple						=  intermediate_size
       __snake_case      :					Union[str, Any]						=  hidden_act
       __snake_case      :					int						=  hidden_dropout_prob
       __snake_case      :					Optional[Any]						=  attention_probs_dropout_prob
       __snake_case      :					Optional[int]						=  max_position_embeddings
       __snake_case      :					Union[str, Any]						=  type_vocab_size
       __snake_case      :					List[Any]						=  type_sequence_label_size
       __snake_case      :					int						=  initializer_range
       __snake_case      :					Union[str, Any]						=  num_labels
       __snake_case      :					List[Any]						=  num_choices
       __snake_case      :					List[str]						=  scope
 def A_					(					self		:				Optional[Any]   )	->					List[str]:







       '''simple docstring'''
       __snake_case      :					str						=  ids_tensor([self.batch_size, self.seq_length]	,			self.vocab_size   )

       __snake_case      :					Any						=  None
       if self.use_input_mask:
             __snake_case      :					List[str]						=  random_attention_mask([self.batch_size, self.seq_length]   )

       __snake_case      :					List[Any]						=  None
       if self.use_token_type_ids:
             __snake_case      :					int						=  ids_tensor([self.batch_size, self.seq_length]	,			self.type_vocab_size   )

       __snake_case      :					List[str]						=  None
       __snake_case      :					Optional[int]						=  None
       __snake_case      :					Any						=  None
       if self.use_labels:
             __snake_case      :					Dict						=  ids_tensor([self.batch_size]	,			self.type_sequence_label_size   )
             __snake_case      :					Union[str, Any]						=  ids_tensor([self.batch_size, self.seq_length]	,			self.num_labels   )
             __snake_case      :					List[str]						=  ids_tensor([self.batch_size]	,			self.num_choices   )

       __snake_case      :					Tuple						=  RemBertConfig(
           vocab_size=self.vocab_size	,			hidden_size=self.hidden_size	,			input_embedding_size=self.input_embedding_size	,			output_embedding_size=self.output_embedding_size	,			num_hidden_layers=self.num_hidden_layers	,			num_attention_heads=self.num_attention_heads	,			intermediate_size=self.intermediate_size	,			hidden_act=self.hidden_act	,			hidden_dropout_prob=self.hidden_dropout_prob	,			attention_probs_dropout_prob=self.attention_probs_dropout_prob	,			max_position_embeddings=self.max_position_embeddings	,			type_vocab_size=self.type_vocab_size	,			is_decoder=_lowerCamelCase	,			initializer_range=self.initializer_range	,			)

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels
 def A_					(					self		:				Any   )	->					int:







       '''simple docstring'''
       (
           __snake_case      
       )      :					Optional[Any]						=  self.prepare_config_and_inputs()

       __snake_case      :					int						=  True
       __snake_case      :					List[str]						=  floats_tensor([self.batch_size, self.seq_length, self.hidden_size]   )
       __snake_case      :					str						=  ids_tensor([self.batch_size, self.seq_length]	,			vocab_size=2   )

       return (
           config,
           input_ids,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
           choice_labels,
           encoder_hidden_states,
           encoder_attention_mask,
       )
 def A_					(					self		:				List[Any]	,			__a		:				Optional[Any]	,			__a		:				List[str]	,			__a		:				Tuple	,			__a		:				Optional[Any]	,			__a		:				Optional[int]	,			__a		:				List[Any]	,			__a		:				Any   )	->					int:







       '''simple docstring'''
       __snake_case      :					Tuple						=  RemBertModel(config=_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()
       __snake_case      :					List[Any]						=  model(_lowerCamelCase	,			attention_mask=_lowerCamelCase	,			token_type_ids=_lowerCamelCase   )
       __snake_case      :					Tuple						=  model(_lowerCamelCase	,			token_type_ids=_lowerCamelCase   )
       __snake_case      :					str						=  model(_lowerCamelCase   )
       self.parent.assertEqual(result.last_hidden_state.shape	,			(self.batch_size, self.seq_length, self.hidden_size)   )
 def A_					(					self		:				Dict	,			__a		:				Dict	,			__a		:				Union[str, Any]	,			__a		:				Optional[Any]	,			__a		:				Tuple	,			__a		:				int	,			__a		:				Optional[Any]	,			__a		:				Optional[int]	,			__a		:				str	,			__a		:				Any	,			)	->					Dict:







       '''simple docstring'''
       __snake_case      :					int						=  True
       __snake_case      :					Optional[int]						=  RemBertModel(_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()
       __snake_case      :					Union[str, Any]						=  model(
           _lowerCamelCase	,			attention_mask=_lowerCamelCase	,			token_type_ids=_lowerCamelCase	,			encoder_hidden_states=_lowerCamelCase	,			encoder_attention_mask=_lowerCamelCase	,			)
       __snake_case      :					Optional[int]						=  model(
           _lowerCamelCase	,			attention_mask=_lowerCamelCase	,			token_type_ids=_lowerCamelCase	,			encoder_hidden_states=_lowerCamelCase	,			)
       __snake_case      :					int						=  model(_lowerCamelCase	,			attention_mask=_lowerCamelCase	,			token_type_ids=_lowerCamelCase   )
       self.parent.assertEqual(result.last_hidden_state.shape	,			(self.batch_size, self.seq_length, self.hidden_size)   )
 def A_					(					self		:				Optional[int]	,			__a		:				Union[str, Any]	,			__a		:				Union[str, Any]	,			__a		:				Tuple	,			__a		:				Dict	,			__a		:				List[Any]	,			__a		:				str	,			__a		:				List[str]	,			__a		:				List[str]	,			__a		:				Optional[Any]	,			)	->					Optional[int]:







       '''simple docstring'''
       __snake_case      :					Tuple						=  RemBertForCausalLM(config=_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()
       __snake_case      :					Union[str, Any]						=  model(_lowerCamelCase	,			attention_mask=_lowerCamelCase	,			token_type_ids=_lowerCamelCase	,			labels=_lowerCamelCase   )
       self.parent.assertEqual(result.logits.shape	,			(self.batch_size, self.seq_length, self.vocab_size)   )
 def A_					(					self		:				int	,			__a		:				Union[str, Any]	,			__a		:				Tuple	,			__a		:				List[str]	,			__a		:				Union[str, Any]	,			__a		:				Dict	,			__a		:				List[Any]	,			__a		:				Union[str, Any]   )	->					Any:







       '''simple docstring'''
       __snake_case      :					List[Any]						=  RemBertForMaskedLM(config=_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()
       __snake_case      :					int						=  model(_lowerCamelCase	,			attention_mask=_lowerCamelCase	,			token_type_ids=_lowerCamelCase	,			labels=_lowerCamelCase   )
       self.parent.assertEqual(result.logits.shape	,			(self.batch_size, self.seq_length, self.vocab_size)   )
 def A_					(					self		:				Union[str, Any]	,			__a		:				List[str]	,			__a		:				Dict	,			__a		:				Optional[int]	,			__a		:				List[Any]	,			__a		:				Optional[Any]	,			__a		:				Tuple	,			__a		:				int	,			__a		:				List[Any]	,			__a		:				Tuple	,			)	->					int:







       '''simple docstring'''
       __snake_case      :					Union[str, Any]						=  True
       __snake_case      :					Optional[int]						=  True
       __snake_case      :					List[Any]						=  RemBertForCausalLM(config=_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()

       # first forward pass
       __snake_case      :					Union[str, Any]						=  model(
           _lowerCamelCase	,			attention_mask=_lowerCamelCase	,			encoder_hidden_states=_lowerCamelCase	,			encoder_attention_mask=_lowerCamelCase	,			use_cache=_lowerCamelCase	,			)
       __snake_case      :					int						=  outputs.past_key_values

       # create hypothetical multiple next token and extent to next_input_ids
       __snake_case      :					List[Any]						=  ids_tensor((self.batch_size, 3)	,			config.vocab_size   )
       __snake_case      :					List[Any]						=  ids_tensor((self.batch_size, 3)	,			vocab_size=2   )

       # append to next input_ids and
       __snake_case      :					Union[str, Any]						=  torch.cat([input_ids, next_tokens]	,			dim=-1   )
       __snake_case      :					str						=  torch.cat([input_mask, next_mask]	,			dim=-1   )

       __snake_case      :					List[str]						=  model(
           _lowerCamelCase	,			attention_mask=_lowerCamelCase	,			encoder_hidden_states=_lowerCamelCase	,			encoder_attention_mask=_lowerCamelCase	,			output_hidden_states=_lowerCamelCase	,			)['''hidden_states'''][0]
       __snake_case      :					str						=  model(
           _lowerCamelCase	,			attention_mask=_lowerCamelCase	,			encoder_hidden_states=_lowerCamelCase	,			encoder_attention_mask=_lowerCamelCase	,			past_key_values=_lowerCamelCase	,			output_hidden_states=_lowerCamelCase	,			)['''hidden_states'''][0]

       # select random slice
       __snake_case      :					Any						=  ids_tensor((1,)	,			output_from_past.shape[-1]   ).item()
       __snake_case      :					str						=  output_from_no_past[:, -3:, random_slice_idx].detach()
       __snake_case      :					Optional[Any]						=  output_from_past[:, :, random_slice_idx].detach()

       self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]   )

       # test that outputs are equal for slice
       self.parent.assertTrue(torch.allclose(_lowerCamelCase	,			_lowerCamelCase	,			atol=1e-3   )   )
 def A_					(					self		:				Tuple	,			__a		:				int	,			__a		:				Tuple	,			__a		:				int	,			__a		:				Union[str, Any]	,			__a		:				Tuple	,			__a		:				Optional[Any]	,			__a		:				int   )	->					List[str]:







       '''simple docstring'''
       __snake_case      :					Optional[int]						=  RemBertForQuestionAnswering(config=_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()
       __snake_case      :					int						=  model(
           _lowerCamelCase	,			attention_mask=_lowerCamelCase	,			token_type_ids=_lowerCamelCase	,			start_positions=_lowerCamelCase	,			end_positions=_lowerCamelCase	,			)
       self.parent.assertEqual(result.start_logits.shape	,			(self.batch_size, self.seq_length)   )
       self.parent.assertEqual(result.end_logits.shape	,			(self.batch_size, self.seq_length)   )
 def A_					(					self		:				int	,			__a		:				Union[str, Any]	,			__a		:				Any	,			__a		:				int	,			__a		:				List[Any]	,			__a		:				List[str]	,			__a		:				Dict	,			__a		:				Dict   )	->					Optional[Any]:







       '''simple docstring'''
       __snake_case      :					Tuple						=  self.num_labels
       __snake_case      :					Optional[int]						=  RemBertForSequenceClassification(_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()
       __snake_case      :					str						=  model(_lowerCamelCase	,			attention_mask=_lowerCamelCase	,			token_type_ids=_lowerCamelCase	,			labels=_lowerCamelCase   )
       self.parent.assertEqual(result.logits.shape	,			(self.batch_size, self.num_labels)   )
 def A_					(					self		:				str	,			__a		:				Optional[Any]	,			__a		:				int	,			__a		:				List[str]	,			__a		:				Union[str, Any]	,			__a		:				Tuple	,			__a		:				str	,			__a		:				List[Any]   )	->					Dict:







       '''simple docstring'''
       __snake_case      :					int						=  self.num_labels
       __snake_case      :					Optional[int]						=  RemBertForTokenClassification(config=_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()
       __snake_case      :					Any						=  model(_lowerCamelCase	,			attention_mask=_lowerCamelCase	,			token_type_ids=_lowerCamelCase	,			labels=_lowerCamelCase   )
       self.parent.assertEqual(result.logits.shape	,			(self.batch_size, self.seq_length, self.num_labels)   )
 def A_					(					self		:				int	,			__a		:				str	,			__a		:				Union[str, Any]	,			__a		:				str	,			__a		:				str	,			__a		:				int	,			__a		:				str	,			__a		:				List[str]   )	->					Tuple:







       '''simple docstring'''
       __snake_case      :					List[str]						=  self.num_choices
       __snake_case      :					Tuple						=  RemBertForMultipleChoice(config=_lowerCamelCase   )
       model.to(_lowerCamelCase   )
       model.eval()
       __snake_case      :					List[Any]						=  input_ids.unsqueeze(1   ).expand(-1	,			self.num_choices	,			-1   ).contiguous()
       __snake_case      :					Optional[int]						=  token_type_ids.unsqueeze(1   ).expand(-1	,			self.num_choices	,			-1   ).contiguous()
       __snake_case      :					Optional[Any]						=  input_mask.unsqueeze(1   ).expand(-1	,			self.num_choices	,			-1   ).contiguous()
       __snake_case      :					Optional[int]						=  model(
           _lowerCamelCase	,			attention_mask=_lowerCamelCase	,			token_type_ids=_lowerCamelCase	,			labels=_lowerCamelCase	,			)
       self.parent.assertEqual(result.logits.shape	,			(self.batch_size, self.num_choices)   )





 def A_					(					self		:				Union[str, Any]   )	->					Optional[int]:







       '''simple docstring'''
       __snake_case      :					str						=  self.prepare_config_and_inputs()
       (
           __snake_case      
       )      :					List[str]						=  config_and_inputs
       __snake_case      :					List[Any]						=  {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
       return config, inputs_dict



@require_torch
class       snake_case__		(  __UpperCAmelCase   ,			__UpperCAmelCase   ,			unittest.TestCase							):
 A__							=      (
     (
         RemBertModel,
         RemBertForMaskedLM,
         RemBertForCausalLM,
         RemBertForMultipleChoice,
         RemBertForQuestionAnswering,
         RemBertForSequenceClassification,
         RemBertForTokenClassification,
     )
     if is_torch_available()
     else ()
 )
 A__							=      (RemBertForCausalLM,) if is_torch_available() else ()
 A__							=      (
     {
         '''feature-extraction''': RemBertModel,
         '''fill-mask''': RemBertForMaskedLM,
         '''question-answering''': RemBertForQuestionAnswering,
         '''text-classification''': RemBertForSequenceClassification,
         '''text-generation''': RemBertForCausalLM,
         '''token-classification''': RemBertForTokenClassification,
         '''zero-shot''': RemBertForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 def A_					(					self		:				Any   )	->					Optional[Any]:







       '''simple docstring'''
       __snake_case      :					Union[str, Any]						=  RemBertModelTester(self   )
       __snake_case      :					Any						=  ConfigTester(self	,			config_class=_lowerCamelCase	,			hidden_size=37   )
 def A_					(					self		:				str   )	->					List[str]:







       '''simple docstring'''
       self.config_tester.run_common_tests()
 def A_					(					self		:				Tuple   )	->					Dict:







       '''simple docstring'''
       __snake_case      :					Any						=  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*_lowerCamelCase   )
 def A_					(					self		:				List[Any]   )	->					List[str]:







       '''simple docstring'''
       __snake_case      :					Optional[int]						=  self.model_tester.prepare_config_and_inputs()
       for type in ["absolute", "relative_key", "relative_key_query"]:
             __snake_case      :					List[Any]						=  type
             self.model_tester.create_and_check_model(*_lowerCamelCase   )
 def A_					(					self		:				Tuple   )	->					int:







       '''simple docstring'''
       __snake_case      :					Union[str, Any]						=  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*_lowerCamelCase   )
 def A_					(					self		:				Optional[int]   )	->					Union[str, Any]:







       '''simple docstring'''
       __snake_case      :					Tuple						=  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_multiple_choice(*_lowerCamelCase   )
 def A_					(					self		:				List[str]   )	->					List[str]:







       '''simple docstring'''
       __snake_case      :					Union[str, Any]						=  self.model_tester.prepare_config_and_inputs_for_decoder()
       self.model_tester.create_and_check_decoder_model_past_large_inputs(*_lowerCamelCase   )
 def A_					(					self		:				Any   )	->					Optional[Any]:







       '''simple docstring'''
       __snake_case      :					List[str]						=  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*_lowerCamelCase   )
 def A_					(					self		:				List[str]   )	->					Union[str, Any]:







       '''simple docstring'''
       __snake_case      :					Optional[Any]						=  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*_lowerCamelCase   )
 def A_					(					self		:				Union[str, Any]   )	->					str:







       '''simple docstring'''
       __snake_case      :					List[str]						=  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*_lowerCamelCase   )
 def A_					(					self		:				str   )	->					Tuple:







       '''simple docstring'''
       __snake_case      :					List[str]						=  self.model_tester.prepare_config_and_inputs_for_decoder()
       self.model_tester.create_and_check_model_as_decoder(*_lowerCamelCase   )
 def A_					(					self		:				Optional[int]   )	->					Any:







       '''simple docstring'''
       # This regression test was failing with PyTorch < 1.3
       (
           __snake_case      
       )      :					List[str]						=  self.model_tester.prepare_config_and_inputs_for_decoder()

       __snake_case      :					List[Any]						=  None

       self.model_tester.create_and_check_model_as_decoder(
           _lowerCamelCase	,			_lowerCamelCase	,			_lowerCamelCase	,			_lowerCamelCase	,			_lowerCamelCase	,			_lowerCamelCase	,			_lowerCamelCase	,			_lowerCamelCase	,			_lowerCamelCase	,			)





 @slow
 def A_					(					self		:				int   )	->					List[str]:







       '''simple docstring'''
       for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             __snake_case      :					str						=  RemBertModel.from_pretrained(_lowerCamelCase   )
             self.assertIsNotNone(_lowerCamelCase   )



@require_torch
class       snake_case__		(  unittest.TestCase							):
 @slow
 def A_					(					self		:				int   )	->					List[str]:







       '''simple docstring'''
       # Test exact values at the last hidden layer
       __snake_case      :					Optional[Any]						=  RemBertModel.from_pretrained('google/rembert'   )
       __snake_case      :					str						=  torch.tensor([[312, 56498, 313, 2125, 313]]   )
       __snake_case      :					List[Any]						=  torch.tensor([[0, 0, 0, 1, 1]]   )
       with torch.no_grad():
             __snake_case      :					List[str]						=  model(_lowerCamelCase	,			token_type_ids=_lowerCamelCase	,			output_hidden_states=_lowerCamelCase   )

       __snake_case      :					int						=  1152

       __snake_case      :					int						=  torch.Size((1, 5, hidden_size)   )
       self.assertEqual(output['last_hidden_state'].shape	,			_lowerCamelCase   )

       __snake_case      :					Tuple						=  torch.tensor(
           [
               [
                   [0.0_7_5_4, -0.2_0_2_2, 0.1_9_0_4],
                   [-0.3_3_5_4, -0.3_6_9_2, -0.4_7_9_1],
                   [-0.2_3_1_4, -0.6_7_2_9, -0.0_7_4_9],
                   [-0.0_3_9_6, -0.3_1_0_5, -0.4_2_3_4],
                   [-0.1_5_7_1, -0.0_5_2_5, 0.5_3_5_3],
               ]
           ]   )

       # Running on the original tf implementation gives slightly different results here.
       # Not clear why this variations is present
       # TODO: Find reason for discrepancy
       # expected_original_implementation = [[
       #     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
       #     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
       #     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
       #     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
       #     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
       # ]]

       self.assertTrue(torch.allclose(output['last_hidden_state'][:, :, :3]	,			_lowerCamelCase	,			atol=1e-4   )   )










"""simple docstring"""





import argparse
import collections

import jax
import jax.numpy as jnp
import torch
import torch.nn as nn
from clip.model import CLIP
from flax.training import checkpoints
from huggingface_hub import Repository

from transformers import (
    CLIPTokenizer,
    OwlViTConfig,
    OwlViTForObjectDetection,
    OwlViTImageProcessor,
    OwlViTModel,
    OwlViTProcessor,
)


_lowerCamelCase       =	{
    '''vit_b32''': {
        '''embed_dim''': 5_12,
        '''image_resolution''': 7_68,
        '''context_length''': 16,
        '''vocab_size''': 4_94_08,
        '''vision_layers''': 12,
        '''vision_width''': 7_68,
        '''vision_patch_size''': 32,
        '''transformer_width''': 5_12,
        '''transformer_heads''': 8,
        '''transformer_layers''': 12,
    },
    '''vit_b16''': {
        '''embed_dim''': 5_12,
        '''image_resolution''': 7_68,
        '''context_length''': 16,
        '''vocab_size''': 4_94_08,
        '''vision_layers''': 12,
        '''vision_width''': 7_68,
        '''vision_patch_size''': 16,
        '''transformer_width''': 5_12,
        '''transformer_heads''': 8,
        '''transformer_layers''': 12,
    },
    '''vit_l14''': {
        '''embed_dim''': 7_68,
        '''image_resolution''': 8_40,
        '''context_length''': 16,
        '''vocab_size''': 4_94_08,
        '''vision_layers''': 24,
        '''vision_width''': 10_24,
        '''vision_patch_size''': 14,
        '''transformer_width''': 7_68,
        '''transformer_heads''': 12,
        '''transformer_layers''': 12,
    },
}




def 		lowerCAmelCase_	(  lowercase_   :							Tuple ,      lowercase_   :							List[Any]="" ,      lowercase_   :							List[Any]="/"		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			Optional[int]												=       []

 for k, v in params.items():
  __SCREAMING_SNAKE_CASE :			List[str]												=       parent_key + sep + k if parent_key else k

  if isinstance(lowercase_ ,      collections.MutableMapping		):
   items.extend(flatten_nested_dict(lowercase_ ,      lowercase_ ,      sep=lowercase_		).items()		)
  else:
   items.append((new_key, v)		)
 return dict(lowercase_		)




def 		lowerCAmelCase_	(  lowercase_   :							Any		):

 '''simple docstring'''






 return jax.tree_util.tree_map(lambda lowercase_		: x.astype(jnp.floataa		) if x.dtype == jnp.bfloataa else x ,      lowercase_		)




def 		lowerCAmelCase_	(  lowercase_   :							List[Any] ,      lowercase_   :							Union[str, Any]		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE							,      __SCREAMING_SNAKE_CASE							,      __SCREAMING_SNAKE_CASE :			Optional[Any]												=       pt_attn_layer.in_proj_weight.chunk(3 ,      dim=0		)
 __SCREAMING_SNAKE_CASE							,      __SCREAMING_SNAKE_CASE							,      __SCREAMING_SNAKE_CASE :			Dict												=       pt_attn_layer.in_proj_bias.chunk(3 ,      dim=0		)

 __SCREAMING_SNAKE_CASE :			Any												=       pt_attn_layer.out_proj.weight
 __SCREAMING_SNAKE_CASE :			str												=       pt_attn_layer.out_proj.bias

 __SCREAMING_SNAKE_CASE :			Any												=       q_proj
 __SCREAMING_SNAKE_CASE :			List[str]												=       q_proj_bias

 __SCREAMING_SNAKE_CASE :			Any												=       k_proj
 __SCREAMING_SNAKE_CASE :			Optional[int]												=       k_proj_bias

 __SCREAMING_SNAKE_CASE :			List[str]												=       v_proj
 __SCREAMING_SNAKE_CASE :			List[str]												=       v_proj_bias

 __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       out_proj_weights
 __SCREAMING_SNAKE_CASE :			int												=       out_proj_bias




def 		lowerCAmelCase_	(  lowercase_   :							int ,      lowercase_   :							Dict		):

 '''simple docstring'''






 copy_linear(hf_mlp.fca ,      pt_mlp.c_fc		)
 copy_linear(hf_mlp.fca ,      pt_mlp.c_proj		)




def 		lowerCAmelCase_	(  lowercase_   :							Optional[int] ,      lowercase_   :							Tuple		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			Optional[Any]												=       pt_linear.weight
 __SCREAMING_SNAKE_CASE :			List[Any]												=       pt_linear.bias




def 		lowerCAmelCase_	(  lowercase_   :							Any ,      lowercase_   :							Any		):

 '''simple docstring'''






 copy_linear(hf_layer.layer_norma ,      pt_layer.ln_a		)
 copy_linear(hf_layer.layer_norma ,      pt_layer.ln_a		)

 # copy MLP
 copy_mlp(hf_layer.mlp ,      pt_layer.mlp		)

 # copy attn
 copy_attn_layer(hf_layer.self_attn ,      pt_layer.attn		)




def 		lowerCAmelCase_	(  lowercase_   :							List[Any] ,      lowercase_   :							Any		):

 '''simple docstring'''






 for hf_layer, pt_layer in zip(lowercase_ ,      lowercase_		):
  copy_layer(lowercase_ ,      lowercase_		)




def 		lowerCAmelCase_	(  lowercase_   :							List[str] ,      lowercase_   :							Tuple		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			Optional[int]												=       pt_model.token_embedding.weight
 __SCREAMING_SNAKE_CASE :			str												=       pt_model.positional_embedding

 # copy layer norm
 copy_linear(hf_encoder.final_layer_norm ,      pt_model.ln_final		)

 # copy hidden layers
 copy_layers(hf_encoder.encoder.layers ,      pt_model.transformer.resblocks		)




def 		lowerCAmelCase_	(  lowercase_   :							Optional[int] ,      lowercase_   :							int		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			Optional[int]												=       pt_model.text_projection.data.T

 # copy text encoder
 copy_encoder(hf_model.text_model ,      lowercase_		)




def 		lowerCAmelCase_	(  lowercase_   :							Any ,      lowercase_   :							Union[str, Any]		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			str												=       pt_model.visual.proj.data.T

 # copy layer norms
 copy_linear(hf_model.vision_model.pre_layernorm ,      pt_model.visual.ln_pre		)
 copy_linear(hf_model.vision_model.post_layernorm ,      pt_model.visual.ln_post		)

 # copy embeds
 __SCREAMING_SNAKE_CASE :			Dict												=       pt_model.visual.conva.weight.data
 __SCREAMING_SNAKE_CASE :			Tuple												=       pt_model.visual.class_embedding
 __SCREAMING_SNAKE_CASE :			Dict												=       pt_model.visual.positional_embedding.data

 # copy encoder
 copy_layers(hf_model.vision_model.encoder.layers ,      pt_model.visual.transformer.resblocks		)




def 		lowerCAmelCase_	(  lowercase_   :							Union[str, Any] ,      lowercase_   :							Optional[int]		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       flatten_nested_dict(flax_params['''backbone''']['''merged_class_token''']		)

 __SCREAMING_SNAKE_CASE :			Dict												=       torch.from_numpy(flax_class_token_params['''scale''']		)
 __SCREAMING_SNAKE_CASE :			List[str]												=       torch.from_numpy(flax_class_token_params['''bias''']		)
 __SCREAMING_SNAKE_CASE :			Dict												=       nn.Parameter(lowercase_		)
 __SCREAMING_SNAKE_CASE :			List[Any]												=       nn.Parameter(lowercase_		)




def 		lowerCAmelCase_	(  lowercase_   :							Optional[Any] ,      lowercase_   :							Union[str, Any]		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			List[Any]												=       hf_model.state_dict()
 __SCREAMING_SNAKE_CASE :			List[Any]												=       {}

 # Rename class prediction head flax params to pytorch HF
 __SCREAMING_SNAKE_CASE :			Dict												=       flatten_nested_dict(flax_params['''class_head''']		)

 for flax_key, v in flax_class_params.items():
  __SCREAMING_SNAKE_CASE :			str												=       flax_key.replace('''/''' ,      '''.'''		)
  __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       torch_key.replace('''.kernel''' ,      '''.weight'''		)
  __SCREAMING_SNAKE_CASE :			Any												=       torch_key.replace('''Dense_0''' ,      '''dense0'''		)
  __SCREAMING_SNAKE_CASE :			Tuple												=       '''class_head.''' + torch_key

  if "weight" in torch_key and v.ndim == 2:
   __SCREAMING_SNAKE_CASE :			int												=       v.T

  __SCREAMING_SNAKE_CASE :			List[str]												=       nn.Parameter(torch.from_numpy(lowercase_		)		)

 # Rename box prediction box flax params to pytorch HF
 __SCREAMING_SNAKE_CASE :			Any												=       flatten_nested_dict(flax_params['''obj_box_head''']		)

 for flax_key, v in flax_box_params.items():
  __SCREAMING_SNAKE_CASE :			Tuple												=       flax_key.replace('''/''' ,      '''.'''		)
  __SCREAMING_SNAKE_CASE :			List[str]												=       torch_key.replace('''.kernel''' ,      '''.weight'''		)
  __SCREAMING_SNAKE_CASE :			Tuple												=       torch_key.replace('''_''' ,      ''''''		).lower()
  __SCREAMING_SNAKE_CASE :			List[str]												=       '''box_head.''' + torch_key

  if "weight" in torch_key and v.ndim == 2:
   __SCREAMING_SNAKE_CASE :			Dict												=       v.T

  __SCREAMING_SNAKE_CASE :			Optional[Any]												=       nn.Parameter(torch.from_numpy(lowercase_		)		)

 # Copy flax params to PyTorch params
 for name, param in new_params.items():
  if name in pt_params.keys():
   pt_params[name].copy_(lowercase_		)




def 		lowerCAmelCase_	(  lowercase_   :							Optional[Any] ,      lowercase_   :							Optional[Any]		):

 '''simple docstring'''






 for k, v in flax_attn_params.items():
  if k.startswith('''transformer'''		):
   __SCREAMING_SNAKE_CASE :			Optional[int]												=       k.replace('''transformer.resblocks''' ,      '''text_model.encoder.layers'''		)
  else:
   __SCREAMING_SNAKE_CASE :			List[Any]												=       k.replace('''visual.transformer.resblocks''' ,      '''vision_model.encoder.layers'''		)

  __SCREAMING_SNAKE_CASE :			List[Any]												=       torch_key.replace('''attn''' ,      '''self_attn'''		)
  __SCREAMING_SNAKE_CASE :			Dict												=       torch_key.replace('''key''' ,      '''k_proj'''		)
  __SCREAMING_SNAKE_CASE :			Optional[Any]												=       torch_key.replace('''value''' ,      '''v_proj'''		)
  __SCREAMING_SNAKE_CASE :			Tuple												=       torch_key.replace('''query''' ,      '''q_proj'''		)
  __SCREAMING_SNAKE_CASE :			Optional[int]												=       torch_key.replace('''out''' ,      '''out_proj'''		)

  if "bias" in torch_key and v.ndim == 2:
   __SCREAMING_SNAKE_CASE :			Optional[Any]												=       v.shape[0] * v.shape[1]
   __SCREAMING_SNAKE_CASE :			Optional[Any]												=       v.reshape(lowercase_		)

  if "weight" in torch_key and "out" in torch_key:
   __SCREAMING_SNAKE_CASE :			List[Any]												=       (v.shape[0] * v.shape[1], v.shape[2])
   __SCREAMING_SNAKE_CASE :			List[str]												=       v.reshape(lowercase_		).T

  if "weight" in torch_key and "out" not in torch_key:
   __SCREAMING_SNAKE_CASE :			Any												=       (v.shape[0], v.shape[1] * v.shape[2])
   __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       v.reshape(lowercase_		).T

  # Copy flax CLIP attn params to HF PyTorch params
  __SCREAMING_SNAKE_CASE :			Optional[int]												=       torch.from_numpy(lowercase_		)
  hf_backbone.state_dict()[torch_key].copy_(lowercase_		)




def 		lowerCAmelCase_	(  lowercase_   :							Optional[int]		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			int												=       {}
 __SCREAMING_SNAKE_CASE :			int												=       []

 for k, v in params.items():
  if "attn." in k:
   __SCREAMING_SNAKE_CASE :			Optional[Any]												=       k[: k.rindex('''attn.'''		) + 5]
   if base in processed_attn_layers:
    continue

   processed_attn_layers.append(lowercase_		)
   __SCREAMING_SNAKE_CASE :			Optional[Any]												=       params[base + '''out.weight'''].shape[-1]
   __SCREAMING_SNAKE_CASE :			Optional[Any]												=       params[base + '''out.weight'''].reshape(lowercase_ ,      lowercase_		).T
   __SCREAMING_SNAKE_CASE :			Dict												=       params[base + '''out.bias''']
  else:
   __SCREAMING_SNAKE_CASE :			Any												=       v
 return new_params




def 		lowerCAmelCase_	(  lowercase_   :							str ,      lowercase_   :							Dict		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			int												=       CLIP(**lowercase_		)
 torch_model.eval()
 __SCREAMING_SNAKE_CASE :			List[str]												=       torch_model.state_dict()

 __SCREAMING_SNAKE_CASE :			str												=       flatten_nested_dict(flax_params['''backbone''']['''clip''']		)
 __SCREAMING_SNAKE_CASE :			str												=       {}

 for flax_key, v in flax_clip_params.items():
  __SCREAMING_SNAKE_CASE :			int												=       flax_key.replace('''/''' ,      '''.'''		)
  __SCREAMING_SNAKE_CASE :			Tuple												=       torch_key.replace('''text.token_embedding.embedding''' ,      '''token_embedding.kernel'''		)

  if (
      torch_key.startswith('''text.transformer'''		)
      or torch_key.startswith('''text.text_projection'''		)
      or torch_key.startswith('''text.ln_final'''		)
      or torch_key.startswith('''text.positional_embedding'''		)
  ):
   __SCREAMING_SNAKE_CASE :			List[str]												=       torch_key[5:]

  __SCREAMING_SNAKE_CASE :			Optional[int]												=       torch_key.replace('''text_projection.kernel''' ,      '''text_projection'''		)
  __SCREAMING_SNAKE_CASE :			str												=       torch_key.replace('''visual.proj.kernel''' ,      '''visual.proj'''		)
  __SCREAMING_SNAKE_CASE :			Tuple												=       torch_key.replace('''.scale''' ,      '''.weight'''		)
  __SCREAMING_SNAKE_CASE :			Dict												=       torch_key.replace('''.kernel''' ,      '''.weight'''		)

  if "conv" in torch_key or "downsample.0.weight" in torch_key:
   __SCREAMING_SNAKE_CASE :			Optional[int]												=       v.transpose(3 ,      2 ,      0 ,      1		)

  elif "weight" in torch_key and v.ndim == 2 and "embedding" not in torch_key:
   # Fully connected layers are transposed, embeddings are not
   __SCREAMING_SNAKE_CASE :			str												=       v.T

  __SCREAMING_SNAKE_CASE :			Any												=       v

 __SCREAMING_SNAKE_CASE :			Optional[int]												=       _convert_attn_layers(lowercase_		)
 new_torch_params.update(lowercase_		)
 __SCREAMING_SNAKE_CASE :			List[Any]												=       {}

 # Copy flax CLIP backbone params to PyTorch params
 for name, param in new_torch_params.items():
  if name in torch_clip_params.keys():
   __SCREAMING_SNAKE_CASE :			Any												=       torch.from_numpy(new_torch_params[name]		)
   torch_clip_params[name].copy_(lowercase_		)
  else:
   __SCREAMING_SNAKE_CASE :			Tuple												=       param

 return torch_clip_params, torch_model, attn_params




@torch.no_grad()
def 		lowerCAmelCase_	(  lowercase_   :							Any ,      lowercase_   :							Union[str, Any] ,      lowercase_   :							str ,      lowercase_   :							Optional[Any] ,      lowercase_   :							List[Any]=None		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			int												=       Repository(lowercase_ ,      clone_from=F'''google/{pytorch_dump_folder_path}'''		)
 repo.git_pull()

 if config_path is not None:
  __SCREAMING_SNAKE_CASE :			Optional[Any]												=       OwlViTConfig.from_pretrained(lowercase_		)
 else:
  __SCREAMING_SNAKE_CASE :			int												=       OwlViTConfig()

 __SCREAMING_SNAKE_CASE :			List[str]												=       OwlViTModel(lowercase_		).eval()
 __SCREAMING_SNAKE_CASE :			Tuple												=       OwlViTForObjectDetection(lowercase_		).eval()

 copy_text_model_and_projection(lowercase_ ,      lowercase_		)
 copy_vision_model_and_projection(lowercase_ ,      lowercase_		)
 __SCREAMING_SNAKE_CASE :			Optional[Any]												=       pt_backbone.logit_scale
 copy_flax_attn_params(lowercase_ ,      lowercase_		)

 __SCREAMING_SNAKE_CASE :			Tuple												=       hf_backbone
 copy_class_merge_token(lowercase_ ,      lowercase_		)
 copy_class_box_heads(lowercase_ ,      lowercase_		)

 # Save HF model
 hf_model.save_pretrained(repo.local_dir		)

 # Initialize image processor
 __SCREAMING_SNAKE_CASE :			List[Any]												=       OwlViTImageProcessor(
     size=config.vision_config.image_size ,      crop_size=config.vision_config.image_size		)
 # Initialize tokenizer
 __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       CLIPTokenizer.from_pretrained('''openai/clip-vit-base-patch32''' ,      pad_token='''!''' ,      model_max_length=16		)

 # Initialize processor
 __SCREAMING_SNAKE_CASE :			Optional[int]												=       OwlViTProcessor(image_processor=lowercase_ ,      tokenizer=lowercase_		)
 image_processor.save_pretrained(repo.local_dir		)
 processor.save_pretrained(repo.local_dir		)

 repo.git_add()
 repo.git_commit('''Upload model and processor'''		)
 repo.git_push()


if __name__ == "__main__":
    _lowerCamelCase       =	argparse.ArgumentParser()
    # Required parameters
    parser.add_argument(
        '''--owlvit_version''',
        default=None,
        type=str,
        required=True,
        help='''OWL-ViT model name [clip_b16, clip_b32, clip_l14].''',
    )
    parser.add_argument(
        '''--owlvit_checkpoint''', default=None, type=str, required=True, help='''Path to flax model checkpoint.'''
    )
    parser.add_argument('''--hf_config''', default=None, type=str, required=True, help='''Path to HF model config.''')
    parser.add_argument(
        '''--pytorch_dump_folder_path''', default='''hf_model''', type=str, help='''Path to the output PyTorch model.'''
    )
    _lowerCamelCase       =	parser.parse_args()

    # Initialize PyToch clip model
    _lowerCamelCase       =	args.owlvit_version
    if model_name == "clip_b16":
        _lowerCamelCase       =	CONFIGS['''vit_b16''']
    elif model_name == "clip_b32":
        _lowerCamelCase       =	CONFIGS['''vit_b32''']
    elif model_name == "clip_l14":
        _lowerCamelCase       =	CONFIGS['''vit_l14''']

    # Load from checkpoint and convert params to float-32
    _lowerCamelCase       =	checkpoints.restore_checkpoint(args.owlvit_checkpoint, target=None)['''optimizer''']['''target''']
    _lowerCamelCase       =	jax.tree_util.tree_map(lambda x: x.astype(jnp.floataa) if x.dtype == jnp.bfloataa else x, variables)
    del variables

    # Convert CLIP backbone
    _lowerCamelCase     ,		_lowerCamelCase     ,		_lowerCamelCase       =	convert_clip_backbone(flax_params, torch_config)

    convert_owlvit_checkpoint(clip_pt, flax_params, attn_params, args.pytorch_dump_folder_path, args.hf_config)

