


'''simple docstring'''
import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ViTImageProcessor, ViTMSNConfig, ViTMSNModel
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD


torch.set_grad_enabled(False)


def 			_lowercase       (				UpperCamelCase__       :					int,				UpperCamelCase__       :					Dict=False			):
	__A  :		Tuple       =			[]
	for i in range(config.num_hidden_layers			):
		# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
		rename_keys.append((f"""module.blocks.{i}.norm1.weight""", f"""vit.encoder.layer.{i}.layernorm_before.weight""")			)
		rename_keys.append((f"""module.blocks.{i}.norm1.bias""", f"""vit.encoder.layer.{i}.layernorm_before.bias""")			)
		rename_keys.append(
		    (f"""module.blocks.{i}.attn.proj.weight""", f"""vit.encoder.layer.{i}.attention.output.dense.weight""")			)
		rename_keys.append((f"""module.blocks.{i}.attn.proj.bias""", f"""vit.encoder.layer.{i}.attention.output.dense.bias""")			)
		rename_keys.append((f"""module.blocks.{i}.norm2.weight""", f"""vit.encoder.layer.{i}.layernorm_after.weight""")			)
		rename_keys.append((f"""module.blocks.{i}.norm2.bias""", f"""vit.encoder.layer.{i}.layernorm_after.bias""")			)
		rename_keys.append((f"""module.blocks.{i}.mlp.fc1.weight""", f"""vit.encoder.layer.{i}.intermediate.dense.weight""")			)
		rename_keys.append((f"""module.blocks.{i}.mlp.fc1.bias""", f"""vit.encoder.layer.{i}.intermediate.dense.bias""")			)
		rename_keys.append((f"""module.blocks.{i}.mlp.fc2.weight""", f"""vit.encoder.layer.{i}.output.dense.weight""")			)
		rename_keys.append((f"""module.blocks.{i}.mlp.fc2.bias""", f"""vit.encoder.layer.{i}.output.dense.bias""")			)

	# projection layer + position embeddings
	rename_keys.extend(
	    [
	        ('module.cls_token', 'vit.embeddings.cls_token'),
	        ('module.patch_embed.proj.weight', 'vit.embeddings.patch_embeddings.projection.weight'),
	        ('module.patch_embed.proj.bias', 'vit.embeddings.patch_embeddings.projection.bias'),
	        ('module.pos_embed', 'vit.embeddings.position_embeddings'),
	    ]			)

	if base_model:
		# layernorm + pooler
		rename_keys.extend(
		    [
		        ('module.norm.weight', 'layernorm.weight'),
		        ('module.norm.bias', 'layernorm.bias'),
		    ]			)

		# if just the base model, we should remove "vit" from all keys that start with "vit"
		__A  :		Dict       =			[(pair[0], pair[1][4:]) if pair[1].startswith('vit'			) else pair for pair in rename_keys]
	else:
		# layernorm + classification head
		rename_keys.extend(
		    [
		        ('norm.weight', 'vit.layernorm.weight'),
		        ('norm.bias', 'vit.layernorm.bias'),
		        ('head.weight', 'classifier.weight'),
		        ('head.bias', 'classifier.bias'),
		    ]			)

	return rename_keys


def 			_lowercase       (				UpperCamelCase__       :					Union[str, Any],				UpperCamelCase__       :					int,				UpperCamelCase__       :					List[str]=False			):


	for i in range(config.num_hidden_layers			):
		if base_model:
			__A  :		Dict       =			""
		else:
			__A  :		Union[str, Any]       =			"vit."
		# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
		__A  :		Any       =			state_dict.pop(f"""module.blocks.{i}.attn.qkv.weight"""			)
		__A  :		List[str]       =			state_dict.pop(f"""module.blocks.{i}.attn.qkv.bias"""			)
		# next, add query, keys and values (in that order) to the state dict
		__A  :		Tuple       =			in_proj_weight[
		    : config.hidden_size, :
		]
		__A  :		str       =			in_proj_bias[: config.hidden_size]
		__A  :		Optional[int]       =			in_proj_weight[
		    config.hidden_size : config.hidden_size * 2, :
		]
		__A  :		Optional[int]       =			in_proj_bias[
		    config.hidden_size : config.hidden_size * 2
		]
		__A  :		Optional[int]       =			in_proj_weight[
		    -config.hidden_size :, :
		]
		__A  :		Union[str, Any]       =			in_proj_bias[-config.hidden_size :]


def 			_lowercase       (				UpperCamelCase__       :					List[str]			):
	__A  :		Dict       =			["head.weight", "head.bias"]
	for k in ignore_keys:
		state_dict.pop(_lowerCamelCase,				_lowerCamelCase			)


def 			_lowercase       (				UpperCamelCase__       :					str			):
	__A  :		List[Any]       =			[
	    "module.fc.fc1.weight",
	    "module.fc.fc1.bias",
	    "module.fc.bn1.weight",
	    "module.fc.bn1.bias",
	    "module.fc.bn1.running_mean",
	    "module.fc.bn1.running_var",
	    "module.fc.bn1.num_batches_tracked",
	    "module.fc.fc2.weight",
	    "module.fc.fc2.bias",
	    "module.fc.bn2.weight",
	    "module.fc.bn2.bias",
	    "module.fc.bn2.running_mean",
	    "module.fc.bn2.running_var",
	    "module.fc.bn2.num_batches_tracked",
	    "module.fc.fc3.weight",
	    "module.fc.fc3.bias",
	]
	for k in ignore_keys:
		state_dict.pop(_lowerCamelCase,				_lowerCamelCase			)


def 			_lowercase       (				UpperCamelCase__       :					Optional[int],				UpperCamelCase__       :					List[str],				UpperCamelCase__       :					int			):
	__A  :		List[Any]       =			dct.pop(_lowerCamelCase			)
	__A  :		Union[str, Any]       =			val


def 			_lowercase       (				UpperCamelCase__       :					List[str],				UpperCamelCase__       :					Optional[int]			):
	__A  :		List[str]       =			ViTMSNConfig()
	__A  :		Any       =			1000

	__A  :		Optional[int]       =			"datasets/huggingface/label-files"
	__A  :		Any       =			"imagenet-1k-id2label.json"
	__A  :		Any       =			json.load(open(hf_hub_download(_lowerCamelCase,				_lowerCamelCase			),				'r'			)			)
	__A  :		Optional[Any]       =			{int(_lowerCamelCase			): v for k, v in idalabel.items()}
	__A  :		Dict       =			idalabel
	__A  :		List[str]       =			{v: k for k, v in idalabel.items()}

	if "s16" in checkpoint_url:
		__A  :		Optional[int]       =			384
		__A  :		Optional[Any]       =			1536
		__A  :		List[Any]       =			6
	elif "l16" in checkpoint_url:
		__A  :		str       =			1024
		__A  :		str       =			4096
		__A  :		Dict       =			24
		__A  :		Optional[int]       =			16
		__A  :		Optional[Any]       =			0.1
	elif "b4" in checkpoint_url:
		__A  :		Optional[int]       =			4
	elif "l7" in checkpoint_url:
		__A  :		int       =			7
		__A  :		Optional[Any]       =			1024
		__A  :		List[Any]       =			4096
		__A  :		Optional[Any]       =			24
		__A  :		Tuple       =			16
		__A  :		int       =			0.1

	__A  :		str       =			ViTMSNModel(_lowerCamelCase			)

	__A  :		List[str]       =			torch.hub.load_state_dict_from_url(_lowerCamelCase,				map_location='cpu'			)["target_encoder"]

	__A  :		Union[str, Any]       =			ViTImageProcessor(size=config.image_size			)

	remove_projection_head(_lowerCamelCase			)
	__A  :		str       =			create_rename_keys(_lowerCamelCase,				base_model=_lowerCamelCase			)

	for src, dest in rename_keys:
		rename_key(_lowerCamelCase,				_lowerCamelCase,				_lowerCamelCase			)
	read_in_q_k_v(_lowerCamelCase,				_lowerCamelCase,				base_model=_lowerCamelCase			)

	model.load_state_dict(_lowerCamelCase			)
	model.eval()

	__A  :		Tuple       =			"http://images.cocodataset.org/val2017/000000039769.jpg"

	__A  :		Tuple       =			Image.open(requests.get(_lowerCamelCase,				stream=_lowerCamelCase			).raw			)
	__A  :		Dict       =			ViTImageProcessor(
	    size=config.image_size,				image_mean=_lowerCamelCase,				image_std=_lowerCamelCase			)
	__A  :		str       =			image_processor(images=_lowerCamelCase,				return_tensors='pt'			)

	# forward pass
	torch.manual_seed(2			)
	__A  :		Tuple       =			model(**_lowerCamelCase			)
	__A  :		Any       =			outputs.last_hidden_state

	# The following Colab Notebook was used to generate these outputs:
	# https://colab.research.google.com/gist/sayakpaul/3672419a04f5997827503fd84079bdd1/scratchpad.ipynb
	if "s16" in checkpoint_url:
		__A  :		Optional[Any]       =			torch.tensor([[-1.0915, -1.4876, -1.1809]]			)
	elif "b16" in checkpoint_url:
		__A  :		Dict       =			torch.tensor([[14.2889, -18.9045, 11.7281]]			)
	elif "l16" in checkpoint_url:
		__A  :		Union[str, Any]       =			torch.tensor([[41.5028, -22.8681, 45.6475]]			)
	elif "b4" in checkpoint_url:
		__A  :		int       =			torch.tensor([[-4.3868, 5.2932, -0.4137]]			)
	else:
		__A  :		Optional[Any]       =			torch.tensor([[-0.1792, -0.6465, 2.4263]]			)

	# verify logits
	assert torch.allclose(last_hidden_state[:, 0, :3],				_lowerCamelCase,				atol=1E-4			)

	print(f"""Saving model to {pytorch_dump_folder_path}"""			)
	model.save_pretrained(_lowerCamelCase			)

	print(f"""Saving image processor to {pytorch_dump_folder_path}"""			)
	image_processor.save_pretrained(_lowerCamelCase			)


if __name__ == "__main__":
	UpperCAmelCase_ :     List[str]								=      argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '--checkpoint_url',
	    default='https://dl.fbaipublicfiles.com/msn/vits16_800ep.pth.tar',
	    type=str,
	    help='URL of the checkpoint you\'d like to convert.',
	)
	parser.add_argument(
	    '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
	)

	UpperCAmelCase_ :     Dict								=      parser.parse_args()
	convert_vit_msn_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)










'''simple docstring'''
import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel

class 						_lowerCamelCase					:




 '''simple docstring'''


 def __init__(      self      ,      __lowercase      ,      __lowercase=13      ,      __lowercase=7      ,      __lowercase=True      ,      __lowercase=True      ,      __lowercase=False      ,      __lowercase=True      ,      __lowercase=99      ,      __lowercase=32      ,      __lowercase=5      ,      __lowercase=4      ,      __lowercase=37      ,      __lowercase="gelu"      ,      __lowercase=0.1      ,      __lowercase=0.1      ,      __lowercase=512      ,      __lowercase=16      ,      __lowercase=2      ,      __lowercase=0.0_2      ,      __lowercase=3      ,      __lowercase=4      ,      __lowercase=None      ,      ):

  """simple docstring"""







  __A  :		Optional[int]       =			parent
  __A  :		Tuple       =			batch_size
  __A  :		Optional[int]       =			seq_length
  __A  :		Tuple       =			is_training
  __A  :		Optional[Any]       =			use_input_mask
  __A  :		Optional[Any]       =			use_token_type_ids
  __A  :		Optional[int]       =			use_labels
  __A  :		str       =			vocab_size
  __A  :		Dict       =			hidden_size
  __A  :		Tuple       =			num_hidden_layers
  __A  :		Optional[int]       =			num_attention_heads
  __A  :		str       =			intermediate_size
  __A  :		List[Any]       =			hidden_act
  __A  :		List[str]       =			hidden_dropout_prob
  __A  :		int       =			attention_probs_dropout_prob
  __A  :		int       =			max_position_embeddings
  __A  :		int       =			type_vocab_size
  __A  :		int       =			type_sequence_label_size
  __A  :		str       =			initializer_range
  __A  :		str       =			num_labels
  __A  :		str       =			num_choices
  __A  :		Any       =			scope


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		Union[str, Any]       =			ids_tensor([self.batch_size, self.seq_length]      ,      self.vocab_size						)

  __A  :		Union[str, Any]       =			None
  if self.use_input_mask:
   __A  :		List[str]       =			random_attention_mask([self.batch_size, self.seq_length]						)

  __A  :		str       =			None
  if self.use_token_type_ids:
   __A  :		List[str]       =			ids_tensor([self.batch_size, self.seq_length]      ,      self.type_vocab_size						)

  __A  :		Union[str, Any]       =			None
  __A  :		Optional[int]       =			None
  __A  :		List[str]       =			None
  if self.use_labels:
   __A  :		str       =			ids_tensor([self.batch_size]      ,      self.type_sequence_label_size						)
   __A  :		List[str]       =			ids_tensor([self.batch_size, self.seq_length]      ,      self.num_labels						)
   __A  :		Union[str, Any]       =			ids_tensor([self.batch_size]      ,      self.num_choices						)

  __A  :		List[Any]       =			self.get_config()

  return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


 def  snake_case__   (      self						):

  """simple docstring"""







  return OpenLlamaConfig(
      vocab_size=self.vocab_size      ,      hidden_size=self.hidden_size      ,      num_hidden_layers=self.num_hidden_layers      ,      num_attention_heads=self.num_attention_heads      ,      intermediate_size=self.intermediate_size      ,      hidden_act=self.hidden_act      ,      hidden_dropout_prob=self.hidden_dropout_prob      ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,      max_position_embeddings=self.max_position_embeddings      ,      type_vocab_size=self.type_vocab_size      ,      is_decoder=__lowercase      ,      initializer_range=self.initializer_range      ,      use_stable_embedding=__lowercase      ,      )


 def  snake_case__   (      self      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase						):

  """simple docstring"""







  __A  :		List[str]       =			OpenLlamaModel(config=__lowercase						)
  model.to(__lowercase						)
  model.eval()
  __A  :		Any       =			model(__lowercase      ,      attention_mask=__lowercase						)
  __A  :		Dict       =			model(__lowercase						)
  self.parent.assertEqual(result.last_hidden_state.shape      ,      (self.batch_size, self.seq_length, self.hidden_size)						)


 def  snake_case__   (      self      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      ):

  """simple docstring"""







  __A  :		List[str]       =			True
  __A  :		int       =			OpenLlamaModel(__lowercase						)
  model.to(__lowercase						)
  model.eval()
  __A  :		List[Any]       =			model(
      __lowercase      ,      attention_mask=__lowercase      ,      encoder_hidden_states=__lowercase      ,      encoder_attention_mask=__lowercase      ,      )
  __A  :		Optional[Any]       =			model(
      __lowercase      ,      attention_mask=__lowercase      ,      encoder_hidden_states=__lowercase      ,      )
  __A  :		Optional[Any]       =			model(__lowercase      ,      attention_mask=__lowercase						)
  self.parent.assertEqual(result.last_hidden_state.shape      ,      (self.batch_size, self.seq_length, self.hidden_size)						)


 def  snake_case__   (      self      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      ):

  """simple docstring"""







  __A  :		Dict       =			OpenLlamaForCausalLM(config=__lowercase						)
  model.to(__lowercase						)
  model.eval()
  __A  :		Optional[Any]       =			model(__lowercase      ,      attention_mask=__lowercase      ,      labels=__lowercase						)
  self.parent.assertEqual(result.logits.shape      ,      (self.batch_size, self.seq_length, self.vocab_size)						)


 def  snake_case__   (      self      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      __lowercase      ,      ):

  """simple docstring"""







  __A  :		List[Any]       =			True
  __A  :		Optional[int]       =			True
  __A  :		Dict       =			OpenLlamaForCausalLM(config=__lowercase						)
  model.to(__lowercase						)
  model.eval()

  # first forward pass
  __A  :		List[str]       =			model(
      __lowercase      ,      attention_mask=__lowercase      ,      encoder_hidden_states=__lowercase      ,      encoder_attention_mask=__lowercase      ,      use_cache=__lowercase      ,      )
  __A  :		Optional[Any]       =			outputs.past_key_values

  # create hypothetical multiple next token and extent to next_input_ids
  __A  :		Dict       =			ids_tensor((self.batch_size, 3)      ,      config.vocab_size						)
  __A  :		Optional[Any]       =			ids_tensor((self.batch_size, 3)      ,      vocab_size=2						)

  # append to next input_ids and
  __A  :		int       =			torch.cat([input_ids, next_tokens]      ,      dim=-1						)
  __A  :		Dict       =			torch.cat([input_mask, next_mask]      ,      dim=-1						)

  __A  :		Tuple       =			model(
      __lowercase      ,      attention_mask=__lowercase      ,      encoder_hidden_states=__lowercase      ,      encoder_attention_mask=__lowercase      ,      output_hidden_states=__lowercase      ,      )['hidden_states'][0]
  __A  :		Any       =			model(
      __lowercase      ,      attention_mask=__lowercase      ,      encoder_hidden_states=__lowercase      ,      encoder_attention_mask=__lowercase      ,      past_key_values=__lowercase      ,      output_hidden_states=__lowercase      ,      )['hidden_states'][0]

  # select random slice
  __A  :		Any       =			ids_tensor((1,)      ,      output_from_past.shape[-1]						).item()
  __A  :		Union[str, Any]       =			output_from_no_past[:, -3:, random_slice_idx].detach()
  __A  :		int       =			output_from_past[:, :, random_slice_idx].detach()

  self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]						)

  # test that outputs are equal for slice
  self.parent.assertTrue(torch.allclose(__lowercase      ,      __lowercase      ,      atol=1E-3						)						)







 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		int       =			self.prepare_config_and_inputs()
  (
      (
      __A
  )     ,(
      __A
  )     ,(
      __A
  )     ,(
      __A
  )     ,(
      __A
  )     ,(
      __A
  )     ,(
      __A
  )     ,
  )  :		Optional[Any]       =			config_and_inputs
  __A  :		Optional[int]       =			{'input_ids': input_ids, 'attention_mask': input_mask}
  return config, inputs_dict


@require_torch
class 						_lowerCamelCase					(    snake_case_    ,		snake_case_    ,		snake_case_    ,		unittest.TestCase				):




 '''simple docstring'''
 __lowercase				:		List[str]    							=						(
     (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
 )
 __lowercase				:		List[Any]    							=						(OpenLlamaForCausalLM,) if is_torch_available() else ()
 __lowercase				:		Optional[int]    							=						(
     {
         '''feature-extraction''': OpenLlamaModel,
         '''text-classification''': OpenLlamaForSequenceClassification,
         '''text-generation''': OpenLlamaForCausalLM,
         '''zero-shot''': OpenLlamaForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )
 __lowercase				:		Union[str, Any]    							=						False
 __lowercase				:		Union[str, Any]    							=						False


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		Optional[int]       =			OpenLlamaModelTester(self						)
  __A  :		List[str]       =			ConfigTester(self      ,      config_class=__lowercase      ,      hidden_size=37						)


 def  snake_case__   (      self						):

  """simple docstring"""







  self.config_tester.run_common_tests()


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		int       =			self.model_tester.prepare_config_and_inputs()
  self.model_tester.create_and_check_model(*__lowercase						)


 def  snake_case__   (      self						):

  """simple docstring"""







  __A  :		int       =			self.model_tester.prepare_config_and_inputs()
  for type in ["absolute", "relative_key", "relative_key_query"]:
   __A  :		List[str]       =			type
   self.model_tester.create_and_check_model(*__lowercase						)


 def  snake_case__   (      self						):

  """simple docstring"""







  __A     ,__A  :		Optional[Any]       =			self.model_tester.prepare_config_and_inputs_for_common()
  __A  :		Union[str, Any]       =			3
  __A  :		int       =			input_dict['input_ids']
  __A  :		int       =			input_ids.ne(1						).to(__lowercase						)
  __A  :		Optional[int]       =			ids_tensor([self.model_tester.batch_size]      ,      self.model_tester.type_sequence_label_size						)
  __A  :		Optional[Any]       =			OpenLlamaForSequenceClassification(__lowercase						)
  model.to(__lowercase						)
  model.eval()
  __A  :		Optional[int]       =			model(__lowercase      ,      attention_mask=__lowercase      ,      labels=__lowercase						)
  self.assertEqual(result.logits.shape      ,      (self.model_tester.batch_size, self.model_tester.num_labels)						)


 def  snake_case__   (      self						):

  """simple docstring"""







  __A     ,__A  :		str       =			self.model_tester.prepare_config_and_inputs_for_common()
  __A  :		Any       =			3
  __A  :		List[str]       =			'single_label_classification'
  __A  :		Dict       =			input_dict['input_ids']
  __A  :		Dict       =			input_ids.ne(1						).to(__lowercase						)
  __A  :		List[str]       =			ids_tensor([self.model_tester.batch_size]      ,      self.model_tester.type_sequence_label_size						)
  __A  :		Union[str, Any]       =			OpenLlamaForSequenceClassification(__lowercase						)
  model.to(__lowercase						)
  model.eval()
  __A  :		Dict       =			model(__lowercase      ,      attention_mask=__lowercase      ,      labels=__lowercase						)
  self.assertEqual(result.logits.shape      ,      (self.model_tester.batch_size, self.model_tester.num_labels)						)


 def  snake_case__   (      self						):

  """simple docstring"""







  __A     ,__A  :		Optional[int]       =			self.model_tester.prepare_config_and_inputs_for_common()
  __A  :		str       =			3
  __A  :		int       =			'multi_label_classification'
  __A  :		Union[str, Any]       =			input_dict['input_ids']
  __A  :		str       =			input_ids.ne(1						).to(__lowercase						)
  __A  :		Union[str, Any]       =			ids_tensor(
      [self.model_tester.batch_size, config.num_labels]      ,      self.model_tester.type_sequence_label_size						).to(torch.float						)
  __A  :		int       =			OpenLlamaForSequenceClassification(__lowercase						)
  model.to(__lowercase						)
  model.eval()
  __A  :		Any       =			model(__lowercase      ,      attention_mask=__lowercase      ,      labels=__lowercase						)
  self.assertEqual(result.logits.shape      ,      (self.model_tester.batch_size, self.model_tester.num_labels)						)


 @unittest.skip('Open-Llama buffers include complex numbers, which breaks this test'						)
 def  snake_case__   (      self						):

  """simple docstring"""







  pass







 @parameterized.expand([('linear',), ('dynamic',)]						)
 def  snake_case__   (      self      ,      __lowercase						):

  """simple docstring"""







  __A     ,__A  :		str       =			self.model_tester.prepare_config_and_inputs_for_common()
  __A  :		List[Any]       =			ids_tensor([1, 10]      ,      config.vocab_size						)
  __A  :		str       =			ids_tensor([1, int(config.max_position_embeddings * 1.5						)]      ,      config.vocab_size						)

  set_seed(42						)  # Fixed seed at init time so the two models get the same random weights
  __A  :		Union[str, Any]       =			OpenLlamaModel(__lowercase						)
  original_model.to(__lowercase						)
  original_model.eval()
  __A  :		Optional[int]       =			original_model(__lowercase						).last_hidden_state
  __A  :		int       =			original_model(__lowercase						).last_hidden_state

  set_seed(42						)  # Fixed seed at init time so the two models get the same random weights
  __A  :		List[Any]       =			{'type': scaling_type, 'factor': 1_0.0}
  __A  :		str       =			OpenLlamaModel(__lowercase						)
  scaled_model.to(__lowercase						)
  scaled_model.eval()
  __A  :		Dict       =			scaled_model(__lowercase						).last_hidden_state
  __A  :		List[str]       =			scaled_model(__lowercase						).last_hidden_state

  # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
  # maximum sequence length, so the outputs for the short input should match.
  if scaling_type == "dynamic":
   self.assertTrue(torch.allclose(__lowercase      ,      __lowercase      ,      atol=1E-5						)						)
  else:
   self.assertFalse(torch.allclose(__lowercase      ,      __lowercase      ,      atol=1E-5						)						)

  # The output should be different for long inputs
  self.assertFalse(torch.allclose(__lowercase      ,      __lowercase      ,      atol=1E-5						)						)






