






"""simple docstring"""





import inspect
import unittest

from transformers import MobileViTVaConfig
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import MobileViTVaForImageClassification, MobileViTVaForSemanticSegmentation, MobileViTVaModel
							from transformers.models.mobilevitva.modeling_mobilevitva import (
							    MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST,
							    make_divisible,
							)


if is_vision_available():
							from PIL import Image

							from transformers import MobileViTImageProcessor



class       __A		(   SCREAMING_SNAKE_CASE_			):

					def   lowerCamelCase__   (			self  :      Optional[int]					)			->							Dict:
											__magic_name__:	int							=       self.config_class(**self.inputs_dict					)
											self.parent.assertTrue(hasattr(__snake_case     ,		"""width_multiplier"""					)					)



class       __A		:

					def __init__(			self  :      Tuple     ,		__snake_case  :      List[str]     ,		__snake_case  :      int=1_3     ,		__snake_case  :      Optional[int]=6_4     ,		__snake_case  :      Optional[int]=2     ,		__snake_case  :      Tuple=3     ,		__snake_case  :      Any="swish"     ,		__snake_case  :      int=3     ,		__snake_case  :      int=3_2     ,		__snake_case  :      List[str]=0.1     ,		__snake_case  :      int=0.02     ,		__snake_case  :      Union[str, Any]=True     ,		__snake_case  :      List[Any]=True     ,		__snake_case  :      Optional[Any]=1_0     ,		__snake_case  :      Tuple=None     ,		__snake_case  :      Any=0.25     ,		__snake_case  :      List[Any]=0.0     ,		__snake_case  :      Dict=0.0     ,		)			->							Optional[int]:
											__magic_name__:	int							=       parent
											__magic_name__:	Union[str, Any]							=       batch_size
											__magic_name__:	Optional[Any]							=       image_size
											__magic_name__:	Optional[Any]							=       patch_size
											__magic_name__:	Union[str, Any]							=       num_channels
											__magic_name__:	Optional[Any]							=       make_divisible(5_1_2 * width_multiplier     ,		divisor=8					)
											__magic_name__:	List[Any]							=       hidden_act
											__magic_name__:	int							=       conv_kernel_size
											__magic_name__:	Tuple							=       output_stride
											__magic_name__:	Dict							=       classifier_dropout_prob
											__magic_name__:	Optional[Any]							=       use_labels
											__magic_name__:	List[str]							=       is_training
											__magic_name__:	List[str]							=       num_labels
											__magic_name__:	str							=       initializer_range
											__magic_name__:	Dict							=       scope
											__magic_name__:	str							=       width_multiplier
											__magic_name__:	str							=       ffn_dropout
											__magic_name__:	Optional[Any]							=       attn_dropout

					def   lowerCamelCase__   (			self  :      Any					)			->							List[str]:
											__magic_name__:	Dict							=       floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]					)

											__magic_name__:	List[str]							=       None
											__magic_name__:	Tuple							=       None
											if self.use_labels:
																	__magic_name__:	List[str]							=       ids_tensor([self.batch_size]     ,		self.num_labels					)
																	__magic_name__:	Any							=       ids_tensor([self.batch_size, self.image_size, self.image_size]     ,		self.num_labels					)

											__magic_name__:	Dict							=       self.get_config()

											return config, pixel_values, labels, pixel_labels

					def   lowerCamelCase__   (			self  :      Union[str, Any]					)			->							Any:
											return MobileViTVaConfig(
											    image_size=self.image_size     ,		patch_size=self.patch_size     ,		num_channels=self.num_channels     ,		hidden_act=self.hidden_act     ,		conv_kernel_size=self.conv_kernel_size     ,		output_stride=self.output_stride     ,		classifier_dropout_prob=self.classifier_dropout_prob     ,		initializer_range=self.initializer_range     ,		width_multiplier=self.width_multiplier     ,		ffn_dropout=self.ffn_dropout_prob     ,		attn_dropout=self.attn_dropout_prob     ,		)

					def   lowerCamelCase__   (			self  :      List[str]     ,		__snake_case  :      Dict     ,		__snake_case  :      List[Any]     ,		__snake_case  :      List[str]     ,		__snake_case  :      str					)			->							Optional[Any]:
											__magic_name__:	Optional[int]							=       MobileViTVaModel(config=__snake_case					)
											model.to(__snake_case					)
											model.eval()
											__magic_name__:	Optional[int]							=       model(__snake_case					)
											self.parent.assertEqual(
											    result.last_hidden_state.shape     ,		(
											        self.batch_size,
											        self.last_hidden_size,
											        self.image_size // self.output_stride,
											        self.image_size // self.output_stride,
											    )     ,		)

					def   lowerCamelCase__   (			self  :      Union[str, Any]     ,		__snake_case  :      Optional[int]     ,		__snake_case  :      Optional[Any]     ,		__snake_case  :      Any     ,		__snake_case  :      List[Any]					)			->							Any:
											__magic_name__:	Union[str, Any]							=       self.num_labels
											__magic_name__:	List[str]							=       MobileViTVaForImageClassification(__snake_case					)
											model.to(__snake_case					)
											model.eval()
											__magic_name__:	Optional[int]							=       model(__snake_case     ,		labels=__snake_case					)
											self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.num_labels)					)

					def   lowerCamelCase__   (			self  :      List[Any]     ,		__snake_case  :      Dict     ,		__snake_case  :      Optional[Any]     ,		__snake_case  :      str     ,		__snake_case  :      Union[str, Any]					)			->							Dict:
											__magic_name__:	Tuple							=       self.num_labels
											__magic_name__:	Any							=       MobileViTVaForSemanticSegmentation(__snake_case					)
											model.to(__snake_case					)
											model.eval()
											__magic_name__:	List[Any]							=       model(__snake_case					)
											self.parent.assertEqual(
											    result.logits.shape     ,		(
											        self.batch_size,
											        self.num_labels,
											        self.image_size // self.output_stride,
											        self.image_size // self.output_stride,
											    )     ,		)
											__magic_name__:	Any							=       model(__snake_case     ,		labels=__snake_case					)
											self.parent.assertEqual(
											    result.logits.shape     ,		(
											        self.batch_size,
											        self.num_labels,
											        self.image_size // self.output_stride,
											        self.image_size // self.output_stride,
											    )     ,		)






					def   lowerCamelCase__   (			self  :      Any					)			->							Union[str, Any]:
											__magic_name__:	str							=       self.prepare_config_and_inputs()
											__magic_name__,			__magic_name__,			__magic_name__,			__magic_name__:	Optional[int]							=       config_and_inputs
											__magic_name__:	Tuple							=       {"""pixel_values""": pixel_values}
											return config, inputs_dict



@require_torch
class       __A		(   SCREAMING_SNAKE_CASE_     ,SCREAMING_SNAKE_CASE_     ,unittest.TestCase			):
					UpperCAmelCase__								=				(
					    (MobileViTVaModel, MobileViTVaForImageClassification, MobileViTVaForSemanticSegmentation)
					    if is_torch_available()
					    else ()
					)

					UpperCAmelCase__								=				(
					    {
					        "feature-extraction": MobileViTVaModel,
					        "image-classification": MobileViTVaForImageClassification,
					        "image-segmentation": MobileViTVaForSemanticSegmentation,
					    }
					    if is_torch_available()
					    else {}
					)

					UpperCAmelCase__								=				False
					UpperCAmelCase__								=				False
					UpperCAmelCase__								=				False
					UpperCAmelCase__								=				False

					def   lowerCamelCase__   (			self  :      List[Any]					)			->							Any:
											__magic_name__:	List[str]							=       MobileViTVaModelTester(self					)
											__magic_name__:	str							=       MobileViTVaConfigTester(self     ,		config_class=__snake_case     ,		has_text_modality=__snake_case					)

					def   lowerCamelCase__   (			self  :      int					)			->							Dict:
											self.config_tester.run_common_tests()

					@unittest.skip(reason="""MobileViTV2 does not use inputs_embeds"""					)
					def   lowerCamelCase__   (			self  :      Optional[Any]					)			->							List[str]:
											pass

					@unittest.skip(reason="""MobileViTV2 does not support input and output embeddings"""					)
					def   lowerCamelCase__   (			self  :      List[str]					)			->							Dict:
											pass

					@unittest.skip(reason="""MobileViTV2 does not output attentions"""					)
					def   lowerCamelCase__   (			self  :      Optional[Any]					)			->							List[Any]:
											pass

					@require_torch_multi_gpu
					@unittest.skip(reason="""Got `CUDA error: misaligned address` for tests after this one being run."""					)
					def   lowerCamelCase__   (			self  :      str					)			->							str:
											pass

					@unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests."""					)
					def   lowerCamelCase__   (			self  :      str					)			->							Dict:
											pass

					def   lowerCamelCase__   (			self  :      Dict					)			->							List[str]:
											__magic_name__,			__magic_name__:	Dict							=       self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																	__magic_name__:	str							=       model_class(__snake_case					)
																	__magic_name__:	Dict							=       inspect.signature(model.forward					)
																	# signature.parameters is an OrderedDict => so arg_names order is deterministic
																	__magic_name__:	int							=       [*signature.parameters.keys()]

																	__magic_name__:	Dict							=       ["""pixel_values"""]
																	self.assertListEqual(arg_names[:1]     ,		__snake_case					)

					def   lowerCamelCase__   (			self  :      str					)			->							Optional[Any]:
											__magic_name__:	Any							=       self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*__snake_case					)

					def   lowerCamelCase__   (			self  :      Tuple					)			->							Optional[Any]:
											def check_hidden_states_output(__snake_case  :      Union[str, Any]     ,		__snake_case  :      str     ,		__snake_case  :      List[Any]					):
																	__magic_name__:	Union[str, Any]							=       model_class(__snake_case					)
																	model.to(__snake_case					)
																	model.eval()

																	with torch.no_grad():
																							__magic_name__:	str							=       model(**self._prepare_for_class(__snake_case     ,		__snake_case					)					)

																	__magic_name__:	List[Any]							=       outputs.hidden_states

																	__magic_name__:	List[Any]							=       5
																	self.assertEqual(len(__snake_case					)     ,		__snake_case					)

																	# MobileViTV2's feature maps are of shape (batch_size, num_channels, height, width)
																	# with the width and height being successively divided by 2.
																	__magic_name__:	int							=       2
																	for i in range(len(__snake_case					)					):
																							self.assertListEqual(
																							    list(hidden_states[i].shape[-2:]					)     ,		[self.model_tester.image_size // divisor, self.model_tester.image_size // divisor]     ,		)
																							divisor *= 2

																	self.assertEqual(self.model_tester.output_stride     ,		divisor // 2					)

											__magic_name__,			__magic_name__:	Dict							=       self.model_tester.prepare_config_and_inputs_for_common()

											for model_class in self.all_model_classes:
																	__magic_name__:	Dict							=       True
																	check_hidden_states_output(__snake_case     ,		__snake_case     ,		__snake_case					)

																	# check that output_hidden_states also work using config
																	del inputs_dict["output_hidden_states"]
																	__magic_name__:	Optional[int]							=       True

																	check_hidden_states_output(__snake_case     ,		__snake_case     ,		__snake_case					)

					def   lowerCamelCase__   (			self  :      Tuple					)			->							Union[str, Any]:
											__magic_name__:	Any							=       self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_image_classification(*__snake_case					)

					def   lowerCamelCase__   (			self  :      Tuple					)			->							str:
											__magic_name__:	Dict							=       self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_semantic_segmentation(*__snake_case					)






					@slow
					def   lowerCamelCase__   (			self  :      Union[str, Any]					)			->							Optional[int]:
											for model_name in MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																	__magic_name__:	Union[str, Any]							=       MobileViTVaModel.from_pretrained(__snake_case					)
																	self.assertIsNotNone(__snake_case					)



def   a    (     )    ->							Any:
						__magic_name__:	Any							=       Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""  )
						return image



@require_torch
@require_vision
class       __A		(   unittest.TestCase			):

					@cached_property
					def   lowerCamelCase__   (			self  :      Dict					)			->							str:
											return (
											    MobileViTImageProcessor.from_pretrained("""apple/mobilevitv2-1.0-imagenet1k-256"""					)
											    if is_vision_available()
											    else None
											)

					@slow
					def   lowerCamelCase__   (			self  :      Dict					)			->							int:
											__magic_name__:	Any							=       MobileViTVaForImageClassification.from_pretrained("""apple/mobilevitv2-1.0-imagenet1k-256"""					).to(
											    __snake_case					)

											__magic_name__:	Optional[Any]							=       self.default_image_processor
											__magic_name__:	Tuple							=       prepare_img()
											__magic_name__:	str							=       image_processor(images=__snake_case     ,		return_tensors="""pt"""					).to(__snake_case					)

											# forward pass
											with torch.no_grad():
																	__magic_name__:	Any							=       model(**__snake_case					)

											# verify the logits
											__magic_name__:	List[Any]							=       torch.Size((1, 1_0_0_0)					)
											self.assertEqual(outputs.logits.shape     ,		__snake_case					)

											__magic_name__:	Optional[int]							=       torch.tensor([-1.63_36E00, -7.32_04E-02, -5.18_83E-01]					).to(__snake_case					)

											self.assertTrue(torch.allclose(outputs.logits[0, :3]     ,		__snake_case     ,		atol=1E-4					)					)

					@slow
					def   lowerCamelCase__   (			self  :      List[Any]					)			->							Optional[Any]:
											__magic_name__:	Any							=       MobileViTVaForSemanticSegmentation.from_pretrained("""shehan97/mobilevitv2-1.0-voc-deeplabv3"""					)
											__magic_name__:	List[Any]							=       model.to(__snake_case					)

											__magic_name__:	str							=       MobileViTImageProcessor.from_pretrained("""shehan97/mobilevitv2-1.0-voc-deeplabv3"""					)

											__magic_name__:	str							=       prepare_img()
											__magic_name__:	str							=       image_processor(images=__snake_case     ,		return_tensors="""pt"""					).to(__snake_case					)

											# forward pass
											with torch.no_grad():
																	__magic_name__:	Any							=       model(**__snake_case					)
											__magic_name__:	Optional[Any]							=       outputs.logits

											# verify the logits
											__magic_name__:	List[str]							=       torch.Size((1, 2_1, 3_2, 3_2)					)
											self.assertEqual(logits.shape     ,		__snake_case					)

											__magic_name__:	Optional[Any]							=       torch.tensor(
											    [
											        [[7.0863, 7.1525, 6.8201], [6.6931, 6.8770, 6.8933], [6.2978, 7.0366, 6.9636]],
											        [[-3.7134, -3.6712, -3.6675], [-3.5825, -3.3549, -3.4777], [-3.3435, -3.3979, -3.2857]],
											        [[-2.9329, -2.8003, -2.7369], [-3.0564, -2.4780, -2.0207], [-2.6889, -1.9298, -1.7640]],
											    ]     ,		device=__snake_case     ,		)

											self.assertTrue(torch.allclose(logits[0, :3, :3, :3]     ,		__snake_case     ,		atol=1E-4					)					)






					@slow
					def   lowerCamelCase__   (			self  :      int					)			->							Optional[Any]:
											__magic_name__:	List[str]							=       MobileViTVaForSemanticSegmentation.from_pretrained("""shehan97/mobilevitv2-1.0-voc-deeplabv3"""					)
											__magic_name__:	Union[str, Any]							=       model.to(__snake_case					)

											__magic_name__:	List[Any]							=       MobileViTImageProcessor.from_pretrained("""shehan97/mobilevitv2-1.0-voc-deeplabv3"""					)

											__magic_name__:	str							=       prepare_img()
											__magic_name__:	List[str]							=       image_processor(images=__snake_case     ,		return_tensors="""pt"""					).to(__snake_case					)

											# forward pass
											with torch.no_grad():
																	__magic_name__:	Tuple							=       model(**__snake_case					)

											__magic_name__:	Union[str, Any]							=       outputs.logits.detach().cpu()

											__magic_name__:	Any							=       image_processor.post_process_semantic_segmentation(outputs=__snake_case     ,		target_sizes=[(5_0, 6_0)]					)
											__magic_name__:	Tuple							=       torch.Size((5_0, 6_0)					)
											self.assertEqual(segmentation[0].shape     ,		__snake_case					)

											__magic_name__:	str							=       image_processor.post_process_semantic_segmentation(outputs=__snake_case					)
											__magic_name__:	Optional[int]							=       torch.Size((3_2, 3_2)					)
											self.assertEqual(segmentation[0].shape     ,		__snake_case					)











"""simple docstring"""





import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
							from transformers import LlamaTokenizerFast
except ImportError as e:
							warnings.warn(e)
							warnings.warn(
							    'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
							)
							__lowerCamelCase							=  None
__lowerCamelCase							=  {
    '7B': 1_10_08,
    '13B': 1_38_24,
    '30B': 1_79_20,
    '65B': 2_20_16,
    '70B': 2_86_72,
}
__lowerCamelCase							=  {
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}



def   a    (     __UpperCAmelCase      :    List[str]      ,   __UpperCAmelCase      :    Optional[Any]=1      ,   __UpperCAmelCase      :    Optional[int]=2_5_6  )    ->							List[str]:
						return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3  )  ) + multiple_of - 1) // multiple_of)



def   a    (     __UpperCAmelCase      :    Union[str, Any]  )    ->							Optional[Any]:
						with open(__UpperCAmelCase      ,   """r"""  ) as f:
												return json.load(__UpperCAmelCase  )



def   a    (     __UpperCAmelCase      :    Tuple      ,   __UpperCAmelCase      :    Union[str, Any]  )    ->							Optional[int]:
						with open(__UpperCAmelCase      ,   """w"""  ) as f:
												json.dump(__UpperCAmelCase      ,   __UpperCAmelCase  )



def   a    (     __UpperCAmelCase      :    List[Any]      ,   __UpperCAmelCase      :    Optional[int]      ,   __UpperCAmelCase      :    str      ,   __UpperCAmelCase      :    Tuple=True  )    ->							Tuple:
						os.makedirs(__UpperCAmelCase      ,   exist_ok=__UpperCAmelCase  )
						__magic_name__:	Any							=       os.path.join(__UpperCAmelCase      ,   """tmp"""  )
						os.makedirs(__UpperCAmelCase      ,   exist_ok=__UpperCAmelCase  )

						__magic_name__:	str							=       read_json(os.path.join(__UpperCAmelCase      ,   """params.json"""  )  )
						__magic_name__:	Dict							=       NUM_SHARDS[model_size]
						__magic_name__:	int							=       params["""n_layers"""]
						__magic_name__:	Optional[Any]							=       params["""n_heads"""]
						__magic_name__:	Optional[int]							=       n_heads // num_shards
						__magic_name__:	Optional[int]							=       params["""dim"""]
						__magic_name__:	Any							=       dim // n_heads
						__magic_name__:	Optional[Any]							=       1_00_00.0
						__magic_name__:	int							=       1.0 / (base ** (torch.arange(0      ,   __UpperCAmelCase      ,   2  ).float() / dims_per_head))

						if "n_kv_heads" in params:
												__magic_name__:	Dict							=       params["""n_kv_heads"""]  # for GQA / MQA
												__magic_name__:	Optional[Any]							=       n_heads_per_shard // num_key_value_heads
												__magic_name__:	Union[str, Any]							=       dim // num_key_value_heads
						else:  # compatibility with other checkpoints
												__magic_name__:	List[Any]							=       n_heads
												__magic_name__:	Union[str, Any]							=       n_heads_per_shard
												__magic_name__:	Tuple							=       dim

						# permute for sliced rotary
						def permute(__UpperCAmelCase      :    Optional[Any]      ,   __UpperCAmelCase      :    List[Any]=n_heads      ,   __UpperCAmelCase      :    Tuple=dim      ,   __UpperCAmelCase      :    Tuple=dim  ):
												return w.view(__UpperCAmelCase      ,   dima // n_heads // 2      ,   2      ,   __UpperCAmelCase  ).transpose(1      ,   2  ).reshape(__UpperCAmelCase      ,   __UpperCAmelCase  )

						print(f'Fetching all parameters from the checkpoint at {input_base_path}.'  )
						# Load weights
						if model_size == "7B":
												# Not sharded
												# (The sharded implementation would also work, but this is simpler.)
												__magic_name__:	Optional[int]							=       torch.load(os.path.join(__UpperCAmelCase      ,   """consolidated.00.pth"""  )      ,   map_location="""cpu"""  )
						else:
												# Sharded
												__magic_name__:	List[Any]							=       [
												    torch.load(os.path.join(__UpperCAmelCase      ,   f'consolidated.{i:02d}.pth'  )      ,   map_location="""cpu"""  )
												    for i in range(__UpperCAmelCase  )
												]
						__magic_name__:	Dict							=       0
						__magic_name__:	Any							=       {"""weight_map""": {}}
						for layer_i in range(__UpperCAmelCase  ):
												__magic_name__:	Tuple							=       f'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'
												if model_size == "7B":
																		# Unsharded
																		__magic_name__:	List[Any]							=       {
																		    f'model.layers.{layer_i}.self_attn.q_proj.weight': permute(
																		        loaded[f'layers.{layer_i}.attention.wq.weight']  ),
																		    f'model.layers.{layer_i}.self_attn.k_proj.weight': permute(
																		        loaded[f'layers.{layer_i}.attention.wk.weight']  ),
																		    f'model.layers.{layer_i}.self_attn.v_proj.weight': loaded[f'layers.{layer_i}.attention.wv.weight'],
																		    f'model.layers.{layer_i}.self_attn.o_proj.weight': loaded[f'layers.{layer_i}.attention.wo.weight'],
																		    f'model.layers.{layer_i}.mlp.gate_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w1.weight'],
																		    f'model.layers.{layer_i}.mlp.down_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w2.weight'],
																		    f'model.layers.{layer_i}.mlp.up_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w3.weight'],
																		    f'model.layers.{layer_i}.input_layernorm.weight': loaded[f'layers.{layer_i}.attention_norm.weight'],
																		    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[f'layers.{layer_i}.ffn_norm.weight'],
																		}
												else:
																		# Sharded
																		# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
																		# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
																		# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

																		__magic_name__:	Dict							=       {
																		    f'model.layers.{layer_i}.input_layernorm.weight': loaded[0][
																		        f'layers.{layer_i}.attention_norm.weight'
																		    ].clone(),
																		    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[0][
																		        f'layers.{layer_i}.ffn_norm.weight'
																		    ].clone(),
																		}
																		__magic_name__:	Union[str, Any]							=       permute(
																		    torch.cat(
																		        [
																		            loaded[i][f'layers.{layer_i}.attention.wq.weight'].view(__UpperCAmelCase      ,   __UpperCAmelCase      ,   __UpperCAmelCase  )
																		            for i in range(__UpperCAmelCase  )
																		        ]      ,   dim=0      ,   ).reshape(__UpperCAmelCase      ,   __UpperCAmelCase  )  )
																		__magic_name__:	int							=       permute(
																		    torch.cat(
																		        [
																		            loaded[i][f'layers.{layer_i}.attention.wk.weight'].view(
																		                __UpperCAmelCase      ,   __UpperCAmelCase      ,   __UpperCAmelCase  )
																		            for i in range(__UpperCAmelCase  )
																		        ]      ,   dim=0      ,   ).reshape(__UpperCAmelCase      ,   __UpperCAmelCase  )      ,   __UpperCAmelCase      ,   __UpperCAmelCase      ,   __UpperCAmelCase      ,   )
																		__magic_name__:	Dict							=       torch.cat(
																		    [
																		        loaded[i][f'layers.{layer_i}.attention.wv.weight'].view(
																		            __UpperCAmelCase      ,   __UpperCAmelCase      ,   __UpperCAmelCase  )
																		        for i in range(__UpperCAmelCase  )
																		    ]      ,   dim=0      ,   ).reshape(__UpperCAmelCase      ,   __UpperCAmelCase  )

																		__magic_name__:	Tuple							=       torch.cat(
																		    [loaded[i][f'layers.{layer_i}.attention.wo.weight'] for i in range(__UpperCAmelCase  )]      ,   dim=1  )
																		__magic_name__:	Optional[int]							=       torch.cat(
																		    [loaded[i][f'layers.{layer_i}.feed_forward.w1.weight'] for i in range(__UpperCAmelCase  )]      ,   dim=0  )
																		__magic_name__:	str							=       torch.cat(
																		    [loaded[i][f'layers.{layer_i}.feed_forward.w2.weight'] for i in range(__UpperCAmelCase  )]      ,   dim=1  )
																		__magic_name__:	str							=       torch.cat(
																		    [loaded[i][f'layers.{layer_i}.feed_forward.w3.weight'] for i in range(__UpperCAmelCase  )]      ,   dim=0  )

												__magic_name__:	Union[str, Any]							=       inv_freq
												for k, v in state_dict.items():
																		__magic_name__:	Tuple							=       filename
																		param_count += v.numel()
												torch.save(__UpperCAmelCase      ,   os.path.join(__UpperCAmelCase      ,   __UpperCAmelCase  )  )

						__magic_name__:	str							=       f'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'
						if model_size == "7B":
												# Unsharded
												__magic_name__:	List[Any]							=       {
												    """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
												    """model.norm.weight""": loaded["""norm.weight"""],
												    """lm_head.weight""": loaded["""output.weight"""],
												}
						else:
												__magic_name__:	int							=       {
												    """model.norm.weight""": loaded[0]["""norm.weight"""],
												    """model.embed_tokens.weight""": torch.cat(
												        [loaded[i]["""tok_embeddings.weight"""] for i in range(__UpperCAmelCase  )]      ,   dim=1  ),
												    """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(__UpperCAmelCase  )]      ,   dim=0  ),
												}

						for k, v in state_dict.items():
												__magic_name__:	Tuple							=       filename
												param_count += v.numel()
						torch.save(__UpperCAmelCase      ,   os.path.join(__UpperCAmelCase      ,   __UpperCAmelCase  )  )

						# Write configs
						__magic_name__:	Optional[Any]							=       {"""total_size""": param_count * 2}
						write_json(__UpperCAmelCase      ,   os.path.join(__UpperCAmelCase      ,   """pytorch_model.bin.index.json"""  )  )
						__magic_name__:	Tuple							=       params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
						__magic_name__:	Union[str, Any]							=       params["""multiple_of"""] if """multiple_of""" in params else 2_5_6
						__magic_name__:	Optional[int]							=       LlamaConfig(
						    hidden_size=__UpperCAmelCase      ,   intermediate_size=compute_intermediate_size(__UpperCAmelCase      ,   __UpperCAmelCase      ,   __UpperCAmelCase  )      ,   num_attention_heads=params["""n_heads"""]      ,   num_hidden_layers=params["""n_layers"""]      ,   rms_norm_eps=params["""norm_eps"""]      ,   num_key_value_heads=__UpperCAmelCase      ,   )
						config.save_pretrained(__UpperCAmelCase  )

						# Make space so we can load the model properly now.
						del state_dict
						del loaded
						gc.collect()

						print("""Loading the checkpoint in a Llama model."""  )
						__magic_name__:	Dict							=       LlamaForCausalLM.from_pretrained(__UpperCAmelCase      ,   torch_dtype=torch.floataa      ,   low_cpu_mem_usage=__UpperCAmelCase  )
						# Avoid saving this as part of the config.
						del model.config._name_or_path

						print("""Saving in the Transformers format."""  )
						model.save_pretrained(__UpperCAmelCase      ,   safe_serialization=__UpperCAmelCase  )
						shutil.rmtree(__UpperCAmelCase  )



def   a    (     __UpperCAmelCase      :    Dict      ,   __UpperCAmelCase      :    Dict  )    ->							List[Any]:
						# Initialize the tokenizer based on the `spm` model
						__magic_name__:	Optional[int]							=       LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
						print(f'Saving a {tokenizer_class.__name__} to {tokenizer_path}.'  )
						__magic_name__:	Dict							=       tokenizer_class(__UpperCAmelCase  )
						tokenizer.save_pretrained(__UpperCAmelCase  )



def   a    (     )    ->							Any:
						__magic_name__:	List[str]							=       argparse.ArgumentParser()
						parser.add_argument(
						    """--input_dir"""      ,   help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""      ,   )
						parser.add_argument(
						    """--model_size"""      ,   choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]      ,   )
						parser.add_argument(
						    """--output_dir"""      ,   help="""Location to write HF model and tokenizer"""      ,   )
						parser.add_argument("""--safe_serialization"""      ,   type=__UpperCAmelCase      ,   help="""Whether or not to save using `safetensors`."""  )
						__magic_name__:	str							=       parser.parse_args()
						if args.model_size != "tokenizer_only":
												write_model(
												    model_path=args.output_dir      ,   input_base_path=os.path.join(args.input_dir      ,   args.model_size  )      ,   model_size=args.model_size      ,   safe_serialization=args.safe_serialization      ,   )
						__magic_name__:	List[Any]							=       os.path.join(args.input_dir      ,   """tokenizer.model"""  )
						write_tokenizer(args.output_dir      ,   __UpperCAmelCase  )


if __name__ == "__main__":
							main()



