






"""simple docstring"""

import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
					from transformers import LlamaTokenizerFast
except ImportError as e:
					warnings.warn(e)
					warnings.warn(
					    """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
					)
					lowercase__  			=					None
lowercase__  			=					{
    """7B""": 11008,
    """13B""": 13824,
    """30B""": 17920,
    """65B""": 22016,
    """70B""": 28672,
}
lowercase__  			=					{
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}

def 						__lowerCamelCase      (    __UpperCamelCase ,			__UpperCamelCase=1 ,			__UpperCamelCase=256			)		->       Union[str, Any]:

		"""simple docstring"""







		return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3			)			) + multiple_of - 1) // multiple_of)

def 						__lowerCamelCase      (    __UpperCamelCase			)		->       Dict:

		"""simple docstring"""







		with open(__UpperCamelCase ,			"r"			) as f:
				return json.load(__UpperCamelCase			)

def 						__lowerCamelCase      (    __UpperCamelCase ,			__UpperCamelCase			)		->       List[Any]:

		"""simple docstring"""







		with open(__UpperCamelCase ,			"w"			) as f:
				json.dump(__UpperCamelCase ,			__UpperCamelCase			)

def 						__lowerCamelCase      (    __UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase=True			)		->       Tuple:

		"""simple docstring"""







		os.makedirs(__UpperCamelCase ,			exist_ok=__UpperCamelCase			)
		lowerCAmelCase_  :				Tuple   							=     os.path.join(__UpperCamelCase ,			"tmp"			)
		os.makedirs(__UpperCamelCase ,			exist_ok=__UpperCamelCase			)

		lowerCAmelCase_  :				int   							=     read_json(os.path.join(__UpperCamelCase ,			"params.json"			)			)
		lowerCAmelCase_  :				List[str]   							=     NUM_SHARDS[model_size]
		lowerCAmelCase_  :				Tuple   							=     params['''n_layers''']
		lowerCAmelCase_  :				Union[str, Any]   							=     params['''n_heads''']
		lowerCAmelCase_  :				Union[str, Any]   							=     n_heads // num_shards
		lowerCAmelCase_  :				Dict   							=     params['''dim''']
		lowerCAmelCase_  :				Tuple   							=     dim // n_heads
		lowerCAmelCase_  :				str   							=     10000.0
		lowerCAmelCase_  :				Union[str, Any]   							=     1.0 / (base ** (torch.arange(0 ,			__UpperCamelCase ,			2			).float() / dims_per_head))

		if "n_kv_heads" in params:
				lowerCAmelCase_  :				Any   							=     params['''n_kv_heads''']  # for GQA / MQA
				lowerCAmelCase_  :				Union[str, Any]   							=     n_heads_per_shard // num_key_value_heads
				lowerCAmelCase_  :				int   							=     dim // num_key_value_heads
		else:  # compatibility with other checkpoints
				lowerCAmelCase_  :				Union[str, Any]   							=     n_heads
				lowerCAmelCase_  :				Any   							=     n_heads_per_shard
				lowerCAmelCase_  :				int   							=     dim

		# permute for sliced rotary
		def permute(__UpperCamelCase ,			__UpperCamelCase=n_heads ,			__UpperCamelCase=dim ,			__UpperCamelCase=dim			):
				return w.view(__UpperCamelCase ,			dima // n_heads // 2 ,			2 ,			__UpperCamelCase			).transpose(1 ,			2			).reshape(__UpperCamelCase ,			__UpperCamelCase			)

		print(f'''Fetching all parameters from the checkpoint at {input_base_path}.'''			)
		# Load weights
		if model_size == "7B":
				# Not sharded
				# (The sharded implementation would also work, but this is simpler.)
				lowerCAmelCase_  :				List[Any]   							=     torch.load(os.path.join(__UpperCamelCase ,			"consolidated.00.pth"			) ,			map_location="cpu"			)
		else:
				# Sharded
				lowerCAmelCase_  :				Optional[Any]   							=     [
				    torch.load(os.path.join(__UpperCamelCase ,			f'''consolidated.{i:02d}.pth'''			) ,			map_location="cpu"			)
				    for i in range(__UpperCamelCase			)
				]
		lowerCAmelCase_  :				List[str]   							=     0
		lowerCAmelCase_  :				Optional[int]   							=     {'''weight_map''': {}}
		for layer_i in range(__UpperCamelCase			):
				lowerCAmelCase_  :				Any   							=     f'''pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'''
				if model_size == "7B":
						# Unsharded
						lowerCAmelCase_  :				List[str]   							=     {
						    f'''model.layers.{layer_i}.self_attn.q_proj.weight''': permute(
						        loaded[f'''layers.{layer_i}.attention.wq.weight''']			),
						    f'''model.layers.{layer_i}.self_attn.k_proj.weight''': permute(
						        loaded[f'''layers.{layer_i}.attention.wk.weight''']			),
						    f'''model.layers.{layer_i}.self_attn.v_proj.weight''': loaded[f'''layers.{layer_i}.attention.wv.weight'''],
						    f'''model.layers.{layer_i}.self_attn.o_proj.weight''': loaded[f'''layers.{layer_i}.attention.wo.weight'''],
						    f'''model.layers.{layer_i}.mlp.gate_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w1.weight'''],
						    f'''model.layers.{layer_i}.mlp.down_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w2.weight'''],
						    f'''model.layers.{layer_i}.mlp.up_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w3.weight'''],
						    f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[f'''layers.{layer_i}.attention_norm.weight'''],
						    f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[f'''layers.{layer_i}.ffn_norm.weight'''],
						}
				else:
						# Sharded
						# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
						# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
						# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

						lowerCAmelCase_  :				Any   							=     {
						    f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[0][
						        f'''layers.{layer_i}.attention_norm.weight'''
						    ].clone(),
						    f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[0][
						        f'''layers.{layer_i}.ffn_norm.weight'''
						    ].clone(),
						}
						lowerCAmelCase_  :				Dict   							=     permute(
						    torch.cat(
						        [
						            loaded[i][f'''layers.{layer_i}.attention.wq.weight'''].view(__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase			)
						            for i in range(__UpperCamelCase			)
						        ] ,			dim=0 ,			).reshape(__UpperCamelCase ,			__UpperCamelCase			)			)
						lowerCAmelCase_  :				Optional[int]   							=     permute(
						    torch.cat(
						        [
						            loaded[i][f'''layers.{layer_i}.attention.wk.weight'''].view(
						                __UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase			)
						            for i in range(__UpperCamelCase			)
						        ] ,			dim=0 ,			).reshape(__UpperCamelCase ,			__UpperCamelCase			) ,			__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase ,			)
						lowerCAmelCase_  :				Any   							=     torch.cat(
						    [
						        loaded[i][f'''layers.{layer_i}.attention.wv.weight'''].view(
						            __UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase			)
						        for i in range(__UpperCamelCase			)
						    ] ,			dim=0 ,			).reshape(__UpperCamelCase ,			__UpperCamelCase			)

						lowerCAmelCase_  :				Optional[Any]   							=     torch.cat(
						    [loaded[i][f'''layers.{layer_i}.attention.wo.weight'''] for i in range(__UpperCamelCase			)] ,			dim=1			)
						lowerCAmelCase_  :				Union[str, Any]   							=     torch.cat(
						    [loaded[i][f'''layers.{layer_i}.feed_forward.w1.weight'''] for i in range(__UpperCamelCase			)] ,			dim=0			)
						lowerCAmelCase_  :				Union[str, Any]   							=     torch.cat(
						    [loaded[i][f'''layers.{layer_i}.feed_forward.w2.weight'''] for i in range(__UpperCamelCase			)] ,			dim=1			)
						lowerCAmelCase_  :				Any   							=     torch.cat(
						    [loaded[i][f'''layers.{layer_i}.feed_forward.w3.weight'''] for i in range(__UpperCamelCase			)] ,			dim=0			)

				lowerCAmelCase_  :				List[str]   							=     inv_freq
				for k, v in state_dict.items():
						lowerCAmelCase_  :				Union[str, Any]   							=     filename
						param_count += v.numel()
				torch.save(__UpperCamelCase ,			os.path.join(__UpperCamelCase ,			__UpperCamelCase			)			)

		lowerCAmelCase_  :				List[Any]   							=     f'''pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'''
		if model_size == "7B":
				# Unsharded
				lowerCAmelCase_  :				Any   							=     {
				    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
				    '''model.norm.weight''': loaded['''norm.weight'''],
				    '''lm_head.weight''': loaded['''output.weight'''],
				}
		else:
				lowerCAmelCase_  :				int   							=     {
				    '''model.norm.weight''': loaded[0]['''norm.weight'''],
				    '''model.embed_tokens.weight''': torch.cat(
				        [loaded[i]["tok_embeddings.weight"] for i in range(__UpperCamelCase			)] ,			dim=1			),
				    '''lm_head.weight''': torch.cat([loaded[i]["output.weight"] for i in range(__UpperCamelCase			)] ,			dim=0			),
				}

		for k, v in state_dict.items():
				lowerCAmelCase_  :				Optional[int]   							=     filename
				param_count += v.numel()
		torch.save(__UpperCamelCase ,			os.path.join(__UpperCamelCase ,			__UpperCamelCase			)			)

		# Write configs
		lowerCAmelCase_  :				Optional[int]   							=     {'''total_size''': param_count * 2}
		write_json(__UpperCamelCase ,			os.path.join(__UpperCamelCase ,			"pytorch_model.bin.index.json"			)			)
		lowerCAmelCase_  :				Optional[int]   							=     params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
		lowerCAmelCase_  :				List[Any]   							=     params['''multiple_of'''] if '''multiple_of''' in params else 256
		lowerCAmelCase_  :				Optional[Any]   							=     LlamaConfig(
		    hidden_size=__UpperCamelCase ,			intermediate_size=compute_intermediate_size(__UpperCamelCase ,			__UpperCamelCase ,			__UpperCamelCase			) ,			num_attention_heads=params["n_heads"] ,			num_hidden_layers=params["n_layers"] ,			rms_norm_eps=params["norm_eps"] ,			num_key_value_heads=__UpperCamelCase ,			)
		config.save_pretrained(__UpperCamelCase			)

		# Make space so we can load the model properly now.
		del state_dict
		del loaded
		gc.collect()

		print("Loading the checkpoint in a Llama model."			)
		lowerCAmelCase_  :				Dict   							=     LlamaForCausalLM.from_pretrained(__UpperCamelCase ,			torch_dtype=torch.floataa ,			low_cpu_mem_usage=__UpperCamelCase			)
		# Avoid saving this as part of the config.
		del model.config._name_or_path

		print("Saving in the Transformers format."			)
		model.save_pretrained(__UpperCamelCase ,			safe_serialization=__UpperCamelCase			)
		shutil.rmtree(__UpperCamelCase			)

def 						__lowerCamelCase      (    __UpperCamelCase ,			__UpperCamelCase			)		->       List[str]:

		"""simple docstring"""







		lowerCAmelCase_  :				List[Any]   							=     LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
		print(f'''Saving a {tokenizer_class.__name__} to {tokenizer_path}.'''			)
		lowerCAmelCase_  :				Optional[Any]   							=     tokenizer_class(__UpperCamelCase			)
		tokenizer.save_pretrained(__UpperCamelCase			)

def 						__lowerCamelCase      (    )		->       Any:

		"""simple docstring"""







		lowerCAmelCase_  :				int   							=     argparse.ArgumentParser()
		parser.add_argument(
		    "--input_dir" ,			help="Location of LLaMA weights, which contains tokenizer.model and model folders" ,			)
		parser.add_argument(
		    "--model_size" ,			choices=["7B", "7Bf", "13B", "13Bf", "30B", "65B", "70B", "70Bf", "tokenizer_only"] ,			)
		parser.add_argument(
		    "--output_dir" ,			help="Location to write HF model and tokenizer" ,			)
		parser.add_argument("--safe_serialization" ,			type=__UpperCamelCase ,			help="Whether or not to save using `safetensors`."			)
		lowerCAmelCase_  :				Optional[Any]   							=     parser.parse_args()
		if args.model_size != "tokenizer_only":
				write_model(
				    model_path=args.output_dir ,			input_base_path=os.path.join(args.input_dir ,			args.model_size			) ,			model_size=args.model_size ,			safe_serialization=args.safe_serialization ,			)
		lowerCAmelCase_  :				Any   							=     os.path.join(args.input_dir ,			"tokenizer.model"			)
		write_tokenizer(args.output_dir ,			__UpperCamelCase			)


if __name__ == "__main__":
					main()







"""simple docstring"""





from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
							import PIL


_UpperCamelCase					      =   logging.get_logger(__name__)
class 		SCREAMING_SNAKE_CASE_				(     snake_case__ ):
				"""simple docstring"""
				__snake_case :							str									=		["""pixel_values"""]




				def __init__(  self	:str    ,     __lowercase	:bool = True    ,     __lowercase	:Dict[str, int] = None    ,     __lowercase	:float = None    ,     __lowercase	:PILImageResampling = PILImageResampling.BILINEAR    ,     __lowercase	:bool = True    ,     __lowercase	:Union[int, float] = 1 / 255    ,     __lowercase	:bool = True    ,     __lowercase	:Optional[Union[float, List[float]]] = None    ,     __lowercase	:Optional[Union[float, List[float]]] = None    ,     **__lowercase	:Tuple    ,     ):
											super().__init__(**__lowercase      )
											__lowerCamelCase   :   int       =size if size is not None else {'''shortest_edge''': 384}
											__lowerCamelCase   :   List[str]       =get_size_dict(__lowercase    ,     default_to_square=__lowercase      )

											__lowerCamelCase   :   Any       =do_resize
											__lowerCamelCase   :   List[str]       =size
											# Default value set here for backwards compatibility where the value in config is None
											__lowerCamelCase   :   Tuple       =crop_pct if crop_pct is not None else 224 / 256
											__lowerCamelCase   :   Any       =resample
											__lowerCamelCase   :   List[str]       =do_rescale
											__lowerCamelCase   :   Dict       =rescale_factor
											__lowerCamelCase   :   Any       =do_normalize
											__lowerCamelCase   :   int       =image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
											__lowerCamelCase   :   List[Any]       =image_std if image_std is not None else IMAGENET_STANDARD_STD




				def        __lowercase	(  self	:Any    ,     __lowercase	:np.ndarray    ,     __lowercase	:Dict[str, int]    ,     __lowercase	:float    ,     __lowercase	:PILImageResampling = PILImageResampling.BICUBIC    ,     __lowercase	:Optional[Union[str, ChannelDimension]] = None    ,     **__lowercase	:int    ,     ):
											__lowerCamelCase   :   int       =get_size_dict(__lowercase    ,     default_to_square=__lowercase      )
											if "shortest_edge" not in size:
																		raise ValueError(f'Size dictionary must contain \'shortest_edge\' key. Got {size.keys()}'      )
											__lowerCamelCase   :   Optional[int]       =size['''shortest_edge''']

											if shortest_edge < 384:
																		# maintain same ratio, resizing shortest edge to shortest_edge/crop_pct
																		__lowerCamelCase   :   int       =int(shortest_edge / crop_pct      )
																		__lowerCamelCase   :   str       =get_resize_output_image_size(__lowercase    ,     size=__lowercase    ,     default_to_square=__lowercase      )
																		__lowerCamelCase   :   str       =resize(image=__lowercase    ,     size=__lowercase    ,     resample=__lowercase    ,     data_format=__lowercase    ,     **__lowercase      )
																		# then crop to (shortest_edge, shortest_edge)
																		return center_crop(image=__lowercase    ,     size=(shortest_edge, shortest_edge)    ,     data_format=__lowercase    ,     **__lowercase      )
											else:
																		# warping (no cropping) when evaluated at 384 or larger
																		return resize(
																		    __lowercase    ,     size=(shortest_edge, shortest_edge)    ,     resample=__lowercase    ,     data_format=__lowercase    ,     **__lowercase      )




				def        __lowercase	(  self	:int    ,     __lowercase	:np.ndarray    ,     __lowercase	:Union[int, float]    ,     __lowercase	:Optional[Union[str, ChannelDimension]] = None    ,     **__lowercase	:List[Any]    ,     ):
											return rescale(__lowercase    ,     scale=__lowercase    ,     data_format=__lowercase    ,     **__lowercase      )




				def        __lowercase	(  self	:str    ,     __lowercase	:np.ndarray    ,     __lowercase	:Union[float, List[float]]    ,     __lowercase	:Union[float, List[float]]    ,     __lowercase	:Optional[Union[str, ChannelDimension]] = None    ,     **__lowercase	:Optional[int]    ,     ):
											return normalize(__lowercase    ,     mean=__lowercase    ,     std=__lowercase    ,     data_format=__lowercase    ,     **__lowercase      )




				def        __lowercase	(  self	:int    ,     __lowercase	:ImageInput    ,     __lowercase	:bool = None    ,     __lowercase	:Dict[str, int] = None    ,     __lowercase	:float = None    ,     __lowercase	:PILImageResampling = None    ,     __lowercase	:bool = None    ,     __lowercase	:float = None    ,     __lowercase	:bool = None    ,     __lowercase	:Optional[Union[float, List[float]]] = None    ,     __lowercase	:Optional[Union[float, List[float]]] = None    ,     __lowercase	:Optional[Union[str, TensorType]] = None    ,     __lowercase	:ChannelDimension = ChannelDimension.FIRST    ,     **__lowercase	:Optional[Any]    ,     ):
											__lowerCamelCase   :   Optional[Any]       =do_resize if do_resize is not None else self.do_resize
											__lowerCamelCase   :   int       =crop_pct if crop_pct is not None else self.crop_pct
											__lowerCamelCase   :   List[Any]       =resample if resample is not None else self.resample
											__lowerCamelCase   :   List[Any]       =do_rescale if do_rescale is not None else self.do_rescale
											__lowerCamelCase   :   List[Any]       =rescale_factor if rescale_factor is not None else self.rescale_factor
											__lowerCamelCase   :   List[Any]       =do_normalize if do_normalize is not None else self.do_normalize
											__lowerCamelCase   :   Optional[Any]       =image_mean if image_mean is not None else self.image_mean
											__lowerCamelCase   :   str       =image_std if image_std is not None else self.image_std

											__lowerCamelCase   :   str       =size if size is not None else self.size
											__lowerCamelCase   :   Optional[int]       =get_size_dict(__lowercase    ,     default_to_square=__lowercase      )

											__lowerCamelCase   :   Optional[int]       =make_list_of_images(__lowercase      )

											if not valid_images(__lowercase      ):
																		raise ValueError(
																		    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
																		    '''torch.Tensor, tf.Tensor or jax.ndarray.'''      )

											if do_resize and size is None or resample is None:
																		raise ValueError('''Size and resample must be specified if do_resize is True.'''      )

											if do_resize and size["shortest_edge"] < 384 and crop_pct is None:
																		raise ValueError('''crop_pct must be specified if size < 384.'''      )

											if do_rescale and rescale_factor is None:
																		raise ValueError('''Rescale factor must be specified if do_rescale is True.'''      )

											if do_normalize and (image_mean is None or image_std is None):
																		raise ValueError('''Image mean and std must be specified if do_normalize is True.'''      )

											# All transformations expect numpy arrays.
											__lowerCamelCase   :   List[Any]       =[to_numpy_array(__lowercase      ) for image in images]

											if do_resize:
																		__lowerCamelCase   :   Union[str, Any]       =[self.resize(image=__lowercase    ,     size=__lowercase    ,     crop_pct=__lowercase    ,     resample=__lowercase      ) for image in images]

											if do_rescale:
																		__lowerCamelCase   :   int       =[self.rescale(image=__lowercase    ,     scale=__lowercase      ) for image in images]

											if do_normalize:
																		__lowerCamelCase   :   Union[str, Any]       =[self.normalize(image=__lowercase    ,     mean=__lowercase    ,     std=__lowercase      ) for image in images]

											__lowerCamelCase   :   Union[str, Any]       =[to_channel_dimension_format(__lowercase    ,     __lowercase      ) for image in images]

											__lowerCamelCase   :   Tuple       ={'''pixel_values''': images}
											return BatchFeature(data=__lowercase    ,     tensor_type=__lowercase      )




