


'''simple docstring'''



import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
		from transformers import LlamaTokenizerFast
except ImportError as e:
		warnings.warn(e)
		warnings.warn(
		    'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
		)
		__A	=None
__A	={
    '7B': 1_10_08,
    '13B': 1_38_24,
    '30B': 1_79_20,
    '65B': 2_20_16,
    '70B': 2_86_72,
}
__A	={
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}







def     _UpperCamelCase      (		UpperCamelCase__ ,      UpperCamelCase__=1 ,      UpperCamelCase__=2_5_6					):
			return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3					)					) + multiple_of - 1) // multiple_of)







def     _UpperCamelCase      (		UpperCamelCase__					):
			with open(UpperCamelCase__ ,      """r"""					) as f:
						return json.load(UpperCamelCase__					)







def     _UpperCamelCase      (		UpperCamelCase__ ,      UpperCamelCase__					):
			with open(UpperCamelCase__ ,      """w"""					) as f:
						json.dump(UpperCamelCase__ ,      UpperCamelCase__					)







def     _UpperCamelCase      (		UpperCamelCase__ ,      UpperCamelCase__ ,      UpperCamelCase__ ,      UpperCamelCase__=True					):
			os.makedirs(UpperCamelCase__ ,      exist_ok=UpperCamelCase__					)
			UpperCAmelCase__  :					str        =     os.path.join(UpperCamelCase__ ,      """tmp"""					)
			os.makedirs(UpperCamelCase__ ,      exist_ok=UpperCamelCase__					)

			UpperCAmelCase__  :					Tuple        =     read_json(os.path.join(UpperCamelCase__ ,      """params.json"""					)					)
			UpperCAmelCase__  :					Union[str, Any]        =     NUM_SHARDS[model_size]
			UpperCAmelCase__  :					Union[str, Any]        =     params["""n_layers"""]
			UpperCAmelCase__  :					Any        =     params["""n_heads"""]
			UpperCAmelCase__  :					int        =     n_heads // num_shards
			UpperCAmelCase__  :					Union[str, Any]        =     params["""dim"""]
			UpperCAmelCase__  :					List[Any]        =     dim // n_heads
			UpperCAmelCase__  :					Union[str, Any]        =     1_00_00.0
			UpperCAmelCase__  :					Union[str, Any]        =     1.0 / (base ** (torch.arange(0 ,      UpperCamelCase__ ,      2					).float() / dims_per_head))

			if "n_kv_heads" in params:
						UpperCAmelCase__  :					Union[str, Any]        =     params["""n_kv_heads"""]  # for GQA / MQA
						UpperCAmelCase__  :					Tuple        =     n_heads_per_shard // num_key_value_heads
						UpperCAmelCase__  :					Any        =     dim // num_key_value_heads
			else:  # compatibility with other checkpoints
						UpperCAmelCase__  :					Optional[int]        =     n_heads
						UpperCAmelCase__  :					Any        =     n_heads_per_shard
						UpperCAmelCase__  :					Union[str, Any]        =     dim

			# permute for sliced rotary
			def permute(UpperCamelCase__ ,      UpperCamelCase__=n_heads ,      UpperCamelCase__=dim ,      UpperCamelCase__=dim					):
						return w.view(UpperCamelCase__ ,      dima // n_heads // 2 ,      2 ,      UpperCamelCase__					).transpose(1 ,      2					).reshape(UpperCamelCase__ ,      UpperCamelCase__					)

			print(f'''Fetching all parameters from the checkpoint at {input_base_path}.'''					)
			# Load weights
			if model_size == "7B":
						# Not sharded
						# (The sharded implementation would also work, but this is simpler.)
						UpperCAmelCase__  :					Optional[int]        =     torch.load(os.path.join(UpperCamelCase__ ,      """consolidated.00.pth"""					) ,      map_location="""cpu"""					)
			else:
						# Sharded
						UpperCAmelCase__  :					Union[str, Any]        =     [
						    torch.load(os.path.join(UpperCamelCase__ ,      f'''consolidated.{i:02d}.pth'''					) ,      map_location="""cpu"""					)
						    for i in range(UpperCamelCase__					)
						]
			UpperCAmelCase__  :					List[Any]        =     0
			UpperCAmelCase__  :					Optional[Any]        =     {"""weight_map""": {}}
			for layer_i in range(UpperCamelCase__					):
						UpperCAmelCase__  :					Any        =     f'''pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'''
						if model_size == "7B":
									# Unsharded
									UpperCAmelCase__  :					Optional[int]        =     {
									    f'''model.layers.{layer_i}.self_attn.q_proj.weight''': permute(
									        loaded[f'''layers.{layer_i}.attention.wq.weight''']					),
									    f'''model.layers.{layer_i}.self_attn.k_proj.weight''': permute(
									        loaded[f'''layers.{layer_i}.attention.wk.weight''']					),
									    f'''model.layers.{layer_i}.self_attn.v_proj.weight''': loaded[f'''layers.{layer_i}.attention.wv.weight'''],
									    f'''model.layers.{layer_i}.self_attn.o_proj.weight''': loaded[f'''layers.{layer_i}.attention.wo.weight'''],
									    f'''model.layers.{layer_i}.mlp.gate_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w1.weight'''],
									    f'''model.layers.{layer_i}.mlp.down_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w2.weight'''],
									    f'''model.layers.{layer_i}.mlp.up_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w3.weight'''],
									    f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[f'''layers.{layer_i}.attention_norm.weight'''],
									    f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[f'''layers.{layer_i}.ffn_norm.weight'''],
									}
						else:
									# Sharded
									# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
									# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
									# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

									UpperCAmelCase__  :					str        =     {
									    f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[0][
									        f'''layers.{layer_i}.attention_norm.weight'''
									    ].clone(),
									    f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[0][
									        f'''layers.{layer_i}.ffn_norm.weight'''
									    ].clone(),
									}
									UpperCAmelCase__  :					List[str]        =     permute(
									    torch.cat(
									        [
									            loaded[i][f'''layers.{layer_i}.attention.wq.weight'''].view(UpperCamelCase__ ,      UpperCamelCase__ ,      UpperCamelCase__					)
									            for i in range(UpperCamelCase__					)
									        ] ,      dim=0 ,      ).reshape(UpperCamelCase__ ,      UpperCamelCase__					)					)
									UpperCAmelCase__  :					str        =     permute(
									    torch.cat(
									        [
									            loaded[i][f'''layers.{layer_i}.attention.wk.weight'''].view(
									                UpperCamelCase__ ,      UpperCamelCase__ ,      UpperCamelCase__					)
									            for i in range(UpperCamelCase__					)
									        ] ,      dim=0 ,      ).reshape(UpperCamelCase__ ,      UpperCamelCase__					) ,      UpperCamelCase__ ,      UpperCamelCase__ ,      UpperCamelCase__ ,      )
									UpperCAmelCase__  :					Optional[Any]        =     torch.cat(
									    [
									        loaded[i][f'''layers.{layer_i}.attention.wv.weight'''].view(
									            UpperCamelCase__ ,      UpperCamelCase__ ,      UpperCamelCase__					)
									        for i in range(UpperCamelCase__					)
									    ] ,      dim=0 ,      ).reshape(UpperCamelCase__ ,      UpperCamelCase__					)

									UpperCAmelCase__  :					Tuple        =     torch.cat(
									    [loaded[i][f'''layers.{layer_i}.attention.wo.weight'''] for i in range(UpperCamelCase__					)] ,      dim=1					)
									UpperCAmelCase__  :					Any        =     torch.cat(
									    [loaded[i][f'''layers.{layer_i}.feed_forward.w1.weight'''] for i in range(UpperCamelCase__					)] ,      dim=0					)
									UpperCAmelCase__  :					Any        =     torch.cat(
									    [loaded[i][f'''layers.{layer_i}.feed_forward.w2.weight'''] for i in range(UpperCamelCase__					)] ,      dim=1					)
									UpperCAmelCase__  :					Optional[int]        =     torch.cat(
									    [loaded[i][f'''layers.{layer_i}.feed_forward.w3.weight'''] for i in range(UpperCamelCase__					)] ,      dim=0					)

						UpperCAmelCase__  :					Dict        =     inv_freq
						for k, v in state_dict.items():
									UpperCAmelCase__  :					Tuple        =     filename
									param_count += v.numel()
						torch.save(UpperCamelCase__ ,      os.path.join(UpperCamelCase__ ,      UpperCamelCase__					)					)

			UpperCAmelCase__  :					Optional[Any]        =     f'''pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'''
			if model_size == "7B":
						# Unsharded
						UpperCAmelCase__  :					Union[str, Any]        =     {
						    """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
						    """model.norm.weight""": loaded["""norm.weight"""],
						    """lm_head.weight""": loaded["""output.weight"""],
						}
			else:
						UpperCAmelCase__  :					int        =     {
						    """model.norm.weight""": loaded[0]["""norm.weight"""],
						    """model.embed_tokens.weight""": torch.cat(
						        [loaded[i]["""tok_embeddings.weight"""] for i in range(UpperCamelCase__					)] ,      dim=1					),
						    """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(UpperCamelCase__					)] ,      dim=0					),
						}

			for k, v in state_dict.items():
						UpperCAmelCase__  :					Optional[Any]        =     filename
						param_count += v.numel()
			torch.save(UpperCamelCase__ ,      os.path.join(UpperCamelCase__ ,      UpperCamelCase__					)					)

			# Write configs
			UpperCAmelCase__  :					Optional[int]        =     {"""total_size""": param_count * 2}
			write_json(UpperCamelCase__ ,      os.path.join(UpperCamelCase__ ,      """pytorch_model.bin.index.json"""					)					)
			UpperCAmelCase__  :					Any        =     params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
			UpperCAmelCase__  :					Tuple        =     params["""multiple_of"""] if """multiple_of""" in params else 2_5_6
			UpperCAmelCase__  :					int        =     LlamaConfig(
			    hidden_size=UpperCamelCase__ ,      intermediate_size=compute_intermediate_size(UpperCamelCase__ ,      UpperCamelCase__ ,      UpperCamelCase__					) ,      num_attention_heads=params["""n_heads"""] ,      num_hidden_layers=params["""n_layers"""] ,      rms_norm_eps=params["""norm_eps"""] ,      num_key_value_heads=UpperCamelCase__ ,      )
			config.save_pretrained(UpperCamelCase__					)

			# Make space so we can load the model properly now.
			del state_dict
			del loaded
			gc.collect()

			print("""Loading the checkpoint in a Llama model."""					)
			UpperCAmelCase__  :					Optional[Any]        =     LlamaForCausalLM.from_pretrained(UpperCamelCase__ ,      torch_dtype=torch.floataa ,      low_cpu_mem_usage=UpperCamelCase__					)
			# Avoid saving this as part of the config.
			del model.config._name_or_path

			print("""Saving in the Transformers format."""					)
			model.save_pretrained(UpperCamelCase__ ,      safe_serialization=UpperCamelCase__					)
			shutil.rmtree(UpperCamelCase__					)







def     _UpperCamelCase      (		UpperCamelCase__ ,      UpperCamelCase__					):
			# Initialize the tokenizer based on the `spm` model
			UpperCAmelCase__  :					Tuple        =     LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
			print(f'''Saving a {tokenizer_class.__name__} to {tokenizer_path}.'''					)
			UpperCAmelCase__  :					Dict        =     tokenizer_class(UpperCamelCase__					)
			tokenizer.save_pretrained(UpperCamelCase__					)







def     _UpperCamelCase      (		):
			UpperCAmelCase__  :					Any        =     argparse.ArgumentParser()
			parser.add_argument(
			    """--input_dir""" ,      help="""Location of LLaMA weights, which contains tokenizer.model and model folders""" ,      )
			parser.add_argument(
			    """--model_size""" ,      choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""] ,      )
			parser.add_argument(
			    """--output_dir""" ,      help="""Location to write HF model and tokenizer""" ,      )
			parser.add_argument("""--safe_serialization""" ,      type=UpperCamelCase__ ,      help="""Whether or not to save using `safetensors`."""					)
			UpperCAmelCase__  :					List[Any]        =     parser.parse_args()
			if args.model_size != "tokenizer_only":
						write_model(
						    model_path=args.output_dir ,      input_base_path=os.path.join(args.input_dir ,      args.model_size					) ,      model_size=args.model_size ,      safe_serialization=args.safe_serialization ,      )
			UpperCAmelCase__  :					Optional[int]        =     os.path.join(args.input_dir ,      """tokenizer.model"""					)
			write_tokenizer(args.output_dir ,      UpperCamelCase__					)


if __name__ == "__main__":
		main()



'''simple docstring'''



import importlib
import os
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, Optional, Union

import torch

from ..utils import BaseOutput


__A	='scheduler_config.json'


class 		_snake_case					(     a__	):
					lowerCAmelCase      :Union[str, Any]   =					1
					lowerCAmelCase      :Union[str, Any]   =					2
					lowerCAmelCase      :Optional[int]   =					3
					lowerCAmelCase      :Optional[int]   =					4
					lowerCAmelCase      :Any   =					5
					lowerCAmelCase      :Tuple   =					6
					lowerCAmelCase      :List[Any]   =					7
					lowerCAmelCase      :str   =					8
					lowerCAmelCase      :List[Any]   =					9
					lowerCAmelCase      :List[str]   =					10
					lowerCAmelCase      :Union[str, Any]   =					11
					lowerCAmelCase      :Optional[int]   =					12
					lowerCAmelCase      :str   =					13
					lowerCAmelCase      :Dict   =					14


@dataclass
class 		_snake_case					(     a__	):
					lowerCAmelCase      :torch.FloatTensor


class 		_snake_case					:
					lowerCAmelCase      :str   =					SCHEDULER_CONFIG_NAME
					lowerCAmelCase      :Union[str, Any]   =					[]
					lowerCAmelCase      :List[str]   =					True




					@classmethod
					def 				snake_case__						(      cls							,							_lowerCamelCase = None							,							_lowerCamelCase = None							,							_lowerCamelCase=False							,							**_lowerCamelCase							,							):
								UpperCAmelCase__   ,       UpperCAmelCase__   ,       UpperCAmelCase__  :					Optional[int]        =     cls.load_config(
								    pretrained_model_name_or_path=_lowerCamelCase							,							subfolder=_lowerCamelCase							,							return_unused_kwargs=_lowerCamelCase							,							return_commit_hash=_lowerCamelCase							,							**_lowerCamelCase							,							)
								return cls.from_config(_lowerCamelCase							,							return_unused_kwargs=_lowerCamelCase							,							**_lowerCamelCase)




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase = False							,							**_lowerCamelCase):
								self.save_config(save_directory=_lowerCamelCase							,							push_to_hub=_lowerCamelCase							,							**_lowerCamelCase)




					@property
					def 				snake_case__						(      self):
								return self._get_compatibles()




					@classmethod
					def 				snake_case__						(      cls):
								UpperCAmelCase__  :					List[Any]        =     list(set([cls.__name__] + cls._compatibles))
								UpperCAmelCase__  :					int        =     importlib.import_module(__name__.split(""".""")[0])
								UpperCAmelCase__  :					Dict        =     [
								    getattr(_lowerCamelCase							,							_lowerCamelCase) for c in compatible_classes_str if hasattr(_lowerCamelCase							,							_lowerCamelCase)
								]
								return compatible_classes