





"""simple docstring"""






import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
		from transformers import LlamaTokenizerFast
except ImportError as e:
		warnings.warn(e)
		warnings.warn(
		    '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
		)
		A 			=  None
A 			=  {
    '''7B''': 11_008,
    '''13B''': 13_824,
    '''30B''': 17_920,
    '''65B''': 22_016,
    '''70B''': 28_672,
}
A 			=  {
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}

def     __A      (			a_						:Dict   ,       a_						:str=1   ,       a_						:List[str]=2_56)		->		Optional[int]:
				return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3)) + multiple_of - 1) // multiple_of)

def     __A      (			a_						:List[str])		->		Optional[int]:
				with open(a_   ,       '''r''') as f:
								return json.load(a_)

def     __A      (			a_						:List[Any]   ,       a_						:List[Any])		->		str:
				with open(a_   ,       '''w''') as f:
								json.dump(a_   ,       a_)

def     __A      (			a_						:int   ,       a_						:Dict   ,       a_						:List[Any]   ,       a_						:Any=True)		->		Tuple:
				os.makedirs(a_   ,       exist_ok=a_)
				__a :						Any			       =  os.path.join(a_   ,       '''tmp''')
				os.makedirs(a_   ,       exist_ok=a_)

				__a :						Union[str, Any]			       =  read_json(os.path.join(a_   ,       '''params.json'''))
				__a :						List[str]			       =  NUM_SHARDS[model_size]
				__a :						Union[str, Any]			       =  params['''n_layers''']
				__a :						List[Any]			       =  params['''n_heads''']
				__a :						Any			       =  n_heads // num_shards
				__a :						Tuple			       =  params['''dim''']
				__a :						List[Any]			       =  dim // n_heads
				__a :						List[Any]			       =  1_0_0_0_0.0
				__a :						List[Any]			       =  1.0 / (base ** (torch.arange(0   ,       a_   ,       2).float() / dims_per_head))

				if "n_kv_heads" in params:
								__a :						int			       =  params['''n_kv_heads''']  # for GQA / MQA
								__a :						Optional[Any]			       =  n_heads_per_shard // num_key_value_heads
								__a :						Any			       =  dim // num_key_value_heads
				else:  # compatibility with other checkpoints
								__a :						List[Any]			       =  n_heads
								__a :						int			       =  n_heads_per_shard
								__a :						Union[str, Any]			       =  dim

				# permute for sliced rotary
				def permute(a_						:Any   ,       a_						:Optional[int]=n_heads   ,       a_						:Any=dim   ,       a_						:int=dim):
								return w.view(a_   ,       dima // n_heads // 2   ,       2   ,       a_).transpose(1   ,       2).reshape(a_   ,       a_)

				print(F"""Fetching all parameters from the checkpoint at {input_base_path}.""")
				# Load weights
				if model_size == "7B":
								# Not sharded
								# (The sharded implementation would also work, but this is simpler.)
								__a :						Any			       =  torch.load(os.path.join(a_   ,       '''consolidated.00.pth''')   ,       map_location='''cpu''')
				else:
								# Sharded
								__a :						Dict			       =  [
								    torch.load(os.path.join(a_   ,       F"""consolidated.{i:02d}.pth""")   ,       map_location='''cpu''')
								    for i in range(a_)
								]
				__a :						Tuple			       =  0
				__a :						Any			       =  {'''weight_map''': {}}
				for layer_i in range(a_):
								__a :						Optional[int]			       =  F"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
								if model_size == "7B":
												# Unsharded
												__a :						Tuple			       =  {
												    F"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
												        loaded[F"""layers.{layer_i}.attention.wq.weight"""]),
												    F"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
												        loaded[F"""layers.{layer_i}.attention.wk.weight"""]),
												    F"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[F"""layers.{layer_i}.attention.wv.weight"""],
												    F"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[F"""layers.{layer_i}.attention.wo.weight"""],
												    F"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w1.weight"""],
												    F"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w2.weight"""],
												    F"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w3.weight"""],
												    F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[F"""layers.{layer_i}.attention_norm.weight"""],
												    F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[F"""layers.{layer_i}.ffn_norm.weight"""],
												}
								else:
												# Sharded
												# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
												# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
												# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

												__a :						int			       =  {
												    F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
												        F"""layers.{layer_i}.attention_norm.weight"""
												    ].clone(),
												    F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
												        F"""layers.{layer_i}.ffn_norm.weight"""
												    ].clone(),
												}
												__a :						Union[str, Any]			       =  permute(
												    torch.cat(
												        [
												            loaded[i][F"""layers.{layer_i}.attention.wq.weight"""].view(a_   ,       a_   ,       a_)
												            for i in range(a_)
												        ]   ,       dim=0   ,       ).reshape(a_   ,       a_))
												__a :						Optional[Any]			       =  permute(
												    torch.cat(
												        [
												            loaded[i][F"""layers.{layer_i}.attention.wk.weight"""].view(
												                a_   ,       a_   ,       a_)
												            for i in range(a_)
												        ]   ,       dim=0   ,       ).reshape(a_   ,       a_)   ,       a_   ,       a_   ,       a_   ,       )
												__a :						Dict			       =  torch.cat(
												    [
												        loaded[i][F"""layers.{layer_i}.attention.wv.weight"""].view(
												            a_   ,       a_   ,       a_)
												        for i in range(a_)
												    ]   ,       dim=0   ,       ).reshape(a_   ,       a_)

												__a :						Dict			       =  torch.cat(
												    [loaded[i][F"""layers.{layer_i}.attention.wo.weight"""] for i in range(a_)]   ,       dim=1)
												__a :						List[Any]			       =  torch.cat(
												    [loaded[i][F"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(a_)]   ,       dim=0)
												__a :						List[Any]			       =  torch.cat(
												    [loaded[i][F"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(a_)]   ,       dim=1)
												__a :						List[str]			       =  torch.cat(
												    [loaded[i][F"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(a_)]   ,       dim=0)

								__a :						Union[str, Any]			       =  inv_freq
								for k, v in state_dict.items():
												__a :						List[Any]			       =  filename
												param_count += v.numel()
								torch.save(a_   ,       os.path.join(a_   ,       a_))

				__a :						List[Any]			       =  F"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
				if model_size == "7B":
								# Unsharded
								__a :						Optional[int]			       =  {
								    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
								    '''model.norm.weight''': loaded['''norm.weight'''],
								    '''lm_head.weight''': loaded['''output.weight'''],
								}
				else:
								__a :						List[Any]			       =  {
								    '''model.norm.weight''': loaded[0]['''norm.weight'''],
								    '''model.embed_tokens.weight''': torch.cat(
								        [loaded[i]['''tok_embeddings.weight'''] for i in range(a_)]   ,       dim=1),
								    '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(a_)]   ,       dim=0),
								}

				for k, v in state_dict.items():
								__a :						Any			       =  filename
								param_count += v.numel()
				torch.save(a_   ,       os.path.join(a_   ,       a_))

				# Write configs
				__a :						Optional[int]			       =  {'''total_size''': param_count * 2}
				write_json(a_   ,       os.path.join(a_   ,       '''pytorch_model.bin.index.json'''))
				__a :						Union[str, Any]			       =  params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
				__a :						Any			       =  params['''multiple_of'''] if '''multiple_of''' in params else 2_56
				__a :						Optional[int]			       =  LlamaConfig(
				    hidden_size=a_   ,       intermediate_size=compute_intermediate_size(a_   ,       a_   ,       a_)   ,       num_attention_heads=params['''n_heads''']   ,       num_hidden_layers=params['''n_layers''']   ,       rms_norm_eps=params['''norm_eps''']   ,       num_key_value_heads=a_   ,       )
				config.save_pretrained(a_)

				# Make space so we can load the model properly now.
				del state_dict
				del loaded
				gc.collect()

				print('''Loading the checkpoint in a Llama model.''')
				__a :						str			       =  LlamaForCausalLM.from_pretrained(a_   ,       torch_dtype=torch.floataa   ,       low_cpu_mem_usage=a_)
				# Avoid saving this as part of the config.
				del model.config._name_or_path

				print('''Saving in the Transformers format.''')
				model.save_pretrained(a_   ,       safe_serialization=a_)
				shutil.rmtree(a_)

def     __A      (			a_						:Any   ,       a_						:str)		->		int:
				# Initialize the tokenizer based on the `spm` model
				__a :						Optional[int]			       =  LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
				print(F"""Saving a {tokenizer_class.__name__} to {tokenizer_path}.""")
				__a :						str			       =  tokenizer_class(a_)
				tokenizer.save_pretrained(a_)

def     __A      (			)		->		Tuple:
				__a :						Optional[int]			       =  argparse.ArgumentParser()
				parser.add_argument(
				    '''--input_dir'''   ,       help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''   ,       )
				parser.add_argument(
				    '''--model_size'''   ,       choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']   ,       )
				parser.add_argument(
				    '''--output_dir'''   ,       help='''Location to write HF model and tokenizer'''   ,       )
				parser.add_argument('''--safe_serialization'''   ,       type=a_   ,       help='''Whether or not to save using `safetensors`.''')
				__a :						Optional[Any]			       =  parser.parse_args()
				if args.model_size != "tokenizer_only":
								write_model(
								    model_path=args.output_dir   ,       input_base_path=os.path.join(args.input_dir   ,       args.model_size)   ,       model_size=args.model_size   ,       safe_serialization=args.safe_serialization   ,       )
				__a :						Optional[Any]			       =  os.path.join(args.input_dir   ,       '''tokenizer.model''')
				write_tokenizer(args.output_dir   ,       a_)


if __name__ == "__main__":
		main()






"""simple docstring"""






def     __A      (			a_						:int = 2_00)		->		int:
				__a :						int			       =  [1, 2, 5, 10, 20, 50, 1_00, 2_00]
				__a :						List[Any]			       =  [0] * (pence + 1)
				__a :						Tuple			       =  1  # base case: 1 way to make 0 pence

				for coin in coins:
								for i in range(a_   ,       pence + 1   ,       1):
												number_of_ways[i] += number_of_ways[i - coin]
				return number_of_ways[pence]


if __name__ == "__main__":
		assert solution(200) == 73_682