



"""simple docstring"""
import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
	from transformers import LlamaTokenizerFast
except ImportError as e:
	warnings.warn(e)
	warnings.warn(
	    """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
	)
	SCREAMING_SNAKE_CASE_				= None
SCREAMING_SNAKE_CASE_				= {
    """7B""": 1_10_08,
    """13B""": 1_38_24,
    """30B""": 1_79_20,
    """65B""": 2_20_16,
    """70B""": 2_86_72,
}
SCREAMING_SNAKE_CASE_				= {
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__=1, SCREAMING_SNAKE_CASE__=256	)			->      Optional[Any]:
		return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3	)	) + multiple_of - 1) // multiple_of)



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__	)			->      Dict:


		with open(lowerCAmelCase_, "r"	) as f:
				return json.load(lowerCAmelCase_	)



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)			->      Tuple:


		with open(lowerCAmelCase_, "w"	) as f:
				json.dump(lowerCAmelCase_, lowerCAmelCase_	)



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__=True	)			->      Optional[int]:
		os.makedirs(lowerCAmelCase_, exist_ok=lowerCAmelCase_	)
		a_     :      Optional[int] 						=     os.path.join(lowerCAmelCase_, "tmp"	)
		os.makedirs(lowerCAmelCase_, exist_ok=lowerCAmelCase_	)

		a_     :      Tuple 						=     read_json(os.path.join(lowerCAmelCase_, "params.json"	)	)
		a_     :      str 						=     NUM_SHARDS[model_size]
		a_     :      Dict 						=     params["n_layers"]
		a_     :      List[str] 						=     params["n_heads"]
		a_     :      Dict 						=     n_heads // num_shards
		a_     :      List[str] 						=     params["dim"]
		a_     :      str 						=     dim // n_heads
		a_     :      Optional[int] 						=     10_000.0
		a_     :      str 						=     1.0 / (base ** (torch.arange(0, lowerCAmelCase_, 2	).float() / dims_per_head))

		if "n_kv_heads" in params:
				a_     :      int 						=     params["n_kv_heads"]  # for GQA / MQA
				a_     :      Dict 						=     n_heads_per_shard // num_key_value_heads
				a_     :      Tuple 						=     dim // num_key_value_heads
		else:  # compatibility with other checkpoints
				a_     :      List[str] 						=     n_heads
				a_     :      Optional[int] 						=     n_heads_per_shard
				a_     :      List[Any] 						=     dim

		# permute for sliced rotary
		def permute(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__=n_heads, SCREAMING_SNAKE_CASE__=dim, SCREAMING_SNAKE_CASE__=dim	):
				return w.view(lowerCAmelCase_, dima // n_heads // 2, 2, lowerCAmelCase_	).transpose(1, 2	).reshape(lowerCAmelCase_, lowerCAmelCase_	)

		print(F"""Fetching all parameters from the checkpoint at {input_base_path}."""	)
		# Load weights
		if model_size == "7B":
				# Not sharded
				# (The sharded implementation would also work, but this is simpler.)
				a_     :      str 						=     torch.load(os.path.join(lowerCAmelCase_, "consolidated.00.pth"	), map_location="cpu"	)
		else:
				# Sharded
				a_     :      Tuple 						=     [
				    torch.load(os.path.join(lowerCAmelCase_, F"""consolidated.{i:02d}.pth"""	), map_location="cpu"	)
				    for i in range(lowerCAmelCase_	)
				]
		a_     :      Any 						=     0
		a_     :      Union[str, Any] 						=     {"weight_map": {}}
		for layer_i in range(lowerCAmelCase_	):
				a_     :      str 						=     F"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
				if model_size == "7B":
						# Unsharded
						a_     :      Optional[Any] 						=     {
						    F"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
						        loaded[F"""layers.{layer_i}.attention.wq.weight"""]	),
						    F"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
						        loaded[F"""layers.{layer_i}.attention.wk.weight"""]	),
						    F"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[F"""layers.{layer_i}.attention.wv.weight"""],
						    F"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[F"""layers.{layer_i}.attention.wo.weight"""],
						    F"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w1.weight"""],
						    F"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w2.weight"""],
						    F"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w3.weight"""],
						    F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[F"""layers.{layer_i}.attention_norm.weight"""],
						    F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[F"""layers.{layer_i}.ffn_norm.weight"""],
						}
				else:
						# Sharded
						# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
						# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
						# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

						a_     :      List[str] 						=     {
						    F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
						        F"""layers.{layer_i}.attention_norm.weight"""
						    ].clone(),
						    F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
						        F"""layers.{layer_i}.ffn_norm.weight"""
						    ].clone(),
						}
						a_     :      str 						=     permute(
						    torch.cat(
						        [
						            loaded[i][F"""layers.{layer_i}.attention.wq.weight"""].view(lowerCAmelCase_, lowerCAmelCase_, lowerCAmelCase_	)
						            for i in range(lowerCAmelCase_	)
						        ], dim=0, ).reshape(lowerCAmelCase_, lowerCAmelCase_	)	)
						a_     :      Tuple 						=     permute(
						    torch.cat(
						        [
						            loaded[i][F"""layers.{layer_i}.attention.wk.weight"""].view(
						                lowerCAmelCase_, lowerCAmelCase_, lowerCAmelCase_	)
						            for i in range(lowerCAmelCase_	)
						        ], dim=0, ).reshape(lowerCAmelCase_, lowerCAmelCase_	), lowerCAmelCase_, lowerCAmelCase_, lowerCAmelCase_, )
						a_     :      Dict 						=     torch.cat(
						    [
						        loaded[i][F"""layers.{layer_i}.attention.wv.weight"""].view(
						            lowerCAmelCase_, lowerCAmelCase_, lowerCAmelCase_	)
						        for i in range(lowerCAmelCase_	)
						    ], dim=0, ).reshape(lowerCAmelCase_, lowerCAmelCase_	)

						a_     :      Optional[int] 						=     torch.cat(
						    [loaded[i][F"""layers.{layer_i}.attention.wo.weight"""] for i in range(lowerCAmelCase_	)], dim=1	)
						a_     :      Dict 						=     torch.cat(
						    [loaded[i][F"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(lowerCAmelCase_	)], dim=0	)
						a_     :      Union[str, Any] 						=     torch.cat(
						    [loaded[i][F"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(lowerCAmelCase_	)], dim=1	)
						a_     :      Dict 						=     torch.cat(
						    [loaded[i][F"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(lowerCAmelCase_	)], dim=0	)

				a_     :      Optional[Any] 						=     inv_freq
				for k, v in state_dict.items():
						a_     :      List[Any] 						=     filename
						param_count += v.numel()
				torch.save(lowerCAmelCase_, os.path.join(lowerCAmelCase_, lowerCAmelCase_	)	)

		a_     :      Dict 						=     F"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
		if model_size == "7B":
				# Unsharded
				a_     :      Dict 						=     {
				    "model.embed_tokens.weight": loaded["tok_embeddings.weight"],
				    "model.norm.weight": loaded["norm.weight"],
				    "lm_head.weight": loaded["output.weight"],
				}
		else:
				a_     :      Any 						=     {
				    "model.norm.weight": loaded[0]["norm.weight"],
				    "model.embed_tokens.weight": torch.cat(
				        [loaded[i]["tok_embeddings.weight"] for i in range(lowerCAmelCase_	)], dim=1	),
				    "lm_head.weight": torch.cat([loaded[i]["output.weight"] for i in range(lowerCAmelCase_	)], dim=0	),
				}

		for k, v in state_dict.items():
				a_     :      str 						=     filename
				param_count += v.numel()
		torch.save(lowerCAmelCase_, os.path.join(lowerCAmelCase_, lowerCAmelCase_	)	)

		# Write configs
		a_     :      str 						=     {"total_size": param_count * 2}
		write_json(lowerCAmelCase_, os.path.join(lowerCAmelCase_, "pytorch_model.bin.index.json"	)	)
		a_     :      Union[str, Any] 						=     params["ffn_dim_multiplier"] if "ffn_dim_multiplier" in params else 1
		a_     :      Any 						=     params["multiple_of"] if "multiple_of" in params else 256
		a_     :      Optional[int] 						=     LlamaConfig(
		    hidden_size=lowerCAmelCase_, intermediate_size=compute_intermediate_size(lowerCAmelCase_, lowerCAmelCase_, lowerCAmelCase_	), num_attention_heads=params["n_heads"], num_hidden_layers=params["n_layers"], rms_norm_eps=params["norm_eps"], num_key_value_heads=lowerCAmelCase_, )
		config.save_pretrained(lowerCAmelCase_	)

		# Make space so we can load the model properly now.
		del state_dict
		del loaded
		gc.collect()

		print("Loading the checkpoint in a Llama model."	)
		a_     :      List[Any] 						=     LlamaForCausalLM.from_pretrained(lowerCAmelCase_, torch_dtype=torch.floataa, low_cpu_mem_usage=lowerCAmelCase_	)
		# Avoid saving this as part of the config.
		del model.config._name_or_path

		print("Saving in the Transformers format."	)
		model.save_pretrained(lowerCAmelCase_, safe_serialization=lowerCAmelCase_	)
		shutil.rmtree(lowerCAmelCase_	)



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)			->      str:
		a_     :      str 						=     LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
		print(F"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""	)
		a_     :      Tuple 						=     tokenizer_class(lowerCAmelCase_	)
		tokenizer.save_pretrained(lowerCAmelCase_	)



def        lowerCAmelCase_						(							)			->      Tuple:
		a_     :      Any 						=     argparse.ArgumentParser()
		parser.add_argument(
		    "--input_dir", help="Location of LLaMA weights, which contains tokenizer.model and model folders", )
		parser.add_argument(
		    "--model_size", choices=["7B", "7Bf", "13B", "13Bf", "30B", "65B", "70B", "70Bf", "tokenizer_only"], )
		parser.add_argument(
		    "--output_dir", help="Location to write HF model and tokenizer", )
		parser.add_argument("--safe_serialization", type=lowerCAmelCase_, help="Whether or not to save using `safetensors`."	)
		a_     :      Dict 						=     parser.parse_args()
		if args.model_size != "tokenizer_only":
				write_model(
				    model_path=args.output_dir, input_base_path=os.path.join(args.input_dir, args.model_size	), model_size=args.model_size, safe_serialization=args.safe_serialization, )
		a_     :      Tuple 						=     os.path.join(args.input_dir, "tokenizer.model"	)
		write_tokenizer(args.output_dir, lowerCAmelCase_	)


if __name__ == "__main__":
	main()




"""simple docstring"""
SCREAMING_SNAKE_CASE_				= 0  # The first color of the flag.
SCREAMING_SNAKE_CASE_				= 1  # The second color of the flag.
SCREAMING_SNAKE_CASE_				= 2  # The third color of the flag.
SCREAMING_SNAKE_CASE_				= (red, white, blue)



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__	)			->      list:
		if not sequence:
				return []
		if len(SCREAMING_SNAKE_CASE__	) == 1:
				return list(SCREAMING_SNAKE_CASE__	)
		a_     :      Dict 						=     0
		a_     :      List[Any] 						=     len(SCREAMING_SNAKE_CASE__	) - 1
		a_     :      str 						=     0
		while mid <= high:
				if sequence[mid] == colors[0]:
						a_						,				a_     :      int 						=     sequence[mid], sequence[low]
						low += 1
						mid += 1
				elif sequence[mid] == colors[1]:
						mid += 1
				elif sequence[mid] == colors[2]:
						a_						,				a_     :      List[Any] 						=     sequence[high], sequence[mid]
						high -= 1
				else:
						a_     :      Dict 						=     F"""The elements inside the sequence must contains only {colors} values"""
						raise ValueError(SCREAMING_SNAKE_CASE__	)
		return sequence


if __name__ == "__main__":
					import doctest

					doctest.testmod()

					SCREAMING_SNAKE_CASE_				= input("""Enter numbers separated by commas:\n""").strip()
					SCREAMING_SNAKE_CASE_				= [int(item.strip()) for item in user_input.split(""",""")]
					print(F"""{dutch_national_flag_sort(unsorted)}""")