


'''simple docstring'''



import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
		from transformers import LlamaTokenizerFast
except ImportError as e:
		warnings.warn(e)
		warnings.warn(
		    'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
		)
		__A		=None
__A		={
    '7B': 1_10_08,
    '13B': 1_38_24,
    '30B': 1_79_20,
    '65B': 2_20_16,
    '70B': 2_86_72,
}
__A		={
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}







def       _UpperCamelCase (						UpperCamelCase__	, UpperCamelCase__=1	, UpperCamelCase__=2_5_6   ):
			return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3   )   ) + multiple_of - 1) // multiple_of)







def       _UpperCamelCase (						UpperCamelCase__   ):
			with open(UpperCamelCase__	, """r"""   ) as f:
						return json.load(UpperCamelCase__   )







def       _UpperCamelCase (						UpperCamelCase__	, UpperCamelCase__   ):
			with open(UpperCamelCase__	, """w"""   ) as f:
						json.dump(UpperCamelCase__	, UpperCamelCase__   )







def       _UpperCamelCase (						UpperCamelCase__	, UpperCamelCase__	, UpperCamelCase__	, UpperCamelCase__=True   ):
			os.makedirs(UpperCamelCase__	, exist_ok=UpperCamelCase__   )
			UpperCAmelCase__					:   int       				=       os.path.join(UpperCamelCase__	, """tmp"""   )
			os.makedirs(UpperCamelCase__	, exist_ok=UpperCamelCase__   )

			UpperCAmelCase__					:   Any       				=       read_json(os.path.join(UpperCamelCase__	, """params.json"""   )   )
			UpperCAmelCase__					:   Tuple       				=       NUM_SHARDS[model_size]
			UpperCAmelCase__					:   Optional[int]       				=       params["""n_layers"""]
			UpperCAmelCase__					:   List[str]       				=       params["""n_heads"""]
			UpperCAmelCase__					:   Dict       				=       n_heads // num_shards
			UpperCAmelCase__					:   Optional[int]       				=       params["""dim"""]
			UpperCAmelCase__					:   Optional[Any]       				=       dim // n_heads
			UpperCAmelCase__					:   List[Any]       				=       1_00_00.0
			UpperCAmelCase__					:   Any       				=       1.0 / (base ** (torch.arange(0	, UpperCamelCase__	, 2   ).float() / dims_per_head))

			if "n_kv_heads" in params:
						UpperCAmelCase__					:   Optional[Any]       				=       params["""n_kv_heads"""]  # for GQA / MQA
						UpperCAmelCase__					:   Optional[int]       				=       n_heads_per_shard // num_key_value_heads
						UpperCAmelCase__					:   Optional[int]       				=       dim // num_key_value_heads
			else:  # compatibility with other checkpoints
						UpperCAmelCase__					:   Tuple       				=       n_heads
						UpperCAmelCase__					:   Optional[int]       				=       n_heads_per_shard
						UpperCAmelCase__					:   List[Any]       				=       dim

			# permute for sliced rotary
			def permute(UpperCamelCase__	, UpperCamelCase__=n_heads	, UpperCamelCase__=dim	, UpperCamelCase__=dim   ):
						return w.view(UpperCamelCase__	, dima // n_heads // 2	, 2	, UpperCamelCase__   ).transpose(1	, 2   ).reshape(UpperCamelCase__	, UpperCamelCase__   )

			print(f'''Fetching all parameters from the checkpoint at {input_base_path}.'''   )
			# Load weights
			if model_size == "7B":
						# Not sharded
						# (The sharded implementation would also work, but this is simpler.)
						UpperCAmelCase__					:   Any       				=       torch.load(os.path.join(UpperCamelCase__	, """consolidated.00.pth"""   )	, map_location="""cpu"""   )
			else:
						# Sharded
						UpperCAmelCase__					:   Tuple       				=       [
						    torch.load(os.path.join(UpperCamelCase__	, f'''consolidated.{i:02d}.pth'''   )	, map_location="""cpu"""   )
						    for i in range(UpperCamelCase__   )
						]
			UpperCAmelCase__					:   Union[str, Any]       				=       0
			UpperCAmelCase__					:   Union[str, Any]       				=       {"""weight_map""": {}}
			for layer_i in range(UpperCamelCase__   ):
						UpperCAmelCase__					:   Any       				=       f'''pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'''
						if model_size == "7B":
									# Unsharded
									UpperCAmelCase__					:   Tuple       				=       {
									    f'''model.layers.{layer_i}.self_attn.q_proj.weight''': permute(
									        loaded[f'''layers.{layer_i}.attention.wq.weight''']   ),
									    f'''model.layers.{layer_i}.self_attn.k_proj.weight''': permute(
									        loaded[f'''layers.{layer_i}.attention.wk.weight''']   ),
									    f'''model.layers.{layer_i}.self_attn.v_proj.weight''': loaded[f'''layers.{layer_i}.attention.wv.weight'''],
									    f'''model.layers.{layer_i}.self_attn.o_proj.weight''': loaded[f'''layers.{layer_i}.attention.wo.weight'''],
									    f'''model.layers.{layer_i}.mlp.gate_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w1.weight'''],
									    f'''model.layers.{layer_i}.mlp.down_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w2.weight'''],
									    f'''model.layers.{layer_i}.mlp.up_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w3.weight'''],
									    f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[f'''layers.{layer_i}.attention_norm.weight'''],
									    f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[f'''layers.{layer_i}.ffn_norm.weight'''],
									}
						else:
									# Sharded
									# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
									# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
									# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

									UpperCAmelCase__					:   str       				=       {
									    f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[0][
									        f'''layers.{layer_i}.attention_norm.weight'''
									    ].clone(),
									    f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[0][
									        f'''layers.{layer_i}.ffn_norm.weight'''
									    ].clone(),
									}
									UpperCAmelCase__					:   Union[str, Any]       				=       permute(
									    torch.cat(
									        [
									            loaded[i][f'''layers.{layer_i}.attention.wq.weight'''].view(UpperCamelCase__	, UpperCamelCase__	, UpperCamelCase__   )
									            for i in range(UpperCamelCase__   )
									        ]	, dim=0	, ).reshape(UpperCamelCase__	, UpperCamelCase__   )   )
									UpperCAmelCase__					:   Optional[Any]       				=       permute(
									    torch.cat(
									        [
									            loaded[i][f'''layers.{layer_i}.attention.wk.weight'''].view(
									                UpperCamelCase__	, UpperCamelCase__	, UpperCamelCase__   )
									            for i in range(UpperCamelCase__   )
									        ]	, dim=0	, ).reshape(UpperCamelCase__	, UpperCamelCase__   )	, UpperCamelCase__	, UpperCamelCase__	, UpperCamelCase__	, )
									UpperCAmelCase__					:   Union[str, Any]       				=       torch.cat(
									    [
									        loaded[i][f'''layers.{layer_i}.attention.wv.weight'''].view(
									            UpperCamelCase__	, UpperCamelCase__	, UpperCamelCase__   )
									        for i in range(UpperCamelCase__   )
									    ]	, dim=0	, ).reshape(UpperCamelCase__	, UpperCamelCase__   )

									UpperCAmelCase__					:   Dict       				=       torch.cat(
									    [loaded[i][f'''layers.{layer_i}.attention.wo.weight'''] for i in range(UpperCamelCase__   )]	, dim=1   )
									UpperCAmelCase__					:   Optional[Any]       				=       torch.cat(
									    [loaded[i][f'''layers.{layer_i}.feed_forward.w1.weight'''] for i in range(UpperCamelCase__   )]	, dim=0   )
									UpperCAmelCase__					:   Any       				=       torch.cat(
									    [loaded[i][f'''layers.{layer_i}.feed_forward.w2.weight'''] for i in range(UpperCamelCase__   )]	, dim=1   )
									UpperCAmelCase__					:   Any       				=       torch.cat(
									    [loaded[i][f'''layers.{layer_i}.feed_forward.w3.weight'''] for i in range(UpperCamelCase__   )]	, dim=0   )

						UpperCAmelCase__					:   str       				=       inv_freq
						for k, v in state_dict.items():
									UpperCAmelCase__					:   Tuple       				=       filename
									param_count += v.numel()
						torch.save(UpperCamelCase__	, os.path.join(UpperCamelCase__	, UpperCamelCase__   )   )

			UpperCAmelCase__					:   str       				=       f'''pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'''
			if model_size == "7B":
						# Unsharded
						UpperCAmelCase__					:   Dict       				=       {
						    """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
						    """model.norm.weight""": loaded["""norm.weight"""],
						    """lm_head.weight""": loaded["""output.weight"""],
						}
			else:
						UpperCAmelCase__					:   Any       				=       {
						    """model.norm.weight""": loaded[0]["""norm.weight"""],
						    """model.embed_tokens.weight""": torch.cat(
						        [loaded[i]["""tok_embeddings.weight"""] for i in range(UpperCamelCase__   )]	, dim=1   ),
						    """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(UpperCamelCase__   )]	, dim=0   ),
						}

			for k, v in state_dict.items():
						UpperCAmelCase__					:   int       				=       filename
						param_count += v.numel()
			torch.save(UpperCamelCase__	, os.path.join(UpperCamelCase__	, UpperCamelCase__   )   )

			# Write configs
			UpperCAmelCase__					:   Optional[int]       				=       {"""total_size""": param_count * 2}
			write_json(UpperCamelCase__	, os.path.join(UpperCamelCase__	, """pytorch_model.bin.index.json"""   )   )
			UpperCAmelCase__					:   int       				=       params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
			UpperCAmelCase__					:   Dict       				=       params["""multiple_of"""] if """multiple_of""" in params else 2_5_6
			UpperCAmelCase__					:   Any       				=       LlamaConfig(
			    hidden_size=UpperCamelCase__	, intermediate_size=compute_intermediate_size(UpperCamelCase__	, UpperCamelCase__	, UpperCamelCase__   )	, num_attention_heads=params["""n_heads"""]	, num_hidden_layers=params["""n_layers"""]	, rms_norm_eps=params["""norm_eps"""]	, num_key_value_heads=UpperCamelCase__	, )
			config.save_pretrained(UpperCamelCase__   )

			# Make space so we can load the model properly now.
			del state_dict
			del loaded
			gc.collect()

			print("""Loading the checkpoint in a Llama model."""   )
			UpperCAmelCase__					:   Optional[int]       				=       LlamaForCausalLM.from_pretrained(UpperCamelCase__	, torch_dtype=torch.floataa	, low_cpu_mem_usage=UpperCamelCase__   )
			# Avoid saving this as part of the config.
			del model.config._name_or_path

			print("""Saving in the Transformers format."""   )
			model.save_pretrained(UpperCamelCase__	, safe_serialization=UpperCamelCase__   )
			shutil.rmtree(UpperCamelCase__   )







def       _UpperCamelCase (						UpperCamelCase__	, UpperCamelCase__   ):
			# Initialize the tokenizer based on the `spm` model
			UpperCAmelCase__					:   Union[str, Any]       				=       LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
			print(f'''Saving a {tokenizer_class.__name__} to {tokenizer_path}.'''   )
			UpperCAmelCase__					:   Union[str, Any]       				=       tokenizer_class(UpperCamelCase__   )
			tokenizer.save_pretrained(UpperCamelCase__   )







def       _UpperCamelCase (						):
			UpperCAmelCase__					:   Any       				=       argparse.ArgumentParser()
			parser.add_argument(
			    """--input_dir"""	, help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""	, )
			parser.add_argument(
			    """--model_size"""	, choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]	, )
			parser.add_argument(
			    """--output_dir"""	, help="""Location to write HF model and tokenizer"""	, )
			parser.add_argument("""--safe_serialization"""	, type=UpperCamelCase__	, help="""Whether or not to save using `safetensors`."""   )
			UpperCAmelCase__					:   Dict       				=       parser.parse_args()
			if args.model_size != "tokenizer_only":
						write_model(
						    model_path=args.output_dir	, input_base_path=os.path.join(args.input_dir	, args.model_size   )	, model_size=args.model_size	, safe_serialization=args.safe_serialization	, )
			UpperCAmelCase__					:   int       				=       os.path.join(args.input_dir	, """tokenizer.model"""   )
			write_tokenizer(args.output_dir	, UpperCamelCase__   )


if __name__ == "__main__":
		main()



'''simple docstring'''



import colorsys

from PIL import Image  # type: ignore







def       _UpperCamelCase (						UpperCamelCase__	, UpperCamelCase__	, UpperCamelCase__   ):
			UpperCAmelCase__					:   Optional[Any]       				=       x
			UpperCAmelCase__					:   Optional[int]       				=       y
			for step in range(UpperCamelCase__   ):  # noqa: B007
						UpperCAmelCase__					:   List[str]       				=       a * a - b * b + x
						UpperCAmelCase__					:   Optional[int]       				=       2 * a * b + y
						UpperCAmelCase__					:   int       				=       a_new

						# divergence happens for all complex number with an absolute value
						# greater than 4
						if a * a + b * b > 4:
									break
			return step / (max_step - 1)







def       _UpperCamelCase (						UpperCamelCase__   ):
			if distance == 1:
						return (0, 0, 0)
			else:
						return (2_5_5, 2_5_5, 2_5_5)







def       _UpperCamelCase (						UpperCamelCase__   ):
			if distance == 1:
						return (0, 0, 0)
			else:
						return tuple(round(i * 2_5_5   ) for i in colorsys.hsv_to_rgb(UpperCamelCase__	, 1	, 1   )   )







def       _UpperCamelCase (						UpperCamelCase__ = 8_0_0	, UpperCamelCase__ = 6_0_0	, UpperCamelCase__ = -0.6	, UpperCamelCase__ = 0	, UpperCamelCase__ = 3.2	, UpperCamelCase__ = 5_0	, UpperCamelCase__ = True	, ):
			UpperCAmelCase__					:   str       				=       Image.new("""RGB"""	, (image_width, image_height)   )
			UpperCAmelCase__					:   Optional[int]       				=       img.load()

			# loop through the image-coordinates
			for image_x in range(UpperCamelCase__   ):
						for image_y in range(UpperCamelCase__   ):
									# determine the figure-coordinates based on the image-coordinates
									UpperCAmelCase__					:   Union[str, Any]       				=       figure_width / image_width * image_height
									UpperCAmelCase__					:   Union[str, Any]       				=       figure_center_x + (image_x / image_width - 0.5) * figure_width
									UpperCAmelCase__					:   Tuple       				=       figure_center_y + (image_y / image_height - 0.5) * figure_height

									UpperCAmelCase__					:   List[str]       				=       get_distance(UpperCamelCase__	, UpperCamelCase__	, UpperCamelCase__   )

									# color the corresponding pixel based on the selected coloring-function
									if use_distance_color_coding:
												UpperCAmelCase__					:   Any       				=       get_color_coded_rgb(UpperCamelCase__   )
									else:
												UpperCAmelCase__					:   Tuple       				=       get_black_and_white_rgb(UpperCamelCase__   )

			return img


if __name__ == "__main__":
		import doctest

		doctest.testmod()

		# colored version, full figure
		__A		=get_image()

		# uncomment for colored version, different section, zoomed in
		# img = get_image(figure_center_x = -0.6, figure_center_y = -0.4,
		# figure_width = 0.8)

		# uncomment for black and white version, full figure
		# img = get_image(use_distance_color_coding = False)

		# uncomment to save the image
		# img.save("mandelbrot.png")

		img.show()