





import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
					from transformers import LlamaTokenizerFast
except ImportError as e:
					warnings.warn(e)
					warnings.warn(
					    '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
					)
					A   : Optional[int]			=				None
A   : List[str]			=				{
    '''7B''': 1_1_0_0_8,
    '''13B''': 1_3_8_2_4,
    '''30B''': 1_7_9_2_0,
    '''65B''': 2_2_0_1_6,
    '''70B''': 2_8_6_7_2,
}
A   : Tuple			=				{
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}







def        __lowerCamelCase (							__a       :List[Any]   ,					__a       :int=1   ,					__a       :Tuple=2_5_6   )		->    Optional[Any]:






		"""simple docstring"""

		return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3   )   ) + multiple_of - 1) // multiple_of)







def        __lowerCamelCase (							__a       :Dict   )		->    List[str]:






		"""simple docstring"""

		with open(__a   ,					"""r"""   ) as f:
				return json.load(__a   )







def        __lowerCamelCase (							__a       :int   ,					__a       :Tuple   )		->    Tuple:






		"""simple docstring"""

		with open(__a   ,					"""w"""   ) as f:
				json.dump(__a   ,					__a   )







def        __lowerCamelCase (							__a       :List[Any]   ,					__a       :int   ,					__a       :Dict   ,					__a       :Tuple=True   )		->    Union[str, Any]:






		"""simple docstring"""

		os.makedirs(__a   ,					exist_ok=__a   )
		A__  				=       os.path.join(__a   ,					"""tmp"""   )
		os.makedirs(__a   ,					exist_ok=__a   )

		A__  				=       read_json(os.path.join(__a   ,					"""params.json"""   )   )
		A__  				=       NUM_SHARDS[model_size]
		A__  				=       params["""n_layers"""]
		A__  				=       params["""n_heads"""]
		A__  				=       n_heads // num_shards
		A__  				=       params["""dim"""]
		A__  				=       dim // n_heads
		A__  				=       10000.0
		A__  				=       1.0 / (base ** (torch.arange(0   ,					__a   ,					2   ).float() / dims_per_head))

		if "n_kv_heads" in params:
				A__  				=       params["""n_kv_heads"""]  # for GQA / MQA
				A__  				=       n_heads_per_shard // num_key_value_heads
				A__  				=       dim // num_key_value_heads
		else:  # compatibility with other checkpoints
				A__  				=       n_heads
				A__  				=       n_heads_per_shard
				A__  				=       dim

		# permute for sliced rotary
		def permute(__a       :str   ,					__a       :Optional[int]=n_heads   ,					__a       :Dict=dim   ,					__a       :Dict=dim   ):
				return w.view(__a   ,					dima // n_heads // 2   ,					2   ,					__a   ).transpose(1   ,					2   ).reshape(__a   ,					__a   )

		print(F'Fetching all parameters from the checkpoint at {input_base_path}.'   )
		# Load weights
		if model_size == "7B":
				# Not sharded
				# (The sharded implementation would also work, but this is simpler.)
				A__  				=       torch.load(os.path.join(__a   ,					"""consolidated.00.pth"""   )   ,					map_location="""cpu"""   )
		else:
				# Sharded
				A__  				=       [
				    torch.load(os.path.join(__a   ,					F'consolidated.{i:02d}.pth'   )   ,					map_location="""cpu"""   )
				    for i in range(__a   )
				]
		A__  				=       0
		A__  				=       {"""weight_map""": {}}
		for layer_i in range(__a   ):
				A__  				=       F'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'
				if model_size == "7B":
						# Unsharded
						A__  				=       {
						    F'model.layers.{layer_i}.self_attn.q_proj.weight': permute(
						        loaded[F'layers.{layer_i}.attention.wq.weight']   ),
						    F'model.layers.{layer_i}.self_attn.k_proj.weight': permute(
						        loaded[F'layers.{layer_i}.attention.wk.weight']   ),
						    F'model.layers.{layer_i}.self_attn.v_proj.weight': loaded[F'layers.{layer_i}.attention.wv.weight'],
						    F'model.layers.{layer_i}.self_attn.o_proj.weight': loaded[F'layers.{layer_i}.attention.wo.weight'],
						    F'model.layers.{layer_i}.mlp.gate_proj.weight': loaded[F'layers.{layer_i}.feed_forward.w1.weight'],
						    F'model.layers.{layer_i}.mlp.down_proj.weight': loaded[F'layers.{layer_i}.feed_forward.w2.weight'],
						    F'model.layers.{layer_i}.mlp.up_proj.weight': loaded[F'layers.{layer_i}.feed_forward.w3.weight'],
						    F'model.layers.{layer_i}.input_layernorm.weight': loaded[F'layers.{layer_i}.attention_norm.weight'],
						    F'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[F'layers.{layer_i}.ffn_norm.weight'],
						}
				else:
						# Sharded
						# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
						# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
						# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

						A__  				=       {
						    F'model.layers.{layer_i}.input_layernorm.weight': loaded[0][
						        F'layers.{layer_i}.attention_norm.weight'
						    ].clone(),
						    F'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[0][
						        F'layers.{layer_i}.ffn_norm.weight'
						    ].clone(),
						}
						A__  				=       permute(
						    torch.cat(
						        [
						            loaded[i][F'layers.{layer_i}.attention.wq.weight'].view(__a   ,					__a   ,					__a   )
						            for i in range(__a   )
						        ]   ,					dim=0   ,					).reshape(__a   ,					__a   )   )
						A__  				=       permute(
						    torch.cat(
						        [
						            loaded[i][F'layers.{layer_i}.attention.wk.weight'].view(
						                __a   ,					__a   ,					__a   )
						            for i in range(__a   )
						        ]   ,					dim=0   ,					).reshape(__a   ,					__a   )   ,					__a   ,					__a   ,					__a   ,					)
						A__  				=       torch.cat(
						    [
						        loaded[i][F'layers.{layer_i}.attention.wv.weight'].view(
						            __a   ,					__a   ,					__a   )
						        for i in range(__a   )
						    ]   ,					dim=0   ,					).reshape(__a   ,					__a   )

						A__  				=       torch.cat(
						    [loaded[i][F'layers.{layer_i}.attention.wo.weight'] for i in range(__a   )]   ,					dim=1   )
						A__  				=       torch.cat(
						    [loaded[i][F'layers.{layer_i}.feed_forward.w1.weight'] for i in range(__a   )]   ,					dim=0   )
						A__  				=       torch.cat(
						    [loaded[i][F'layers.{layer_i}.feed_forward.w2.weight'] for i in range(__a   )]   ,					dim=1   )
						A__  				=       torch.cat(
						    [loaded[i][F'layers.{layer_i}.feed_forward.w3.weight'] for i in range(__a   )]   ,					dim=0   )

				A__  				=       inv_freq
				for k, v in state_dict.items():
						A__  				=       filename
						param_count += v.numel()
				torch.save(__a   ,					os.path.join(__a   ,					__a   )   )

		A__  				=       F'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'
		if model_size == "7B":
				# Unsharded
				A__  				=       {
				    """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
				    """model.norm.weight""": loaded["""norm.weight"""],
				    """lm_head.weight""": loaded["""output.weight"""],
				}
		else:
				A__  				=       {
				    """model.norm.weight""": loaded[0]["""norm.weight"""],
				    """model.embed_tokens.weight""": torch.cat(
				        [loaded[i]["""tok_embeddings.weight"""] for i in range(__a   )]   ,					dim=1   ),
				    """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(__a   )]   ,					dim=0   ),
				}

		for k, v in state_dict.items():
				A__  				=       filename
				param_count += v.numel()
		torch.save(__a   ,					os.path.join(__a   ,					__a   )   )

		# Write configs
		A__  				=       {"""total_size""": param_count * 2}
		write_json(__a   ,					os.path.join(__a   ,					"""pytorch_model.bin.index.json"""   )   )
		A__  				=       params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
		A__  				=       params["""multiple_of"""] if """multiple_of""" in params else 2_5_6
		A__  				=       LlamaConfig(
		    hidden_size=__a   ,					intermediate_size=compute_intermediate_size(__a   ,					__a   ,					__a   )   ,					num_attention_heads=params["""n_heads"""]   ,					num_hidden_layers=params["""n_layers"""]   ,					rms_norm_eps=params["""norm_eps"""]   ,					num_key_value_heads=__a   ,					)
		config.save_pretrained(__a   )

		# Make space so we can load the model properly now.
		del state_dict
		del loaded
		gc.collect()

		print("""Loading the checkpoint in a Llama model."""   )
		A__  				=       LlamaForCausalLM.from_pretrained(__a   ,					torch_dtype=torch.floataa   ,					low_cpu_mem_usage=__a   )
		# Avoid saving this as part of the config.
		del model.config._name_or_path

		print("""Saving in the Transformers format."""   )
		model.save_pretrained(__a   ,					safe_serialization=__a   )
		shutil.rmtree(__a   )







def        __lowerCamelCase (							__a       :List[str]   ,					__a       :Tuple   )		->    Tuple:






		"""simple docstring"""

		A__  				=       LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
		print(F'Saving a {tokenizer_class.__name__} to {tokenizer_path}.'   )
		A__  				=       tokenizer_class(__a   )
		tokenizer.save_pretrained(__a   )







def        __lowerCamelCase (							)		->    List[str]:






		"""simple docstring"""

		A__  				=       argparse.ArgumentParser()
		parser.add_argument(
		    """--input_dir"""   ,					help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""   ,					)
		parser.add_argument(
		    """--model_size"""   ,					choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]   ,					)
		parser.add_argument(
		    """--output_dir"""   ,					help="""Location to write HF model and tokenizer"""   ,					)
		parser.add_argument("""--safe_serialization"""   ,					type=__a   ,					help="""Whether or not to save using `safetensors`."""   )
		A__  				=       parser.parse_args()
		if args.model_size != "tokenizer_only":
				write_model(
				    model_path=args.output_dir   ,					input_base_path=os.path.join(args.input_dir   ,					args.model_size   )   ,					model_size=args.model_size   ,					safe_serialization=args.safe_serialization   ,					)
		A__  				=       os.path.join(args.input_dir   ,					"""tokenizer.model"""   )
		write_tokenizer(args.output_dir   ,					__a   )


if __name__ == "__main__":
					main()










# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import torch

from accelerate import PartialState
from accelerate.utils.operations import broadcast, gather, gather_object, pad_across_processes, reduce







def        __lowerCamelCase (							__a       :List[str]   )		->    Tuple:






		"""simple docstring"""

		return (torch.arange(state.num_processes   ) + 1.0 + (state.num_processes * state.process_index)).to(state.device   )







def        __lowerCamelCase (							__a       :int   )		->    Optional[int]:






		"""simple docstring"""

		A__  				=       create_tensor(__a   )
		A__  				=       gather(__a   )
		assert gathered_tensor.tolist() == list(range(1   ,					state.num_processes**2 + 1   )   )







def        __lowerCamelCase (							__a       :Any   )		->    Any:






		"""simple docstring"""

		A__  				=       [state.process_index]
		A__  				=       gather_object(__a   )
		assert len(__a   ) == state.num_processes, F'{gathered_obj}, {len(__a   )} != {state.num_processes}'
		assert gathered_obj == list(range(state.num_processes   )   ), F'{gathered_obj} != {list(range(state.num_processes   )   )}'







def        __lowerCamelCase (							__a       :List[str]   )		->    List[str]:






		"""simple docstring"""

		A__  				=       create_tensor(__a   )
		A__  				=       broadcast(__a   )
		assert broadcasted_tensor.shape == torch.Size([state.num_processes]   )
		assert broadcasted_tensor.tolist() == list(range(1   ,					state.num_processes + 1   )   )







def        __lowerCamelCase (							__a       :Any   )		->    Any:






		"""simple docstring"""

		if state.is_main_process:
				A__  				=       torch.arange(state.num_processes + 1   ).to(state.device   )
		else:
				A__  				=       torch.arange(state.num_processes   ).to(state.device   )
		A__  				=       pad_across_processes(__a   )
		assert padded_tensor.shape == torch.Size([state.num_processes + 1]   )
		if not state.is_main_process:
				assert padded_tensor.tolist() == list(range(0   ,					state.num_processes   )   ) + [0]







def        __lowerCamelCase (							__a       :Union[str, Any]   )		->    str:






		"""simple docstring"""

		if state.num_processes != 2:
				return
		A__  				=       create_tensor(__a   )
		A__  				=       reduce(__a   ,					"""sum"""   )
		A__  				=       torch.tensor([4.0, 6]   ).to(state.device   )
		assert torch.allclose(__a   ,					__a   ), F'{reduced_tensor} != {truth_tensor}'







def        __lowerCamelCase (							__a       :List[Any]   )		->    List[str]:






		"""simple docstring"""

		if state.num_processes != 2:
				return
		A__  				=       create_tensor(__a   )
		A__  				=       reduce(__a   ,					"""mean"""   )
		A__  				=       torch.tensor([2.0, 3]   ).to(state.device   )
		assert torch.allclose(__a   ,					__a   ), F'{reduced_tensor} != {truth_tensor}'







def        __lowerCamelCase (							__a       :str   )		->    Optional[int]:






		"""simple docstring"""

		main()







def        __lowerCamelCase (							)		->    str:






		"""simple docstring"""

		A__  				=       PartialState()
		state.print(F'State: {state}'   )
		state.print("""testing gather"""   )
		test_gather(__a   )
		state.print("""testing gather_object"""   )
		test_gather_object(__a   )
		state.print("""testing broadcast"""   )
		test_broadcast(__a   )
		state.print("""testing pad_across_processes"""   )
		test_pad_across_processes(__a   )
		state.print("""testing reduce_sum"""   )
		test_reduce_sum(__a   )
		state.print("""testing reduce_mean"""   )
		test_reduce_mean(__a   )


if __name__ == "__main__":
					main()



