
"""simple docstring"""


import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
						from transformers import LlamaTokenizerFast
except ImportError as e:
						warnings.warn(e)
						warnings.warn(
						    """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
						)
						A_    							=					None
A_    							=					{
    """7B""": 1_1008,
    """13B""": 1_3824,
    """30B""": 1_7920,
    """65B""": 2_2016,
    """70B""": 2_8672,
}
A_    							=					{
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}






def  lowercase						(	lowerCAmelCase__	,lowerCAmelCase__=1	,lowerCAmelCase__=256						):
			return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3						)						) + multiple_of - 1) // multiple_of)






def  lowercase						(	lowerCAmelCase__						):
			with open(lowerCAmelCase__	,'''r'''						) as f:
						return json.load(lowerCAmelCase__						)






def  lowercase						(	lowerCAmelCase__	,lowerCAmelCase__						):
			with open(lowerCAmelCase__	,'''w'''						) as f:
						json.dump(lowerCAmelCase__	,lowerCAmelCase__						)






def  lowercase						(	lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__=True						):
			os.makedirs(lowerCAmelCase__	,exist_ok=lowerCAmelCase__						)
			lowerCamelCase_					=  os.path.join(lowerCAmelCase__	,'''tmp'''						)
			os.makedirs(lowerCAmelCase__	,exist_ok=lowerCAmelCase__						)

			lowerCamelCase_					=  read_json(os.path.join(lowerCAmelCase__	,'''params.json'''						)						)
			lowerCamelCase_					=  NUM_SHARDS[model_size]
			lowerCamelCase_					=  params['''n_layers''']
			lowerCamelCase_					=  params['''n_heads''']
			lowerCamelCase_					=  n_heads // num_shards
			lowerCamelCase_					=  params['''dim''']
			lowerCamelCase_					=  dim // n_heads
			lowerCamelCase_					=  10_000.0
			lowerCamelCase_					=  1.0 / (base ** (torch.arange(0	,lowerCAmelCase__	,2						).float() / dims_per_head))

			if "n_kv_heads" in params:
						lowerCamelCase_					=  params['''n_kv_heads''']  # for GQA / MQA
						lowerCamelCase_					=  n_heads_per_shard // num_key_value_heads
						lowerCamelCase_					=  dim // num_key_value_heads
			else:  # compatibility with other checkpoints
						lowerCamelCase_					=  n_heads
						lowerCamelCase_					=  n_heads_per_shard
						lowerCamelCase_					=  dim

			# permute for sliced rotary
			def permute(lowerCAmelCase__	,lowerCAmelCase__=n_heads	,lowerCAmelCase__=dim	,lowerCAmelCase__=dim						):
						return w.view(lowerCAmelCase__	,dima // n_heads // 2	,2	,lowerCAmelCase__						).transpose(1	,2						).reshape(lowerCAmelCase__	,lowerCAmelCase__						)

			print(f"Fetching all parameters from the checkpoint at {input_base_path}."						)
			# Load weights
			if model_size == "7B":
						# Not sharded
						# (The sharded implementation would also work, but this is simpler.)
						lowerCamelCase_					=  torch.load(os.path.join(lowerCAmelCase__	,'''consolidated.00.pth'''						)	,map_location='''cpu'''						)
			else:
						# Sharded
						lowerCamelCase_					=  [
						    torch.load(os.path.join(lowerCAmelCase__	,f"consolidated.{i:02d}.pth"						)	,map_location='''cpu'''						)
						    for i in range(lowerCAmelCase__						)
						]
			lowerCamelCase_					=  0
			lowerCamelCase_					=  {'''weight_map''': {}}
			for layer_i in range(lowerCAmelCase__						):
						lowerCamelCase_					=  f"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
						if model_size == "7B":
									# Unsharded
									lowerCamelCase_					=  {
									    f"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
									        loaded[f"layers.{layer_i}.attention.wq.weight"]						),
									    f"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
									        loaded[f"layers.{layer_i}.attention.wk.weight"]						),
									    f"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[f"layers.{layer_i}.attention.wv.weight"],
									    f"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[f"layers.{layer_i}.attention.wo.weight"],
									    f"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w1.weight"],
									    f"model.layers.{layer_i}.mlp.down_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w2.weight"],
									    f"model.layers.{layer_i}.mlp.up_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w3.weight"],
									    f"model.layers.{layer_i}.input_layernorm.weight": loaded[f"layers.{layer_i}.attention_norm.weight"],
									    f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[f"layers.{layer_i}.ffn_norm.weight"],
									}
						else:
									# Sharded
									# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
									# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
									# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

									lowerCamelCase_					=  {
									    f"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
									        f"layers.{layer_i}.attention_norm.weight"
									    ].clone(),
									    f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
									        f"layers.{layer_i}.ffn_norm.weight"
									    ].clone(),
									}
									lowerCamelCase_					=  permute(
									    torch.cat(
									        [
									            loaded[i][f"layers.{layer_i}.attention.wq.weight"].view(lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__						)
									            for i in range(lowerCAmelCase__						)
									        ]	,dim=0	,).reshape(lowerCAmelCase__	,lowerCAmelCase__						)						)
									lowerCamelCase_					=  permute(
									    torch.cat(
									        [
									            loaded[i][f"layers.{layer_i}.attention.wk.weight"].view(
									                lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__						)
									            for i in range(lowerCAmelCase__						)
									        ]	,dim=0	,).reshape(lowerCAmelCase__	,lowerCAmelCase__						)	,lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__	,)
									lowerCamelCase_					=  torch.cat(
									    [
									        loaded[i][f"layers.{layer_i}.attention.wv.weight"].view(
									            lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__						)
									        for i in range(lowerCAmelCase__						)
									    ]	,dim=0	,).reshape(lowerCAmelCase__	,lowerCAmelCase__						)

									lowerCamelCase_					=  torch.cat(
									    [loaded[i][f"layers.{layer_i}.attention.wo.weight"] for i in range(lowerCAmelCase__						)]	,dim=1						)
									lowerCamelCase_					=  torch.cat(
									    [loaded[i][f"layers.{layer_i}.feed_forward.w1.weight"] for i in range(lowerCAmelCase__						)]	,dim=0						)
									lowerCamelCase_					=  torch.cat(
									    [loaded[i][f"layers.{layer_i}.feed_forward.w2.weight"] for i in range(lowerCAmelCase__						)]	,dim=1						)
									lowerCamelCase_					=  torch.cat(
									    [loaded[i][f"layers.{layer_i}.feed_forward.w3.weight"] for i in range(lowerCAmelCase__						)]	,dim=0						)

						lowerCamelCase_					=  inv_freq
						for k, v in state_dict.items():
									lowerCamelCase_					=  filename
									param_count += v.numel()
						torch.save(lowerCAmelCase__	,os.path.join(lowerCAmelCase__	,lowerCAmelCase__						)						)

			lowerCamelCase_					=  f"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
			if model_size == "7B":
						# Unsharded
						lowerCamelCase_					=  {
						    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
						    '''model.norm.weight''': loaded['''norm.weight'''],
						    '''lm_head.weight''': loaded['''output.weight'''],
						}
			else:
						lowerCamelCase_					=  {
						    '''model.norm.weight''': loaded[0]['''norm.weight'''],
						    '''model.embed_tokens.weight''': torch.cat(
						        [loaded[i]['''tok_embeddings.weight'''] for i in range(lowerCAmelCase__						)]	,dim=1						),
						    '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(lowerCAmelCase__						)]	,dim=0						),
						}

			for k, v in state_dict.items():
						lowerCamelCase_					=  filename
						param_count += v.numel()
			torch.save(lowerCAmelCase__	,os.path.join(lowerCAmelCase__	,lowerCAmelCase__						)						)

			# Write configs
			lowerCamelCase_					=  {'''total_size''': param_count * 2}
			write_json(lowerCAmelCase__	,os.path.join(lowerCAmelCase__	,'''pytorch_model.bin.index.json'''						)						)
			lowerCamelCase_					=  params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
			lowerCamelCase_					=  params['''multiple_of'''] if '''multiple_of''' in params else 256
			lowerCamelCase_					=  LlamaConfig(
			    hidden_size=lowerCAmelCase__	,intermediate_size=compute_intermediate_size(lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__						)	,num_attention_heads=params['''n_heads''']	,num_hidden_layers=params['''n_layers''']	,rms_norm_eps=params['''norm_eps''']	,num_key_value_heads=lowerCAmelCase__	,)
			config.save_pretrained(lowerCAmelCase__						)

			# Make space so we can load the model properly now.
			del state_dict
			del loaded
			gc.collect()

			print('''Loading the checkpoint in a Llama model.'''						)
			lowerCamelCase_					=  LlamaForCausalLM.from_pretrained(lowerCAmelCase__	,torch_dtype=torch.floataa	,low_cpu_mem_usage=lowerCAmelCase__						)
			# Avoid saving this as part of the config.
			del model.config._name_or_path

			print('''Saving in the Transformers format.'''						)
			model.save_pretrained(lowerCAmelCase__	,safe_serialization=lowerCAmelCase__						)
			shutil.rmtree(lowerCAmelCase__						)






def  lowercase						(	lowerCAmelCase__	,lowerCAmelCase__						):
			# Initialize the tokenizer based on the `spm` model
			lowerCamelCase_					=  LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
			print(f"Saving a {tokenizer_class.__name__} to {tokenizer_path}."						)
			lowerCamelCase_					=  tokenizer_class(lowerCAmelCase__						)
			tokenizer.save_pretrained(lowerCAmelCase__						)






def  lowercase						(	):
			lowerCamelCase_					=  argparse.ArgumentParser()
			parser.add_argument(
			    '''--input_dir'''	,help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''	,)
			parser.add_argument(
			    '''--model_size'''	,choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']	,)
			parser.add_argument(
			    '''--output_dir'''	,help='''Location to write HF model and tokenizer'''	,)
			parser.add_argument('''--safe_serialization'''	,type=lowerCAmelCase__	,help='''Whether or not to save using `safetensors`.'''						)
			lowerCamelCase_					=  parser.parse_args()
			if args.model_size != "tokenizer_only":
						write_model(
						    model_path=args.output_dir	,input_base_path=os.path.join(args.input_dir	,args.model_size						)	,model_size=args.model_size	,safe_serialization=args.safe_serialization	,)
			lowerCamelCase_					=  os.path.join(args.input_dir	,'''tokenizer.model'''						)
			write_tokenizer(args.output_dir	,lowerCAmelCase__						)


if __name__ == "__main__":
						main()







'''simple docstring'''
# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os

from ...utils.constants import SAGEMAKER_PARALLEL_EC2_INSTANCES, TORCH_DYNAMO_MODES
from ...utils.dataclasses import ComputeEnvironment, SageMakerDistributedType
from ...utils.imports import is_botoa_available
from .config_args import SageMakerConfig
from .config_utils import (
    DYNAMO_BACKENDS,
    _ask_field,
    _ask_options,
    _convert_dynamo_backend,
    _convert_mixed_precision,
    _convert_sagemaker_distributed_mode,
    _convert_yes_no_to_bool,
)


if is_botoa_available():
							import botoa  # noqa: F401







def       a_				(	UpperCamelCase_ ):
							A_			    =		botoa.client("iam" )

							A_			    =		{
							    "Version": "2012-10-17",
							    "Statement": [
							        {"Effect": "Allow", "Principal": {"Service": "sagemaker.amazonaws.com"}, "Action": "sts:AssumeRole"}
							    ],
							}
							try:
														# create the role, associated with the chosen trust policy
														iam_client.create_role(
														    RoleName=UpperCamelCase_						,  AssumeRolePolicyDocument=json.dumps(UpperCamelCase_						,  indent=2 ) )
														A_			    =		{
														    "Version": "2012-10-17",
														    "Statement": [
														        {
														            "Effect": "Allow",
														            "Action": [
														                "sagemaker:*",
														                "ecr:GetDownloadUrlForLayer",
														                "ecr:BatchGetImage",
														                "ecr:BatchCheckLayerAvailability",
														                "ecr:GetAuthorizationToken",
														                "cloudwatch:PutMetricData",
														                "cloudwatch:GetMetricData",
														                "cloudwatch:GetMetricStatistics",
														                "cloudwatch:ListMetrics",
														                "logs:CreateLogGroup",
														                "logs:CreateLogStream",
														                "logs:DescribeLogStreams",
														                "logs:PutLogEvents",
														                "logs:GetLogEvents",
														                "s3:CreateBucket",
														                "s3:ListBucket",
														                "s3:GetBucketLocation",
														                "s3:GetObject",
														                "s3:PutObject",
														            ],
														            "Resource": "*",
														        }
														    ],
														}
														# attach policy to role
														iam_client.put_role_policy(
														    RoleName=UpperCamelCase_						,  PolicyName=f"{role_name}_policy_permission"						,  PolicyDocument=json.dumps(UpperCamelCase_						,  indent=2 )						,  )
							except iam_client.exceptions.EntityAlreadyExistsException:
														print(f"role {role_name} already exists. Using existing one" )







def       a_				(	UpperCamelCase_ ):
							A_			    =		botoa.client("iam" )
							return iam_client.get_role(RoleName=UpperCamelCase_ )["Role"]["Arn"]







def       a_				(	):
							A_			    =		_ask_options(
							    "How do you want to authorize?"						,  ["AWS Profile", "Credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) "]						,  UpperCamelCase_						,  )
							A_			    =		None
							if credentials_configuration == 0:
														A_			    =		_ask_field("Enter your AWS Profile name: [default] "						,  default="default" )
														A_			    =		aws_profile
							else:
														print(
														    "Note you will need to provide AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY when you launch you training script with,"
														    "`accelerate launch --aws_access_key_id XXX --aws_secret_access_key YYY`" )
														A_			    =		_ask_field("AWS Access Key ID: " )
														A_			    =		aws_access_key_id

														A_			    =		_ask_field("AWS Secret Access Key: " )
														A_			    =		aws_secret_access_key

							A_			    =		_ask_field("Enter your AWS Region: [us-east-1]"						,  default="us-east-1" )
							A_			    =		aws_region

							A_			    =		_ask_options(
							    "Do you already have an IAM Role for executing Amazon SageMaker Training Jobs?"						,  ["Provide IAM Role name", "Create new IAM role using credentials"]						,  UpperCamelCase_						,  )
							if role_management == 0:
														A_			    =		_ask_field("Enter your IAM role name: " )
							else:
														A_			    =		"accelerate_sagemaker_execution_role"
														print(f"Accelerate will create an iam role \"{iam_role_name}\" using the provided credentials" )
														_create_iam_role_for_sagemaker(UpperCamelCase_ )

							A_			    =		_ask_field(
							    "Do you want to use custom Docker image? [yes/NO]: "						,  _convert_yes_no_to_bool						,  default=UpperCamelCase_						,  error_message="Please enter yes or no."						,  )
							A_			    =		None
							if is_custom_docker_image:
														A_			    =		_ask_field("Enter your Docker image: "						,  lambda UpperCamelCase_ : str(UpperCamelCase_ ).lower() )

							A_			    =		_ask_field(
							    "Do you want to provide SageMaker input channels with data locations? [yes/NO]: "						,  _convert_yes_no_to_bool						,  default=UpperCamelCase_						,  error_message="Please enter yes or no."						,  )
							A_			    =		None
							if is_sagemaker_inputs_enabled:
														A_			    =		_ask_field(
														    "Enter the path to the SageMaker inputs TSV file with columns (channel_name, data_location): "						,  lambda UpperCamelCase_ : str(UpperCamelCase_ ).lower()						,  )

							A_			    =		_ask_field(
							    "Do you want to enable SageMaker metrics? [yes/NO]: "						,  _convert_yes_no_to_bool						,  default=UpperCamelCase_						,  error_message="Please enter yes or no."						,  )
							A_			    =		None
							if is_sagemaker_metrics_enabled:
														A_			    =		_ask_field(
														    "Enter the path to the SageMaker metrics TSV file with columns (metric_name, metric_regex): "						,  lambda UpperCamelCase_ : str(UpperCamelCase_ ).lower()						,  )

							A_			    =		_ask_options(
							    "What is the distributed mode?"						,  ["No distributed training", "Data parallelism"]						,  _convert_sagemaker_distributed_mode						,  )
							A_			    =		{}
							A_			    =		_ask_field(
							    "Do you wish to optimize your script with torch dynamo?[yes/NO]:"						,  _convert_yes_no_to_bool						,  default=UpperCamelCase_						,  error_message="Please enter yes or no."						,  )
							if use_dynamo:
														A_			    =		"dynamo_"
														A_			    =		_ask_options(
														    "Which dynamo backend would you like to use?"						,  [x.lower() for x in DYNAMO_BACKENDS]						,  _convert_dynamo_backend						,  default=2						,  )
														A_			    =		_ask_field(
														    "Do you want to customize the defaults sent to torch.compile? [yes/NO]: "						,  _convert_yes_no_to_bool						,  default=UpperCamelCase_						,  error_message="Please enter yes or no."						,  )

														if use_custom_options:
																					A_			    =		_ask_options(
																					    "Which mode do you want to use?"						,  UpperCamelCase_						,  lambda UpperCamelCase_ : TORCH_DYNAMO_MODES[int(UpperCamelCase_ )]						,  default="default"						,  )
																					A_			    =		_ask_field(
																					    "Do you want the fullgraph mode or it is ok to break model into several subgraphs? [yes/NO]: "						,  _convert_yes_no_to_bool						,  default=UpperCamelCase_						,  error_message="Please enter yes or no."						,  )
																					A_			    =		_ask_field(
																					    "Do you want to enable dynamic shape tracing? [yes/NO]: "						,  _convert_yes_no_to_bool						,  default=UpperCamelCase_						,  error_message="Please enter yes or no."						,  )
							A_			    =		"Which EC2 instance type you want to use for your training?"
							if distributed_type != SageMakerDistributedType.NO:
														A_			    =		_ask_options(
														    UpperCamelCase_						,  UpperCamelCase_						,  lambda UpperCamelCase_ : SAGEMAKER_PARALLEL_EC2_INSTANCES[int(UpperCamelCase_ )] )
							else:
														eca_instance_query += "? [ml.p3.2xlarge]:"
														A_			    =		_ask_field(UpperCamelCase_						,  lambda UpperCamelCase_ : str(UpperCamelCase_ ).lower()						,  default="ml.p3.2xlarge" )

							A_			    =		1
							if distributed_type in (SageMakerDistributedType.DATA_PARALLEL, SageMakerDistributedType.MODEL_PARALLEL):
														A_			    =		_ask_field(
														    "How many machines do you want use? [1]: "						,  UpperCamelCase_						,  default=1						,  )

							A_			    =		_ask_options(
							    "Do you wish to use FP16 or BF16 (mixed precision)?"						,  ["no", "fp16", "bf16", "fp8"]						,  _convert_mixed_precision						,  )

							if use_dynamo and mixed_precision == "no":
														print(
														    "Torch dynamo used without mixed precision requires TF32 to be efficient. Accelerate will enable it by default when launching your scripts." )

							return SageMakerConfig(
							    image_uri=UpperCamelCase_						,  compute_environment=ComputeEnvironment.AMAZON_SAGEMAKER						,  distributed_type=UpperCamelCase_						,  use_cpu=UpperCamelCase_						,  dynamo_config=UpperCamelCase_						,  eca_instance_type=UpperCamelCase_						,  profile=UpperCamelCase_						,  region=UpperCamelCase_						,  iam_role_name=UpperCamelCase_						,  mixed_precision=UpperCamelCase_						,  num_machines=UpperCamelCase_						,  sagemaker_inputs_file=UpperCamelCase_						,  sagemaker_metrics_file=UpperCamelCase_						,  )





