
import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
   from transformers import LlamaTokenizerFast
except ImportError as e:
   warnings.warn(e)
   warnings.warn(
       '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
   )
   SCREAMING_SNAKE_CASE_    					=						None
SCREAMING_SNAKE_CASE_    					=						{
    '''7B''': 1_1008,
    '''13B''': 1_3824,
    '''30B''': 1_7920,
    '''65B''': 2_2016,
    '''70B''': 2_8672,
}
SCREAMING_SNAKE_CASE_    					=						{
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}




def     lowercase__						(					lowerCAmelCase				:					Optional[Any] ,				lowerCAmelCase				:					Tuple=1 ,				lowerCAmelCase				:					Dict=256 ) ->      Union[str, Any]:






    """simple docstring"""




    return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3 ) ) + multiple_of - 1) // multiple_of)




def     lowercase__						(					lowerCAmelCase				:					Any ) ->      str:






    """simple docstring"""




    with open(lowerCAmelCase_ ,				'r' ) as f:
        return json.load(lowerCAmelCase_ )




def     lowercase__						(					lowerCAmelCase				:					int ,				lowerCAmelCase				:					Union[str, Any] ) ->      List[str]:






    """simple docstring"""




    with open(lowerCAmelCase_ ,				'w' ) as f:
        json.dump(lowerCAmelCase_ ,				lowerCAmelCase_ )




def     lowercase__						(					lowerCAmelCase				:					Any ,				lowerCAmelCase				:					List[str] ,				lowerCAmelCase				:					List[Any] ,				lowerCAmelCase				:					Tuple=True ) ->      List[Any]:






    """simple docstring"""




    os.makedirs(lowerCAmelCase_ ,				exist_ok=lowerCAmelCase_ )
    UpperCAmelCase									=					os.path.join(lowerCAmelCase_ ,				'tmp' )
    os.makedirs(lowerCAmelCase_ ,				exist_ok=lowerCAmelCase_ )

    UpperCAmelCase									=					read_json(os.path.join(lowerCAmelCase_ ,				'params.json' ) )
    UpperCAmelCase									=					NUM_SHARDS[model_size]
    UpperCAmelCase									=					params["n_layers"]
    UpperCAmelCase									=					params["n_heads"]
    UpperCAmelCase									=					n_heads // num_shards
    UpperCAmelCase									=					params["dim"]
    UpperCAmelCase									=					dim // n_heads
    UpperCAmelCase									=					10_000.0
    UpperCAmelCase									=					1.0 / (base ** (torch.arange(0 ,				lowerCAmelCase_ ,				2 ).float() / dims_per_head))

    if "n_kv_heads" in params:
        UpperCAmelCase									=					params["n_kv_heads"]  # for GQA / MQA
        UpperCAmelCase									=					n_heads_per_shard // num_key_value_heads
        UpperCAmelCase									=					dim // num_key_value_heads
    else:  # compatibility with other checkpoints
        UpperCAmelCase									=					n_heads
        UpperCAmelCase									=					n_heads_per_shard
        UpperCAmelCase									=					dim

    # permute for sliced rotary
    def permute(lowerCAmelCase				:					Optional[Any] ,				lowerCAmelCase				:					Dict=n_heads ,				lowerCAmelCase				:					Optional[int]=dim ,				lowerCAmelCase				:					Optional[Any]=dim ):
        return w.view(lowerCAmelCase_ ,				dima // n_heads // 2 ,				2 ,				lowerCAmelCase_ ).transpose(1 ,				2 ).reshape(lowerCAmelCase_ ,				lowerCAmelCase_ )

    print(F"Fetching all parameters from the checkpoint at {input_base_path}." )
    # Load weights
    if model_size == "7B":
        # Not sharded
        # (The sharded implementation would also work, but this is simpler.)
        UpperCAmelCase									=					torch.load(os.path.join(lowerCAmelCase_ ,				'consolidated.00.pth' ) ,				map_location='cpu' )
    else:
        # Sharded
        UpperCAmelCase									=					[
            torch.load(os.path.join(lowerCAmelCase_ ,				F"consolidated.{i:02d}.pth" ) ,				map_location='cpu' )
            for i in range(lowerCAmelCase_ )
        ]
    UpperCAmelCase									=					0
    UpperCAmelCase									=					{"weight_map": {}}
    for layer_i in range(lowerCAmelCase_ ):
        UpperCAmelCase									=					F"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
        if model_size == "7B":
            # Unsharded
            UpperCAmelCase									=					{
                F"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
                    loaded[F"layers.{layer_i}.attention.wq.weight"] ),
                F"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
                    loaded[F"layers.{layer_i}.attention.wk.weight"] ),
                F"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[F"layers.{layer_i}.attention.wv.weight"],
                F"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[F"layers.{layer_i}.attention.wo.weight"],
                F"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w1.weight"],
                F"model.layers.{layer_i}.mlp.down_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w2.weight"],
                F"model.layers.{layer_i}.mlp.up_proj.weight": loaded[F"layers.{layer_i}.feed_forward.w3.weight"],
                F"model.layers.{layer_i}.input_layernorm.weight": loaded[F"layers.{layer_i}.attention_norm.weight"],
                F"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[F"layers.{layer_i}.ffn_norm.weight"],
            }
        else:
            # Sharded
            # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
            # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
            # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

            UpperCAmelCase									=					{
                F"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
                    F"layers.{layer_i}.attention_norm.weight"
                ].clone(),
                F"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
                    F"layers.{layer_i}.ffn_norm.weight"
                ].clone(),
            }
            UpperCAmelCase									=					permute(
                torch.cat(
                    [
                        loaded[i][F"layers.{layer_i}.attention.wq.weight"].view(lowerCAmelCase_ ,				lowerCAmelCase_ ,				lowerCAmelCase_ )
                        for i in range(lowerCAmelCase_ )
                    ] ,				dim=0 ,				).reshape(lowerCAmelCase_ ,				lowerCAmelCase_ ) )
            UpperCAmelCase									=					permute(
                torch.cat(
                    [
                        loaded[i][F"layers.{layer_i}.attention.wk.weight"].view(
                            lowerCAmelCase_ ,				lowerCAmelCase_ ,				lowerCAmelCase_ )
                        for i in range(lowerCAmelCase_ )
                    ] ,				dim=0 ,				).reshape(lowerCAmelCase_ ,				lowerCAmelCase_ ) ,				lowerCAmelCase_ ,				lowerCAmelCase_ ,				lowerCAmelCase_ ,				)
            UpperCAmelCase									=					torch.cat(
                [
                    loaded[i][F"layers.{layer_i}.attention.wv.weight"].view(
                        lowerCAmelCase_ ,				lowerCAmelCase_ ,				lowerCAmelCase_ )
                    for i in range(lowerCAmelCase_ )
                ] ,				dim=0 ,				).reshape(lowerCAmelCase_ ,				lowerCAmelCase_ )

            UpperCAmelCase									=					torch.cat(
                [loaded[i][F"layers.{layer_i}.attention.wo.weight"] for i in range(lowerCAmelCase_ )] ,				dim=1 )
            UpperCAmelCase									=					torch.cat(
                [loaded[i][F"layers.{layer_i}.feed_forward.w1.weight"] for i in range(lowerCAmelCase_ )] ,				dim=0 )
            UpperCAmelCase									=					torch.cat(
                [loaded[i][F"layers.{layer_i}.feed_forward.w2.weight"] for i in range(lowerCAmelCase_ )] ,				dim=1 )
            UpperCAmelCase									=					torch.cat(
                [loaded[i][F"layers.{layer_i}.feed_forward.w3.weight"] for i in range(lowerCAmelCase_ )] ,				dim=0 )

        UpperCAmelCase									=					inv_freq
        for k, v in state_dict.items():
            UpperCAmelCase									=					filename
            param_count += v.numel()
        torch.save(lowerCAmelCase_ ,				os.path.join(lowerCAmelCase_ ,				lowerCAmelCase_ ) )

    UpperCAmelCase									=					F"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
    if model_size == "7B":
        # Unsharded
        UpperCAmelCase									=					{
            "model.embed_tokens.weight": loaded["tok_embeddings.weight"],
            "model.norm.weight": loaded["norm.weight"],
            "lm_head.weight": loaded["output.weight"],
        }
    else:
        UpperCAmelCase									=					{
            "model.norm.weight": loaded[0]["norm.weight"],
            "model.embed_tokens.weight": torch.cat(
                [loaded[i]['tok_embeddings.weight'] for i in range(lowerCAmelCase_ )] ,				dim=1 ),
            "lm_head.weight": torch.cat([loaded[i]['output.weight'] for i in range(lowerCAmelCase_ )] ,				dim=0 ),
        }

    for k, v in state_dict.items():
        UpperCAmelCase									=					filename
        param_count += v.numel()
    torch.save(lowerCAmelCase_ ,				os.path.join(lowerCAmelCase_ ,				lowerCAmelCase_ ) )

    # Write configs
    UpperCAmelCase									=					{"total_size": param_count * 2}
    write_json(lowerCAmelCase_ ,				os.path.join(lowerCAmelCase_ ,				'pytorch_model.bin.index.json' ) )
    UpperCAmelCase									=					params["ffn_dim_multiplier"] if "ffn_dim_multiplier" in params else 1
    UpperCAmelCase									=					params["multiple_of"] if "multiple_of" in params else 256
    UpperCAmelCase									=					LlamaConfig(
        hidden_size=lowerCAmelCase_ ,				intermediate_size=compute_intermediate_size(lowerCAmelCase_ ,				lowerCAmelCase_ ,				lowerCAmelCase_ ) ,				num_attention_heads=params['n_heads'] ,				num_hidden_layers=params['n_layers'] ,				rms_norm_eps=params['norm_eps'] ,				num_key_value_heads=lowerCAmelCase_ ,				)
    config.save_pretrained(lowerCAmelCase_ )

    # Make space so we can load the model properly now.
    del state_dict
    del loaded
    gc.collect()

    print('Loading the checkpoint in a Llama model.' )
    UpperCAmelCase									=					LlamaForCausalLM.from_pretrained(lowerCAmelCase_ ,				torch_dtype=torch.floataa ,				low_cpu_mem_usage=lowerCAmelCase_ )
    # Avoid saving this as part of the config.
    del model.config._name_or_path

    print('Saving in the Transformers format.' )
    model.save_pretrained(lowerCAmelCase_ ,				safe_serialization=lowerCAmelCase_ )
    shutil.rmtree(lowerCAmelCase_ )




def     lowercase__						(					lowerCAmelCase				:					Tuple ,				lowerCAmelCase				:					List[str] ) ->      Union[str, Any]:






    """simple docstring"""




    UpperCAmelCase									=					LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
    print(F"Saving a {tokenizer_class.__name__} to {tokenizer_path}." )
    UpperCAmelCase									=					tokenizer_class(lowerCAmelCase_ )
    tokenizer.save_pretrained(lowerCAmelCase_ )




def     lowercase__						(					) ->      str:






    """simple docstring"""




    UpperCAmelCase									=					argparse.ArgumentParser()
    parser.add_argument(
        '--input_dir' ,				help='Location of LLaMA weights, which contains tokenizer.model and model folders' ,				)
    parser.add_argument(
        '--model_size' ,				choices=['7B', '7Bf', '13B', '13Bf', '30B', '65B', '70B', '70Bf', 'tokenizer_only'] ,				)
    parser.add_argument(
        '--output_dir' ,				help='Location to write HF model and tokenizer' ,				)
    parser.add_argument('--safe_serialization' ,				type=lowerCAmelCase_ ,				help='Whether or not to save using `safetensors`.' )
    UpperCAmelCase									=					parser.parse_args()
    if args.model_size != "tokenizer_only":
        write_model(
            model_path=args.output_dir ,				input_base_path=os.path.join(args.input_dir ,				args.model_size ) ,				model_size=args.model_size ,				safe_serialization=args.safe_serialization ,				)
    UpperCAmelCase									=					os.path.join(args.input_dir ,				'tokenizer.model' )
    write_tokenizer(args.output_dir ,				lowerCAmelCase_ )


if __name__ == "__main__":
   main()





"""simple docstring"""

from __future__ import annotations

import numpy as np




def     lowercase__						(					lowerCAmelCase				:					list[float] ) ->      Dict:






    """simple docstring"""




    return np.maximum(0 ,				lowerCAmelCase )


if __name__ == "__main__":
   print(np.array(relu([-1, 0, 5])))  # --> [0, 0, 5]



