'''simple docstring'''




import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
  from transformers import LlamaTokenizerFast
except ImportError as e:
  warnings.warn(e)
  warnings.warn(
      """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
  )
  __SCREAMING_SNAKE_CASE			:	Tuple	=    None
__SCREAMING_SNAKE_CASE			:	Any	=    {
    """7B""": 11_008,
    """13B""": 13_824,
    """30B""": 17_920,
    """65B""": 22_016,
    """70B""": 28_672,
}
__SCREAMING_SNAKE_CASE			:	Optional[Any]	=    {
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}

def       UpperCamelCase_				(   _UpperCAmelCase    : Tuple     ,      _UpperCAmelCase    : List[str]=1     ,      _UpperCAmelCase    : List[str]=256       )       ->       Optional[Any]:


       """simple docstring"""







       return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3       )       ) + multiple_of - 1) // multiple_of)

def       UpperCamelCase_				(   _UpperCAmelCase    : int       )       ->       Optional[Any]:


       """simple docstring"""







       with open(_UpperCAmelCase     ,      "r"       ) as f:
              return json.load(_UpperCAmelCase       )

def       UpperCamelCase_				(   _UpperCAmelCase    : int     ,      _UpperCAmelCase    : List[str]       )       ->       str:


       """simple docstring"""







       with open(_UpperCAmelCase     ,      "w"       ) as f:
              json.dump(_UpperCAmelCase     ,      _UpperCAmelCase       )

def       UpperCamelCase_				(   _UpperCAmelCase    : List[str]     ,      _UpperCAmelCase    : Any     ,      _UpperCAmelCase    : Union[str, Any]     ,      _UpperCAmelCase    : Tuple=True       )       ->       Optional[int]:


       """simple docstring"""







       os.makedirs(_UpperCAmelCase     ,      exist_ok=_UpperCAmelCase       )
       _UpperCAmelCase	:    List[str]				 =       os.path.join(_UpperCAmelCase     ,      "tmp"       )
       os.makedirs(_UpperCAmelCase     ,      exist_ok=_UpperCAmelCase       )

       _UpperCAmelCase	:    str				 =       read_json(os.path.join(_UpperCAmelCase     ,      "params.json"       )       )
       _UpperCAmelCase	:    Optional[int]				 =       NUM_SHARDS[model_size]
       _UpperCAmelCase	:    Tuple				 =       params["n_layers"]
       _UpperCAmelCase	:    Tuple				 =       params["n_heads"]
       _UpperCAmelCase	:    Optional[Any]				 =       n_heads // num_shards
       _UpperCAmelCase	:    str				 =       params["dim"]
       _UpperCAmelCase	:    int				 =       dim // n_heads
       _UpperCAmelCase	:    List[str]				 =       10_000.0
       _UpperCAmelCase	:    List[str]				 =       1.0 / (base ** (torch.arange(0     ,      _UpperCAmelCase     ,      2       ).float() / dims_per_head))

       if "n_kv_heads" in params:
              _UpperCAmelCase	:    Dict				 =       params["n_kv_heads"]  # for GQA / MQA
              _UpperCAmelCase	:    int				 =       n_heads_per_shard // num_key_value_heads
              _UpperCAmelCase	:    Union[str, Any]				 =       dim // num_key_value_heads
       else:  # compatibility with other checkpoints
              _UpperCAmelCase	:    Dict				 =       n_heads
              _UpperCAmelCase	:    Optional[int]				 =       n_heads_per_shard
              _UpperCAmelCase	:    Tuple				 =       dim

       # permute for sliced rotary
       def permute(_UpperCAmelCase    : Optional[int]     ,      _UpperCAmelCase    : Any=n_heads     ,      _UpperCAmelCase    : Optional[int]=dim     ,      _UpperCAmelCase    : Any=dim       ):
              return w.view(_UpperCAmelCase     ,      dima // n_heads // 2     ,      2     ,      _UpperCAmelCase       ).transpose(1     ,      2       ).reshape(_UpperCAmelCase     ,      _UpperCAmelCase       )

       print(F"""Fetching all parameters from the checkpoint at {input_base_path}."""       )
       # Load weights
       if model_size == "7B":
              # Not sharded
              # (The sharded implementation would also work, but this is simpler.)
              _UpperCAmelCase	:    List[Any]				 =       torch.load(os.path.join(_UpperCAmelCase     ,      "consolidated.00.pth"       )     ,      map_location="cpu"       )
       else:
              # Sharded
              _UpperCAmelCase	:    Any				 =       [
                  torch.load(os.path.join(_UpperCAmelCase     ,      F"""consolidated.{i:02d}.pth"""       )     ,      map_location="cpu"       )
                  for i in range(_UpperCAmelCase       )
              ]
       _UpperCAmelCase	:    int				 =       0
       _UpperCAmelCase	:    Dict				 =       {"weight_map": {}}
       for layer_i in range(_UpperCAmelCase       ):
              _UpperCAmelCase	:    Union[str, Any]				 =       F"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
              if model_size == "7B":
                     # Unsharded
                     _UpperCAmelCase	:    int				 =       {
                         F"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
                             loaded[F"""layers.{layer_i}.attention.wq.weight"""]       ),
                         F"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
                             loaded[F"""layers.{layer_i}.attention.wk.weight"""]       ),
                         F"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[F"""layers.{layer_i}.attention.wv.weight"""],
                         F"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[F"""layers.{layer_i}.attention.wo.weight"""],
                         F"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w1.weight"""],
                         F"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w2.weight"""],
                         F"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w3.weight"""],
                         F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[F"""layers.{layer_i}.attention_norm.weight"""],
                         F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[F"""layers.{layer_i}.ffn_norm.weight"""],
                     }
              else:
                     # Sharded
                     # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
                     # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
                     # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

                     _UpperCAmelCase	:    int				 =       {
                         F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
                             F"""layers.{layer_i}.attention_norm.weight"""
                         ].clone(),
                         F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
                             F"""layers.{layer_i}.ffn_norm.weight"""
                         ].clone(),
                     }
                     _UpperCAmelCase	:    Dict				 =       permute(
                         torch.cat(
                             [
                                 loaded[i][F"""layers.{layer_i}.attention.wq.weight"""].view(_UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase       )
                                 for i in range(_UpperCAmelCase       )
                             ]     ,      dim=0     ,      ).reshape(_UpperCAmelCase     ,      _UpperCAmelCase       )       )
                     _UpperCAmelCase	:    List[Any]				 =       permute(
                         torch.cat(
                             [
                                 loaded[i][F"""layers.{layer_i}.attention.wk.weight"""].view(
                                     _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase       )
                                 for i in range(_UpperCAmelCase       )
                             ]     ,      dim=0     ,      ).reshape(_UpperCAmelCase     ,      _UpperCAmelCase       )     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      )
                     _UpperCAmelCase	:    Union[str, Any]				 =       torch.cat(
                         [
                             loaded[i][F"""layers.{layer_i}.attention.wv.weight"""].view(
                                 _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase       )
                             for i in range(_UpperCAmelCase       )
                         ]     ,      dim=0     ,      ).reshape(_UpperCAmelCase     ,      _UpperCAmelCase       )

                     _UpperCAmelCase	:    Tuple				 =       torch.cat(
                         [loaded[i][F"""layers.{layer_i}.attention.wo.weight"""] for i in range(_UpperCAmelCase       )]     ,      dim=1       )
                     _UpperCAmelCase	:    Dict				 =       torch.cat(
                         [loaded[i][F"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(_UpperCAmelCase       )]     ,      dim=0       )
                     _UpperCAmelCase	:    Optional[int]				 =       torch.cat(
                         [loaded[i][F"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(_UpperCAmelCase       )]     ,      dim=1       )
                     _UpperCAmelCase	:    Dict				 =       torch.cat(
                         [loaded[i][F"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(_UpperCAmelCase       )]     ,      dim=0       )

              _UpperCAmelCase	:    Union[str, Any]				 =       inv_freq
              for k, v in state_dict.items():
                     _UpperCAmelCase	:    str				 =       filename
                     param_count += v.numel()
              torch.save(_UpperCAmelCase     ,      os.path.join(_UpperCAmelCase     ,      _UpperCAmelCase       )       )

       _UpperCAmelCase	:    Optional[Any]				 =       F"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
       if model_size == "7B":
              # Unsharded
              _UpperCAmelCase	:    str				 =       {
                  "model.embed_tokens.weight": loaded["tok_embeddings.weight"],
                  "model.norm.weight": loaded["norm.weight"],
                  "lm_head.weight": loaded["output.weight"],
              }
       else:
              _UpperCAmelCase	:    Optional[int]				 =       {
                  "model.norm.weight": loaded[0]["norm.weight"],
                  "model.embed_tokens.weight": torch.cat(
                      [loaded[i]["tok_embeddings.weight"] for i in range(_UpperCAmelCase       )]     ,      dim=1       ),
                  "lm_head.weight": torch.cat([loaded[i]["output.weight"] for i in range(_UpperCAmelCase       )]     ,      dim=0       ),
              }

       for k, v in state_dict.items():
              _UpperCAmelCase	:    int				 =       filename
              param_count += v.numel()
       torch.save(_UpperCAmelCase     ,      os.path.join(_UpperCAmelCase     ,      _UpperCAmelCase       )       )

       # Write configs
       _UpperCAmelCase	:    Tuple				 =       {"total_size": param_count * 2}
       write_json(_UpperCAmelCase     ,      os.path.join(_UpperCAmelCase     ,      "pytorch_model.bin.index.json"       )       )
       _UpperCAmelCase	:    int				 =       params["ffn_dim_multiplier"] if "ffn_dim_multiplier" in params else 1
       _UpperCAmelCase	:    str				 =       params["multiple_of"] if "multiple_of" in params else 256
       _UpperCAmelCase	:    Dict				 =       LlamaConfig(
           hidden_size=_UpperCAmelCase     ,      intermediate_size=compute_intermediate_size(_UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase       )     ,      num_attention_heads=params["n_heads"]     ,      num_hidden_layers=params["n_layers"]     ,      rms_norm_eps=params["norm_eps"]     ,      num_key_value_heads=_UpperCAmelCase     ,      )
       config.save_pretrained(_UpperCAmelCase       )

       # Make space so we can load the model properly now.
       del state_dict
       del loaded
       gc.collect()

       print("Loading the checkpoint in a Llama model."       )
       _UpperCAmelCase	:    Any				 =       LlamaForCausalLM.from_pretrained(_UpperCAmelCase     ,      torch_dtype=torch.floataa     ,      low_cpu_mem_usage=_UpperCAmelCase       )
       # Avoid saving this as part of the config.
       del model.config._name_or_path

       print("Saving in the Transformers format."       )
       model.save_pretrained(_UpperCAmelCase     ,      safe_serialization=_UpperCAmelCase       )
       shutil.rmtree(_UpperCAmelCase       )

def       UpperCamelCase_				(   _UpperCAmelCase    : List[Any]     ,      _UpperCAmelCase    : Optional[Any]       )       ->       Tuple:


       """simple docstring"""







       _UpperCAmelCase	:    Tuple				 =       LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
       print(F"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""       )
       _UpperCAmelCase	:    str				 =       tokenizer_class(_UpperCAmelCase       )
       tokenizer.save_pretrained(_UpperCAmelCase       )

def       UpperCamelCase_				(   )       ->       Optional[int]:


       """simple docstring"""







       _UpperCAmelCase	:    str				 =       argparse.ArgumentParser()
       parser.add_argument(
           "--input_dir"     ,      help="Location of LLaMA weights, which contains tokenizer.model and model folders"     ,      )
       parser.add_argument(
           "--model_size"     ,      choices=["7B", "7Bf", "13B", "13Bf", "30B", "65B", "70B", "70Bf", "tokenizer_only"]     ,      )
       parser.add_argument(
           "--output_dir"     ,      help="Location to write HF model and tokenizer"     ,      )
       parser.add_argument("--safe_serialization"     ,      type=_UpperCAmelCase     ,      help="Whether or not to save using `safetensors`."       )
       _UpperCAmelCase	:    str				 =       parser.parse_args()
       if args.model_size != "tokenizer_only":
              write_model(
                  model_path=args.output_dir     ,      input_base_path=os.path.join(args.input_dir     ,      args.model_size       )     ,      model_size=args.model_size     ,      safe_serialization=args.safe_serialization     ,      )
       _UpperCAmelCase	:    Optional[Any]				 =       os.path.join(args.input_dir     ,      "tokenizer.model"       )
       write_tokenizer(args.output_dir     ,      _UpperCAmelCase       )


if __name__ == "__main__":
  main()




def 			_A	(      SCREAMING_SNAKE_CASE      ):
	UpperCAmelCase__:	Tuple									= int(SCREAMING_SNAKE_CASE      )
	if decimal in (0, 1):  # Exit cases for the recursion
		return str(SCREAMING_SNAKE_CASE      )
	UpperCAmelCase__							,						UpperCAmelCase__:	Union[str, Any]									= divmod(SCREAMING_SNAKE_CASE	,2      )
	return binary_recursive(SCREAMING_SNAKE_CASE      ) + str(SCREAMING_SNAKE_CASE      )



def 			_A	(      SCREAMING_SNAKE_CASE      ):
	UpperCAmelCase__:	List[str]									= str(SCREAMING_SNAKE_CASE      ).strip()
	if not number:
		raise ValueError("No input value was provided"      )
	UpperCAmelCase__:	Tuple									= "-" if number.startswith("-"      ) else ""
	UpperCAmelCase__:	Any									= number.lstrip("-"      )
	if not number.isnumeric():
		raise ValueError("Input value is not an integer"      )
	return f"{negative}0b{binary_recursive(int(SCREAMING_SNAKE_CASE      )      )}"


if __name__ == "__main__":
					from doctest import testmod

					testmod()