


import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
   from transformers import LlamaTokenizerFast
except ImportError as e:
   warnings.warn(e)
   warnings.warn(
       """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
   )
   UpperCAmelCase    =					None
UpperCAmelCase    =					{
    """7B""": 1_1008,
    """13B""": 1_3824,
    """30B""": 1_7920,
    """65B""": 2_2016,
    """70B""": 2_8672,
}
UpperCAmelCase    =					{
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}






def  __lowerCAmelCase					(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE=1      ,      SCREAMING_SNAKE_CASE=256      )->						Dict:




      """simple docstring"""

      return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3      )      ) + multiple_of - 1) // multiple_of)






def  __lowerCAmelCase					(SCREAMING_SNAKE_CASE      )->						Any:




      """simple docstring"""

      with open(SCREAMING_SNAKE_CASE      ,      '''r'''      ) as f:
            return json.load(SCREAMING_SNAKE_CASE      )






def  __lowerCAmelCase					(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      )->						Optional[int]:




      """simple docstring"""

      with open(SCREAMING_SNAKE_CASE      ,      '''w'''      ) as f:
            json.dump(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      )






def  __lowerCAmelCase					(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE=True      )->						List[Any]:




      """simple docstring"""

      os.makedirs(SCREAMING_SNAKE_CASE      ,      exist_ok=SCREAMING_SNAKE_CASE      )
      snake_case_ =   os.path.join(SCREAMING_SNAKE_CASE      ,      '''tmp'''      )
      os.makedirs(SCREAMING_SNAKE_CASE      ,      exist_ok=SCREAMING_SNAKE_CASE      )

      snake_case_ =   read_json(os.path.join(SCREAMING_SNAKE_CASE      ,      '''params.json'''      )      )
      snake_case_ =   NUM_SHARDS[model_size]
      snake_case_ =   params['''n_layers''']
      snake_case_ =   params['''n_heads''']
      snake_case_ =   n_heads // num_shards
      snake_case_ =   params['''dim''']
      snake_case_ =   dim // n_heads
      snake_case_ =   1_0_0_0_0.0
      snake_case_ =   1.0 / (base ** (torch.arange(0      ,      SCREAMING_SNAKE_CASE      ,      2      ).float() / dims_per_head))

      if "n_kv_heads" in params:
            snake_case_ =   params['''n_kv_heads''']  # for GQA / MQA
            snake_case_ =   n_heads_per_shard // num_key_value_heads
            snake_case_ =   dim // num_key_value_heads
      else:  # compatibility with other checkpoints
            snake_case_ =   n_heads
            snake_case_ =   n_heads_per_shard
            snake_case_ =   dim

      # permute for sliced rotary
      def permute(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE=n_heads      ,      SCREAMING_SNAKE_CASE=dim      ,      SCREAMING_SNAKE_CASE=dim      ):
            return w.view(SCREAMING_SNAKE_CASE      ,      dima // n_heads // 2      ,      2      ,      SCREAMING_SNAKE_CASE      ).transpose(1      ,      2      ).reshape(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      )

      print(f'''Fetching all parameters from the checkpoint at {input_base_path}.'''      )
      # Load weights
      if model_size == "7B":
            # Not sharded
            # (The sharded implementation would also work, but this is simpler.)
            snake_case_ =   torch.load(os.path.join(SCREAMING_SNAKE_CASE      ,      '''consolidated.00.pth'''      )      ,      map_location='''cpu'''      )
      else:
            # Sharded
            snake_case_ =   [
                torch.load(os.path.join(SCREAMING_SNAKE_CASE      ,      f'''consolidated.{i:02d}.pth'''      )      ,      map_location='''cpu'''      )
                for i in range(SCREAMING_SNAKE_CASE      )
            ]
      snake_case_ =   0
      snake_case_ =   {'''weight_map''': {}}
      for layer_i in range(SCREAMING_SNAKE_CASE      ):
            snake_case_ =   f'''pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'''
            if model_size == "7B":
                  # Unsharded
                  snake_case_ =   {
                      f'''model.layers.{layer_i}.self_attn.q_proj.weight''': permute(
                          loaded[f'''layers.{layer_i}.attention.wq.weight''']      ),
                      f'''model.layers.{layer_i}.self_attn.k_proj.weight''': permute(
                          loaded[f'''layers.{layer_i}.attention.wk.weight''']      ),
                      f'''model.layers.{layer_i}.self_attn.v_proj.weight''': loaded[f'''layers.{layer_i}.attention.wv.weight'''],
                      f'''model.layers.{layer_i}.self_attn.o_proj.weight''': loaded[f'''layers.{layer_i}.attention.wo.weight'''],
                      f'''model.layers.{layer_i}.mlp.gate_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w1.weight'''],
                      f'''model.layers.{layer_i}.mlp.down_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w2.weight'''],
                      f'''model.layers.{layer_i}.mlp.up_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w3.weight'''],
                      f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[f'''layers.{layer_i}.attention_norm.weight'''],
                      f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[f'''layers.{layer_i}.ffn_norm.weight'''],
                  }
            else:
                  # Sharded
                  # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
                  # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
                  # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

                  snake_case_ =   {
                      f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[0][
                          f'''layers.{layer_i}.attention_norm.weight'''
                      ].clone(),
                      f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[0][
                          f'''layers.{layer_i}.ffn_norm.weight'''
                      ].clone(),
                  }
                  snake_case_ =   permute(
                      torch.cat(
                          [
                              loaded[i][f'''layers.{layer_i}.attention.wq.weight'''].view(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      )
                              for i in range(SCREAMING_SNAKE_CASE      )
                          ]      ,      dim=0      ,      ).reshape(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      )      )
                  snake_case_ =   permute(
                      torch.cat(
                          [
                              loaded[i][f'''layers.{layer_i}.attention.wk.weight'''].view(
                                  SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      )
                              for i in range(SCREAMING_SNAKE_CASE      )
                          ]      ,      dim=0      ,      ).reshape(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      )      ,      SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      ,      )
                  snake_case_ =   torch.cat(
                      [
                          loaded[i][f'''layers.{layer_i}.attention.wv.weight'''].view(
                              SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      )
                          for i in range(SCREAMING_SNAKE_CASE      )
                      ]      ,      dim=0      ,      ).reshape(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      )

                  snake_case_ =   torch.cat(
                      [loaded[i][f'''layers.{layer_i}.attention.wo.weight'''] for i in range(SCREAMING_SNAKE_CASE      )]      ,      dim=1      )
                  snake_case_ =   torch.cat(
                      [loaded[i][f'''layers.{layer_i}.feed_forward.w1.weight'''] for i in range(SCREAMING_SNAKE_CASE      )]      ,      dim=0      )
                  snake_case_ =   torch.cat(
                      [loaded[i][f'''layers.{layer_i}.feed_forward.w2.weight'''] for i in range(SCREAMING_SNAKE_CASE      )]      ,      dim=1      )
                  snake_case_ =   torch.cat(
                      [loaded[i][f'''layers.{layer_i}.feed_forward.w3.weight'''] for i in range(SCREAMING_SNAKE_CASE      )]      ,      dim=0      )

            snake_case_ =   inv_freq
            for k, v in state_dict.items():
                  snake_case_ =   filename
                  param_count += v.numel()
            torch.save(SCREAMING_SNAKE_CASE      ,      os.path.join(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      )      )

      snake_case_ =   f'''pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'''
      if model_size == "7B":
            # Unsharded
            snake_case_ =   {
                '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
                '''model.norm.weight''': loaded['''norm.weight'''],
                '''lm_head.weight''': loaded['''output.weight'''],
            }
      else:
            snake_case_ =   {
                '''model.norm.weight''': loaded[0]['''norm.weight'''],
                '''model.embed_tokens.weight''': torch.cat(
                    [loaded[i]['''tok_embeddings.weight'''] for i in range(SCREAMING_SNAKE_CASE      )]      ,      dim=1      ),
                '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(SCREAMING_SNAKE_CASE      )]      ,      dim=0      ),
            }

      for k, v in state_dict.items():
            snake_case_ =   filename
            param_count += v.numel()
      torch.save(SCREAMING_SNAKE_CASE      ,      os.path.join(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      )      )

      # Write configs
      snake_case_ =   {'''total_size''': param_count * 2}
      write_json(SCREAMING_SNAKE_CASE      ,      os.path.join(SCREAMING_SNAKE_CASE      ,      '''pytorch_model.bin.index.json'''      )      )
      snake_case_ =   params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
      snake_case_ =   params['''multiple_of'''] if '''multiple_of''' in params else 256
      snake_case_ =   LlamaConfig(
          hidden_size=SCREAMING_SNAKE_CASE      ,      intermediate_size=compute_intermediate_size(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      )      ,      num_attention_heads=params['''n_heads''']      ,      num_hidden_layers=params['''n_layers''']      ,      rms_norm_eps=params['''norm_eps''']      ,      num_key_value_heads=SCREAMING_SNAKE_CASE      ,      )
      config.save_pretrained(SCREAMING_SNAKE_CASE      )

      # Make space so we can load the model properly now.
      del state_dict
      del loaded
      gc.collect()

      print('''Loading the checkpoint in a Llama model.'''      )
      snake_case_ =   LlamaForCausalLM.from_pretrained(SCREAMING_SNAKE_CASE      ,      torch_dtype=torch.floataa      ,      low_cpu_mem_usage=SCREAMING_SNAKE_CASE      )
      # Avoid saving this as part of the config.
      del model.config._name_or_path

      print('''Saving in the Transformers format.'''      )
      model.save_pretrained(SCREAMING_SNAKE_CASE      ,      safe_serialization=SCREAMING_SNAKE_CASE      )
      shutil.rmtree(SCREAMING_SNAKE_CASE      )






def  __lowerCAmelCase					(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      )->						Tuple:




      """simple docstring"""

      snake_case_ =   LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
      print(f'''Saving a {tokenizer_class.__name__} to {tokenizer_path}.'''      )
      snake_case_ =   tokenizer_class(SCREAMING_SNAKE_CASE      )
      tokenizer.save_pretrained(SCREAMING_SNAKE_CASE      )






def  __lowerCAmelCase					()->						Union[str, Any]:




      """simple docstring"""

      snake_case_ =   argparse.ArgumentParser()
      parser.add_argument(
          '''--input_dir'''      ,      help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''      ,      )
      parser.add_argument(
          '''--model_size'''      ,      choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']      ,      )
      parser.add_argument(
          '''--output_dir'''      ,      help='''Location to write HF model and tokenizer'''      ,      )
      parser.add_argument('''--safe_serialization'''      ,      type=SCREAMING_SNAKE_CASE      ,      help='''Whether or not to save using `safetensors`.'''      )
      snake_case_ =   parser.parse_args()
      if args.model_size != "tokenizer_only":
            write_model(
                model_path=args.output_dir      ,      input_base_path=os.path.join(args.input_dir      ,      args.model_size      )      ,      model_size=args.model_size      ,      safe_serialization=args.safe_serialization      ,      )
      snake_case_ =   os.path.join(args.input_dir      ,      '''tokenizer.model'''      )
      write_tokenizer(args.output_dir      ,      SCREAMING_SNAKE_CASE      )


if __name__ == "__main__":
   main()



from collections import namedtuple

UpperCAmelCase    =					namedtuple("""from_to""", """from_ to""")

UpperCAmelCase    =					{
    """cubicmeter""": from_to(1, 1),
    """litre""": from_to(0.001, 1000),
    """kilolitre""": from_to(1, 1),
    """gallon""": from_to(0.00_454, 264.172),
    """cubicyard""": from_to(0.76_455, 1.30_795),
    """cubicfoot""": from_to(0.028, 35.3_147),
    """cup""": from_to(0.000_236_588, 4_226.75),
}






def  __lowerCAmelCase					(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      )->						float:




      """simple docstring"""

      if from_type not in METRIC_CONVERSION:
            raise ValueError(
                f'''Invalid \'from_type\' value: {from_type!r}  Supported values are:\n'''
                + ''', '''.join(SCREAMING_SNAKE_CASE      )      )
      if to_type not in METRIC_CONVERSION:
            raise ValueError(
                f'''Invalid \'to_type\' value: {to_type!r}.  Supported values are:\n'''
                + ''', '''.join(SCREAMING_SNAKE_CASE      )      )
      return value * METRIC_CONVERSION[from_type].from_ * METRIC_CONVERSION[to_type].to


if __name__ == "__main__":
   import doctest

   doctest.testmod()