
import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
    from transformers import LlamaTokenizerFast
except ImportError as e:
    warnings.warn(e)
    warnings.warn(
        "The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"
    )
    lowerCamelCase  =None
lowerCamelCase  ={
    "7B": 1_1_0_0_8,
    "13B": 1_3_8_2_4,
    "30B": 1_7_9_2_0,
    "65B": 2_2_0_1_6,
    "70B": 2_8_6_7_2,
}
lowerCamelCase  ={
    "7B": 1,
    "7Bf": 1,
    "13B": 2,
    "13Bf": 2,
    "30B": 4,
    "65B": 8,
    "70B": 8,
    "70Bf": 8,
}


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__			,		UpperCamelCase__=1			,		UpperCamelCase__=2_5_6		):
      return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3		)		) + multiple_of - 1) // multiple_of)


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__		):
      with open(UpperCamelCase__			,		'''r'''		) as f:
            return json.load(UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__			,		UpperCamelCase__		):
      with open(UpperCamelCase__			,		'''w'''		) as f:
            json.dump(UpperCamelCase__			,		UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__=True		):
      os.makedirs(UpperCamelCase__			,		exist_ok=UpperCamelCase__		)
      UpperCamelCase__     : List[str]											=					os.path.join(UpperCamelCase__			,		'''tmp'''		)
      os.makedirs(UpperCamelCase__			,		exist_ok=UpperCamelCase__		)

      UpperCamelCase__     : Optional[int]											=					read_json(os.path.join(UpperCamelCase__			,		'''params.json'''		)		)
      UpperCamelCase__     : Optional[Any]											=					NUM_SHARDS[model_size]
      UpperCamelCase__     : Any											=					params['''n_layers''']
      UpperCamelCase__     : Union[str, Any]											=					params['''n_heads''']
      UpperCamelCase__     : str											=					n_heads // num_shards
      UpperCamelCase__     : Dict											=					params['''dim''']
      UpperCamelCase__     : int											=					dim // n_heads
      UpperCamelCase__     : Tuple											=					1_0000.0
      UpperCamelCase__     : Optional[Any]											=					1.0 / (base ** (torch.arange(0			,		UpperCamelCase__			,		2		).float() / dims_per_head))

      if "n_kv_heads" in params:
            UpperCamelCase__     : str											=					params['''n_kv_heads''']  # for GQA / MQA
            UpperCamelCase__     : Tuple											=					n_heads_per_shard // num_key_value_heads
            UpperCamelCase__     : Union[str, Any]											=					dim // num_key_value_heads
      else:  # compatibility with other checkpoints
            UpperCamelCase__     : List[Any]											=					n_heads
            UpperCamelCase__     : List[str]											=					n_heads_per_shard
            UpperCamelCase__     : List[str]											=					dim

      # permute for sliced rotary
      def permute(UpperCamelCase__			,		UpperCamelCase__=n_heads			,		UpperCamelCase__=dim			,		UpperCamelCase__=dim		):
            return w.view(UpperCamelCase__			,		dima // n_heads // 2			,		2			,		UpperCamelCase__		).transpose(1			,		2		).reshape(UpperCamelCase__			,		UpperCamelCase__		)

      print(f'''Fetching all parameters from the checkpoint at {input_base_path}.'''		)
      # Load weights
      if model_size == "7B":
            # Not sharded
            # (The sharded implementation would also work, but this is simpler.)
            UpperCamelCase__     : int											=					torch.load(os.path.join(UpperCamelCase__			,		'''consolidated.00.pth'''		)			,		map_location='''cpu'''		)
      else:
            # Sharded
            UpperCamelCase__     : Dict											=					[
                torch.load(os.path.join(UpperCamelCase__			,		f'''consolidated.{i:02d}.pth'''		)			,		map_location='''cpu'''		)
                for i in range(UpperCamelCase__		)
            ]
      UpperCamelCase__     : Union[str, Any]											=					0
      UpperCamelCase__     : List[Any]											=					{'''weight_map''': {}}
      for layer_i in range(UpperCamelCase__		):
            UpperCamelCase__     : Optional[Any]											=					f'''pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'''
            if model_size == "7B":
                  # Unsharded
                  UpperCamelCase__     : List[str]											=					{
                      f'''model.layers.{layer_i}.self_attn.q_proj.weight''': permute(
                          loaded[f'''layers.{layer_i}.attention.wq.weight''']		),
                      f'''model.layers.{layer_i}.self_attn.k_proj.weight''': permute(
                          loaded[f'''layers.{layer_i}.attention.wk.weight''']		),
                      f'''model.layers.{layer_i}.self_attn.v_proj.weight''': loaded[f'''layers.{layer_i}.attention.wv.weight'''],
                      f'''model.layers.{layer_i}.self_attn.o_proj.weight''': loaded[f'''layers.{layer_i}.attention.wo.weight'''],
                      f'''model.layers.{layer_i}.mlp.gate_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w1.weight'''],
                      f'''model.layers.{layer_i}.mlp.down_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w2.weight'''],
                      f'''model.layers.{layer_i}.mlp.up_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w3.weight'''],
                      f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[f'''layers.{layer_i}.attention_norm.weight'''],
                      f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[f'''layers.{layer_i}.ffn_norm.weight'''],
                  }
            else:
                  # Sharded
                  # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
                  # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
                  # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

                  UpperCamelCase__     : Optional[Any]											=					{
                      f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[0][
                          f'''layers.{layer_i}.attention_norm.weight'''
                      ].clone(),
                      f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[0][
                          f'''layers.{layer_i}.ffn_norm.weight'''
                      ].clone(),
                  }
                  UpperCamelCase__     : int											=					permute(
                      torch.cat(
                          [
                              loaded[i][f'''layers.{layer_i}.attention.wq.weight'''].view(UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__		)
                              for i in range(UpperCamelCase__		)
                          ]			,		dim=0			,		).reshape(UpperCamelCase__			,		UpperCamelCase__		)		)
                  UpperCamelCase__     : int											=					permute(
                      torch.cat(
                          [
                              loaded[i][f'''layers.{layer_i}.attention.wk.weight'''].view(
                                  UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__		)
                              for i in range(UpperCamelCase__		)
                          ]			,		dim=0			,		).reshape(UpperCamelCase__			,		UpperCamelCase__		)			,		UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		)
                  UpperCamelCase__     : List[str]											=					torch.cat(
                      [
                          loaded[i][f'''layers.{layer_i}.attention.wv.weight'''].view(
                              UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__		)
                          for i in range(UpperCamelCase__		)
                      ]			,		dim=0			,		).reshape(UpperCamelCase__			,		UpperCamelCase__		)

                  UpperCamelCase__     : Optional[Any]											=					torch.cat(
                      [loaded[i][f'''layers.{layer_i}.attention.wo.weight'''] for i in range(UpperCamelCase__		)]			,		dim=1		)
                  UpperCamelCase__     : Optional[Any]											=					torch.cat(
                      [loaded[i][f'''layers.{layer_i}.feed_forward.w1.weight'''] for i in range(UpperCamelCase__		)]			,		dim=0		)
                  UpperCamelCase__     : Dict											=					torch.cat(
                      [loaded[i][f'''layers.{layer_i}.feed_forward.w2.weight'''] for i in range(UpperCamelCase__		)]			,		dim=1		)
                  UpperCamelCase__     : Any											=					torch.cat(
                      [loaded[i][f'''layers.{layer_i}.feed_forward.w3.weight'''] for i in range(UpperCamelCase__		)]			,		dim=0		)

            UpperCamelCase__     : Union[str, Any]											=					inv_freq
            for k, v in state_dict.items():
                  UpperCamelCase__     : Optional[int]											=					filename
                  param_count += v.numel()
            torch.save(UpperCamelCase__			,		os.path.join(UpperCamelCase__			,		UpperCamelCase__		)		)

      UpperCamelCase__     : Optional[int]											=					f'''pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'''
      if model_size == "7B":
            # Unsharded
            UpperCamelCase__     : Dict											=					{
                '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
                '''model.norm.weight''': loaded['''norm.weight'''],
                '''lm_head.weight''': loaded['''output.weight'''],
            }
      else:
            UpperCamelCase__     : Optional[int]											=					{
                '''model.norm.weight''': loaded[0]['''norm.weight'''],
                '''model.embed_tokens.weight''': torch.cat(
                    [loaded[i]['''tok_embeddings.weight'''] for i in range(UpperCamelCase__		)]			,		dim=1		),
                '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(UpperCamelCase__		)]			,		dim=0		),
            }

      for k, v in state_dict.items():
            UpperCamelCase__     : Optional[Any]											=					filename
            param_count += v.numel()
      torch.save(UpperCamelCase__			,		os.path.join(UpperCamelCase__			,		UpperCamelCase__		)		)

      # Write configs
      UpperCamelCase__     : Optional[Any]											=					{'''total_size''': param_count * 2}
      write_json(UpperCamelCase__			,		os.path.join(UpperCamelCase__			,		'''pytorch_model.bin.index.json'''		)		)
      UpperCamelCase__     : int											=					params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
      UpperCamelCase__     : Union[str, Any]											=					params['''multiple_of'''] if '''multiple_of''' in params else 2_5_6
      UpperCamelCase__     : List[Any]											=					LlamaConfig(
          hidden_size=UpperCamelCase__			,		intermediate_size=compute_intermediate_size(UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__		)			,		num_attention_heads=params['''n_heads''']			,		num_hidden_layers=params['''n_layers''']			,		rms_norm_eps=params['''norm_eps''']			,		num_key_value_heads=UpperCamelCase__			,		)
      config.save_pretrained(UpperCamelCase__		)

      # Make space so we can load the model properly now.
      del state_dict
      del loaded
      gc.collect()

      print('''Loading the checkpoint in a Llama model.'''		)
      UpperCamelCase__     : int											=					LlamaForCausalLM.from_pretrained(UpperCamelCase__			,		torch_dtype=torch.floataa			,		low_cpu_mem_usage=UpperCamelCase__		)
      # Avoid saving this as part of the config.
      del model.config._name_or_path

      print('''Saving in the Transformers format.'''		)
      model.save_pretrained(UpperCamelCase__			,		safe_serialization=UpperCamelCase__		)
      shutil.rmtree(UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__			,		UpperCamelCase__		):
      # Initialize the tokenizer based on the `spm` model
      UpperCamelCase__     : str											=					LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
      print(f'''Saving a {tokenizer_class.__name__} to {tokenizer_path}.'''		)
      UpperCamelCase__     : List[Any]											=					tokenizer_class(UpperCamelCase__		)
      tokenizer.save_pretrained(UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      UpperCamelCase__     : Union[str, Any]											=					argparse.ArgumentParser()
      parser.add_argument(
          '''--input_dir'''			,		help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''			,		)
      parser.add_argument(
          '''--model_size'''			,		choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']			,		)
      parser.add_argument(
          '''--output_dir'''			,		help='''Location to write HF model and tokenizer'''			,		)
      parser.add_argument('''--safe_serialization'''			,		type=UpperCamelCase__			,		help='''Whether or not to save using `safetensors`.'''		)
      UpperCamelCase__     : Union[str, Any]											=					parser.parse_args()
      if args.model_size != "tokenizer_only":
            write_model(
                model_path=args.output_dir			,		input_base_path=os.path.join(args.input_dir			,		args.model_size		)			,		model_size=args.model_size			,		safe_serialization=args.safe_serialization			,		)
      UpperCamelCase__     : Optional[int]											=					os.path.join(args.input_dir			,		'''tokenizer.model'''		)
      write_tokenizer(args.output_dir			,		UpperCamelCase__		)


if __name__ == "__main__":
    main()




from typing import Any


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		):
      _validation(
          UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		)
      # Creates data structures and fill initial step
      UpperCamelCase__     : dict											=					{}
      UpperCamelCase__     : dict											=					{}
      for state in states_space:
            UpperCamelCase__     : Optional[int]											=					observations_space[0]
            UpperCamelCase__     : Any											=					(
                initial_probabilities[state] * emission_probabilities[state][observation]
            )
            UpperCamelCase__     : Union[str, Any]											=					None

      # Fills the data structure with the probabilities of
      # different transitions and pointers to previous states
      for o in range(1			,		len(UpperCamelCase__		)		):
            UpperCamelCase__     : str											=					observations_space[o]
            UpperCamelCase__     : Union[str, Any]											=					observations_space[o - 1]
            for state in states_space:
                  # Calculates the argmax for probability function
                  UpperCamelCase__     : int											=					''''''
                  UpperCamelCase__     : List[str]											=					-1
                  for k_state in states_space:
                        UpperCamelCase__     : Union[str, Any]											=					(
                            probabilities[(k_state, prior_observation)]
                            * transition_probabilities[k_state][state]
                            * emission_probabilities[state][observation]
                        )
                        if probability > max_probability:
                              UpperCamelCase__     : Tuple											=					probability
                              UpperCamelCase__     : Union[str, Any]											=					k_state

            # Update probabilities and pointers dicts
                  UpperCamelCase__     : Tuple											=					(
                      probabilities[(arg_max, prior_observation)]
                      * transition_probabilities[arg_max][state]
                      * emission_probabilities[state][observation]
                  )

                  UpperCamelCase__     : Optional[Any]											=					arg_max

    # The final observation
      UpperCamelCase__     : List[str]											=					observations_space[len(UpperCamelCase__		) - 1]

      # argmax for given final observation
      UpperCamelCase__     : Dict											=					''''''
      UpperCamelCase__     : Tuple											=					-1
      for k_state in states_space:
            UpperCamelCase__     : Any											=					probabilities[(k_state, final_observation)]
            if probability > max_probability:
                  UpperCamelCase__     : List[str]											=					probability
                  UpperCamelCase__     : Tuple											=					k_state
      UpperCamelCase__     : Any											=					arg_max

      # Process pointers backwards
      UpperCamelCase__     : List[Any]											=					last_state
      UpperCamelCase__     : int											=					[]
      for o in range(len(UpperCamelCase__		) - 1			,		-1			,		-1		):
            result.append(UpperCamelCase__		)
            UpperCamelCase__     : int											=					pointers[previous, observations_space[o]]
      result.reverse()

      return result


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		):
      _validate_not_empty(
          UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		)
      _validate_lists(UpperCamelCase__			,		UpperCamelCase__		)
      _validate_dicts(
          UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		):
      if not all(
          [
              observations_space,
              states_space,
              initial_probabilities,
              transition_probabilities,
              emission_probabilities,
          ]		):
            raise ValueError('''There\'s an empty parameter'''		)


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__			,		UpperCamelCase__		):
      _validate_list(UpperCamelCase__			,		'''observations_space'''		)
      _validate_list(UpperCamelCase__			,		'''states_space'''		)


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__			,		UpperCamelCase__		):
      if not isinstance(_object			,		UpperCamelCase__		):
            UpperCamelCase__     : List[Any]											=					f'''{var_name} must be a list'''
            raise ValueError(UpperCamelCase__		)
      else:
            for x in _object:
                  if not isinstance(UpperCamelCase__			,		UpperCamelCase__		):
                        UpperCamelCase__     : List[Any]											=					f'''{var_name} must be a list of strings'''
                        raise ValueError(UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		):
      _validate_dict(UpperCamelCase__			,		'''initial_probabilities'''			,		UpperCamelCase__		)
      _validate_nested_dict(UpperCamelCase__			,		'''transition_probabilities'''		)
      _validate_nested_dict(UpperCamelCase__			,		'''emission_probabilities'''		)


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__			,		UpperCamelCase__		):
      _validate_dict(_object			,		UpperCamelCase__			,		UpperCamelCase__		)
      for x in _object.values():
            _validate_dict(UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__ = False		):
      if not isinstance(_object			,		UpperCamelCase__		):
            UpperCamelCase__     : List[str]											=					f'''{var_name} must be a dict'''
            raise ValueError(UpperCamelCase__		)
      if not all(isinstance(UpperCamelCase__			,		UpperCamelCase__		) for x in _object		):
            UpperCamelCase__     : Dict											=					f'''{var_name} all keys must be strings'''
            raise ValueError(UpperCamelCase__		)
      if not all(isinstance(UpperCamelCase__			,		UpperCamelCase__		) for x in _object.values()		):
            UpperCamelCase__     : Optional[Any]											=					'''nested dictionary ''' if nested else ''''''
            UpperCamelCase__     : Optional[Any]											=					f'''{var_name} {nested_text}all values must be {value_type.__name__}'''
            raise ValueError(UpperCamelCase__		)


if __name__ == "__main__":
    from doctest import testmod

    testmod()


