'''simple docstring'''


import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
 from transformers import LlamaTokenizerFast
except ImportError as e:
 warnings.warn(e)
 warnings.warn(
     '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
 )
 snake_case__					:      Union[str, Any]				      =  None
snake_case__					:      Union[str, Any]				      =  {
    '''7B''': 1_1008,
    '''13B''': 1_3824,
    '''30B''': 1_7920,
    '''65B''': 2_2016,
    '''70B''': 2_8672,
}
snake_case__					:      Union[str, Any]				      =  {
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}


def  _lowerCamelCase   (				lowerCamelCase_ :							Dict     ,					lowerCamelCase_ :							Optional[Any]=1     ,					lowerCamelCase_ :							Union[str, Any]=256       ):



     """simple docstring"""





     return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3       )       ) + multiple_of - 1) // multiple_of)


def  _lowerCamelCase   (				lowerCamelCase_ :							Tuple       ):



     """simple docstring"""





     with open(lowerCamelCase_     ,					'r'       ) as f:
          return json.load(lowerCamelCase_       )


def  _lowerCamelCase   (				lowerCamelCase_ :							Union[str, Any]     ,					lowerCamelCase_ :							Union[str, Any]       ):



     """simple docstring"""





     with open(lowerCamelCase_     ,					'w'       ) as f:
          json.dump(lowerCamelCase_     ,					lowerCamelCase_       )


def  _lowerCamelCase   (				lowerCamelCase_ :							Any     ,					lowerCamelCase_ :							Union[str, Any]     ,					lowerCamelCase_ :							Dict     ,					lowerCamelCase_ :							Any=True       ):



     """simple docstring"""





     os.makedirs(lowerCamelCase_     ,					exist_ok=lowerCamelCase_       )
     UpperCAmelCase_							:       Dict       =				os.path.join(lowerCamelCase_     ,					'tmp'       )
     os.makedirs(lowerCamelCase_     ,					exist_ok=lowerCamelCase_       )

     UpperCAmelCase_							:       Optional[Any]       =				read_json(os.path.join(lowerCamelCase_     ,					'params.json'       )       )
     UpperCAmelCase_							:       List[Any]       =				NUM_SHARDS[model_size]
     UpperCAmelCase_							:       Union[str, Any]       =				params['n_layers']
     UpperCAmelCase_							:       Any       =				params['n_heads']
     UpperCAmelCase_							:       Dict       =				n_heads // num_shards
     UpperCAmelCase_							:       int       =				params['dim']
     UpperCAmelCase_							:       Optional[int]       =				dim // n_heads
     UpperCAmelCase_							:       Any       =				1_0000.0
     UpperCAmelCase_							:       str       =				1.0 / (base ** (torch.arange(0     ,					lowerCamelCase_     ,					2       ).float() / dims_per_head))

     if "n_kv_heads" in params:
          UpperCAmelCase_							:       str       =				params['n_kv_heads']  # for GQA / MQA
          UpperCAmelCase_							:       Optional[int]       =				n_heads_per_shard // num_key_value_heads
          UpperCAmelCase_							:       Tuple       =				dim // num_key_value_heads
     else:  # compatibility with other checkpoints
          UpperCAmelCase_							:       Optional[int]       =				n_heads
          UpperCAmelCase_							:       List[str]       =				n_heads_per_shard
          UpperCAmelCase_							:       Any       =				dim

     # permute for sliced rotary
     def permute(lowerCamelCase_ :							Tuple     ,					lowerCamelCase_ :							Dict=n_heads     ,					lowerCamelCase_ :							str=dim     ,					lowerCamelCase_ :							Union[str, Any]=dim       ):
          return w.view(lowerCamelCase_     ,					dima // n_heads // 2     ,					2     ,					lowerCamelCase_       ).transpose(1     ,					2       ).reshape(lowerCamelCase_     ,					lowerCamelCase_       )

     print(F'''Fetching all parameters from the checkpoint at {input_base_path}.'''       )
     # Load weights
     if model_size == "7B":
          # Not sharded
          # (The sharded implementation would also work, but this is simpler.)
          UpperCAmelCase_							:       Tuple       =				torch.load(os.path.join(lowerCamelCase_     ,					'consolidated.00.pth'       )     ,					map_location='cpu'       )
     else:
          # Sharded
          UpperCAmelCase_							:       List[Any]       =				[
              torch.load(os.path.join(lowerCamelCase_     ,					F'''consolidated.{i:02d}.pth'''       )     ,					map_location='cpu'       )
              for i in range(lowerCamelCase_       )
          ]
     UpperCAmelCase_							:       List[Any]       =				0
     UpperCAmelCase_							:       Tuple       =				{'weight_map': {}}
     for layer_i in range(lowerCamelCase_       ):
          UpperCAmelCase_							:       Dict       =				F'''pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'''
          if model_size == "7B":
               # Unsharded
               UpperCAmelCase_							:       Optional[Any]       =				{
                   F'''model.layers.{layer_i}.self_attn.q_proj.weight''': permute(
                       loaded[F'''layers.{layer_i}.attention.wq.weight''']       ),
                   F'''model.layers.{layer_i}.self_attn.k_proj.weight''': permute(
                       loaded[F'''layers.{layer_i}.attention.wk.weight''']       ),
                   F'''model.layers.{layer_i}.self_attn.v_proj.weight''': loaded[F'''layers.{layer_i}.attention.wv.weight'''],
                   F'''model.layers.{layer_i}.self_attn.o_proj.weight''': loaded[F'''layers.{layer_i}.attention.wo.weight'''],
                   F'''model.layers.{layer_i}.mlp.gate_proj.weight''': loaded[F'''layers.{layer_i}.feed_forward.w1.weight'''],
                   F'''model.layers.{layer_i}.mlp.down_proj.weight''': loaded[F'''layers.{layer_i}.feed_forward.w2.weight'''],
                   F'''model.layers.{layer_i}.mlp.up_proj.weight''': loaded[F'''layers.{layer_i}.feed_forward.w3.weight'''],
                   F'''model.layers.{layer_i}.input_layernorm.weight''': loaded[F'''layers.{layer_i}.attention_norm.weight'''],
                   F'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[F'''layers.{layer_i}.ffn_norm.weight'''],
               }
          else:
               # Sharded
               # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
               # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
               # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

               UpperCAmelCase_							:       int       =				{
                   F'''model.layers.{layer_i}.input_layernorm.weight''': loaded[0][
                       F'''layers.{layer_i}.attention_norm.weight'''
                   ].clone(),
                   F'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[0][
                       F'''layers.{layer_i}.ffn_norm.weight'''
                   ].clone(),
               }
               UpperCAmelCase_							:       Union[str, Any]       =				permute(
                   torch.cat(
                       [
                           loaded[i][F'''layers.{layer_i}.attention.wq.weight'''].view(lowerCamelCase_     ,					lowerCamelCase_     ,					lowerCamelCase_       )
                           for i in range(lowerCamelCase_       )
                       ]     ,					dim=0     ,					).reshape(lowerCamelCase_     ,					lowerCamelCase_       )       )
               UpperCAmelCase_							:       List[str]       =				permute(
                   torch.cat(
                       [
                           loaded[i][F'''layers.{layer_i}.attention.wk.weight'''].view(
                               lowerCamelCase_     ,					lowerCamelCase_     ,					lowerCamelCase_       )
                           for i in range(lowerCamelCase_       )
                       ]     ,					dim=0     ,					).reshape(lowerCamelCase_     ,					lowerCamelCase_       )     ,					lowerCamelCase_     ,					lowerCamelCase_     ,					lowerCamelCase_     ,					)
               UpperCAmelCase_							:       List[str]       =				torch.cat(
                   [
                       loaded[i][F'''layers.{layer_i}.attention.wv.weight'''].view(
                           lowerCamelCase_     ,					lowerCamelCase_     ,					lowerCamelCase_       )
                       for i in range(lowerCamelCase_       )
                   ]     ,					dim=0     ,					).reshape(lowerCamelCase_     ,					lowerCamelCase_       )

               UpperCAmelCase_							:       Tuple       =				torch.cat(
                   [loaded[i][F'''layers.{layer_i}.attention.wo.weight'''] for i in range(lowerCamelCase_       )]     ,					dim=1       )
               UpperCAmelCase_							:       int       =				torch.cat(
                   [loaded[i][F'''layers.{layer_i}.feed_forward.w1.weight'''] for i in range(lowerCamelCase_       )]     ,					dim=0       )
               UpperCAmelCase_							:       Any       =				torch.cat(
                   [loaded[i][F'''layers.{layer_i}.feed_forward.w2.weight'''] for i in range(lowerCamelCase_       )]     ,					dim=1       )
               UpperCAmelCase_							:       Optional[int]       =				torch.cat(
                   [loaded[i][F'''layers.{layer_i}.feed_forward.w3.weight'''] for i in range(lowerCamelCase_       )]     ,					dim=0       )

          UpperCAmelCase_							:       List[Any]       =				inv_freq
          for k, v in state_dict.items():
               UpperCAmelCase_							:       int       =				filename
               param_count += v.numel()
          torch.save(lowerCamelCase_     ,					os.path.join(lowerCamelCase_     ,					lowerCamelCase_       )       )

     UpperCAmelCase_							:       Dict       =				F'''pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'''
     if model_size == "7B":
          # Unsharded
          UpperCAmelCase_							:       List[Any]       =				{
              'model.embed_tokens.weight': loaded['tok_embeddings.weight'],
              'model.norm.weight': loaded['norm.weight'],
              'lm_head.weight': loaded['output.weight'],
          }
     else:
          UpperCAmelCase_							:       Tuple       =				{
              'model.norm.weight': loaded[0]['norm.weight'],
              'model.embed_tokens.weight': torch.cat(
                  [loaded[i]['tok_embeddings.weight'] for i in range(lowerCamelCase_       )]     ,					dim=1       ),
              'lm_head.weight': torch.cat([loaded[i]['output.weight'] for i in range(lowerCamelCase_       )]     ,					dim=0       ),
          }

     for k, v in state_dict.items():
          UpperCAmelCase_							:       List[Any]       =				filename
          param_count += v.numel()
     torch.save(lowerCamelCase_     ,					os.path.join(lowerCamelCase_     ,					lowerCamelCase_       )       )

     # Write configs
     UpperCAmelCase_							:       List[Any]       =				{'total_size': param_count * 2}
     write_json(lowerCamelCase_     ,					os.path.join(lowerCamelCase_     ,					'pytorch_model.bin.index.json'       )       )
     UpperCAmelCase_							:       List[str]       =				params['ffn_dim_multiplier'] if 'ffn_dim_multiplier' in params else 1
     UpperCAmelCase_							:       Optional[int]       =				params['multiple_of'] if 'multiple_of' in params else 256
     UpperCAmelCase_							:       List[Any]       =				LlamaConfig(
         hidden_size=lowerCamelCase_     ,					intermediate_size=compute_intermediate_size(lowerCamelCase_     ,					lowerCamelCase_     ,					lowerCamelCase_       )     ,					num_attention_heads=params['n_heads']     ,					num_hidden_layers=params['n_layers']     ,					rms_norm_eps=params['norm_eps']     ,					num_key_value_heads=lowerCamelCase_     ,					)
     config.save_pretrained(lowerCamelCase_       )

     # Make space so we can load the model properly now.
     del state_dict
     del loaded
     gc.collect()

     print('Loading the checkpoint in a Llama model.'       )
     UpperCAmelCase_							:       Dict       =				LlamaForCausalLM.from_pretrained(lowerCamelCase_     ,					torch_dtype=torch.floataa     ,					low_cpu_mem_usage=lowerCamelCase_       )
     # Avoid saving this as part of the config.
     del model.config._name_or_path

     print('Saving in the Transformers format.'       )
     model.save_pretrained(lowerCamelCase_     ,					safe_serialization=lowerCamelCase_       )
     shutil.rmtree(lowerCamelCase_       )


def  _lowerCamelCase   (				lowerCamelCase_ :							List[Any]     ,					lowerCamelCase_ :							Optional[Any]       ):



     """simple docstring"""





     UpperCAmelCase_							:       str       =				LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
     print(F'''Saving a {tokenizer_class.__name__} to {tokenizer_path}.'''       )
     UpperCAmelCase_							:       str       =				tokenizer_class(lowerCamelCase_       )
     tokenizer.save_pretrained(lowerCamelCase_       )


def  _lowerCamelCase   (				):



     """simple docstring"""





     UpperCAmelCase_							:       Union[str, Any]       =				argparse.ArgumentParser()
     parser.add_argument(
         '--input_dir'     ,					help='Location of LLaMA weights, which contains tokenizer.model and model folders'     ,					)
     parser.add_argument(
         '--model_size'     ,					choices=['7B', '7Bf', '13B', '13Bf', '30B', '65B', '70B', '70Bf', 'tokenizer_only']     ,					)
     parser.add_argument(
         '--output_dir'     ,					help='Location to write HF model and tokenizer'     ,					)
     parser.add_argument('--safe_serialization'     ,					type=lowerCamelCase_     ,					help='Whether or not to save using `safetensors`.'       )
     UpperCAmelCase_							:       Any       =				parser.parse_args()
     if args.model_size != "tokenizer_only":
          write_model(
              model_path=args.output_dir     ,					input_base_path=os.path.join(args.input_dir     ,					args.model_size       )     ,					model_size=args.model_size     ,					safe_serialization=args.safe_serialization     ,					)
     UpperCAmelCase_							:       Dict       =				os.path.join(args.input_dir     ,					'tokenizer.model'       )
     write_tokenizer(args.output_dir     ,					lowerCamelCase_       )


if __name__ == "__main__":
 main()







'''simple docstring'''


from __future__ import annotations

from collections import deque
from collections.abc import Iterator
from dataclasses import dataclass






@dataclass
class 					__SCREAMING_SNAKE_CASE						:







  '''simple docstring'''






  lowerCamelCase_		:int
  lowerCamelCase_		:int







class 					__SCREAMING_SNAKE_CASE						:







  '''simple docstring'''






  def __init__(			self				,		snake_case_						):




       '''simple docstring'''

       UpperCAmelCase_							:       list[list[Edge]]       =				[[] for _ in range(snake_case_						)]
       UpperCAmelCase_							:       Any       =				size


  def __getitem__(			self				,		snake_case_						):




       '''simple docstring'''

       return iter(self._graph[vertex]						)


  @property
  def _UpperCamelCase					(			self						):




       '''simple docstring'''

       return self._size


  def _UpperCamelCase					(			self				,		snake_case_				,		snake_case_				,		snake_case_						):




       '''simple docstring'''

       if weight not in (0, 1):
            raise ValueError('Edge weight must be either 0 or 1.'						)

       if to_vertex < 0 or to_vertex >= self.size:
            raise ValueError('Vertex indexes must be in [0; size).'						)

       self._graph[from_vertex].append(Edge(snake_case_				,		snake_case_						)						)






  def _UpperCamelCase					(			self				,		snake_case_				,		snake_case_						):




       '''simple docstring'''

       UpperCAmelCase_							:       Optional[int]       =				deque([start_vertex]						)
       UpperCAmelCase_							:       list[int | None]       =				[None] * self.size
       UpperCAmelCase_							:       str       =				0

       while queue:
            UpperCAmelCase_							:       Optional[int]       =				queue.popleft()
            UpperCAmelCase_							:       List[str]       =				distances[current_vertex]
            if current_distance is None:
                 continue

            for edge in self[current_vertex]:
                 UpperCAmelCase_							:       Union[str, Any]       =				current_distance + edge.weight
                 UpperCAmelCase_							:       Union[str, Any]       =				distances[edge.destination_vertex]
                 if (
                     isinstance(snake_case_				,		snake_case_						)
                     and new_distance >= dest_vertex_distance
                 ):
                      continue
                 UpperCAmelCase_							:       Optional[Any]       =				new_distance
                 if edge.weight == 0:
                      queue.appendleft(edge.destination_vertex						)
                 else:
                      queue.append(edge.destination_vertex						)

       if distances[finish_vertex] is None:
            raise ValueError('No path from start_vertex to finish_vertex.'						)

       return distances[finish_vertex]


if __name__ == "__main__":
 import doctest

 doctest.testmod()






