'''simple docstring'''





import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
    from transformers import LlamaTokenizerFast
except ImportError as e:
    warnings.warn(e)
    warnings.warn(
        'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
    )
    SCREAMING_SNAKE_CASE_:	Optional[int]    =None
SCREAMING_SNAKE_CASE_:	List[Any]    ={
    '7B': 1_10_08,
    '13B': 1_38_24,
    '30B': 1_79_20,
    '65B': 2_20_16,
    '70B': 2_86_72,
}
SCREAMING_SNAKE_CASE_:	Any    ={
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}





def       lowerCAmelCase_							(   snake_case_			:					int   ,       snake_case_			:					int=1   ,       snake_case_			:					Optional[Any]=2_56     )       -> int:


     '''simple docstring'''




     return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3     )     ) + multiple_of - 1) // multiple_of)





def       lowerCAmelCase_							(   snake_case_			:					Optional[int]     )       -> Union[str, Any]:


     '''simple docstring'''




     with open(snake_case_   ,       "r"     ) as f:
          return json.load(snake_case_     )





def       lowerCAmelCase_							(   snake_case_			:					List[str]   ,       snake_case_			:					Tuple     )       -> Union[str, Any]:


     '''simple docstring'''




     with open(snake_case_   ,       "w"     ) as f:
          json.dump(snake_case_   ,       snake_case_     )





def       lowerCAmelCase_							(   snake_case_			:					Any   ,       snake_case_			:					Optional[int]   ,       snake_case_			:					int   ,       snake_case_			:					Any=True     )       -> int:


     '''simple docstring'''




     os.makedirs(snake_case_   ,       exist_ok=snake_case_     )
     UpperCAmelCase_			       =  os.path.join(snake_case_   ,       "tmp"     )
     os.makedirs(snake_case_   ,       exist_ok=snake_case_     )

     UpperCAmelCase_			       =  read_json(os.path.join(snake_case_   ,       "params.json"     )     )
     UpperCAmelCase_			       =  NUM_SHARDS[model_size]
     UpperCAmelCase_			       =  params["n_layers"]
     UpperCAmelCase_			       =  params["n_heads"]
     UpperCAmelCase_			       =  n_heads // num_shards
     UpperCAmelCase_			       =  params["dim"]
     UpperCAmelCase_			       =  dim // n_heads
     UpperCAmelCase_			       =  1_0000.0
     UpperCAmelCase_			       =  1.0 / (base ** (torch.arange(0   ,       snake_case_   ,       2     ).float() / dims_per_head))

     if "n_kv_heads" in params:
          UpperCAmelCase_			       =  params["n_kv_heads"]  # for GQA / MQA
          UpperCAmelCase_			       =  n_heads_per_shard // num_key_value_heads
          UpperCAmelCase_			       =  dim // num_key_value_heads
     else:  # compatibility with other checkpoints
          UpperCAmelCase_			       =  n_heads
          UpperCAmelCase_			       =  n_heads_per_shard
          UpperCAmelCase_			       =  dim

     # permute for sliced rotary
     def permute(snake_case_			:					List[Any]   ,       snake_case_			:					Tuple=n_heads   ,       snake_case_			:					Tuple=dim   ,       snake_case_			:					List[str]=dim     ):
          return w.view(snake_case_   ,       dima // n_heads // 2   ,       2   ,       snake_case_     ).transpose(1   ,       2     ).reshape(snake_case_   ,       snake_case_     )

     print(f"""Fetching all parameters from the checkpoint at {input_base_path}."""     )
     # Load weights
     if model_size == "7B":
          # Not sharded
          # (The sharded implementation would also work, but this is simpler.)
          UpperCAmelCase_			       =  torch.load(os.path.join(snake_case_   ,       "consolidated.00.pth"     )   ,       map_location="cpu"     )
     else:
          # Sharded
          UpperCAmelCase_			       =  [
              torch.load(os.path.join(snake_case_   ,       f"""consolidated.{i:02d}.pth"""     )   ,       map_location="cpu"     )
              for i in range(snake_case_     )
          ]
     UpperCAmelCase_			       =  0
     UpperCAmelCase_			       =  {"weight_map": {}}
     for layer_i in range(snake_case_     ):
          UpperCAmelCase_			       =  f"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
          if model_size == "7B":
               # Unsharded
               UpperCAmelCase_			       =  {
                   f"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
                       loaded[f"""layers.{layer_i}.attention.wq.weight"""]     ),
                   f"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
                       loaded[f"""layers.{layer_i}.attention.wk.weight"""]     ),
                   f"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[f"""layers.{layer_i}.attention.wv.weight"""],
                   f"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[f"""layers.{layer_i}.attention.wo.weight"""],
                   f"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w1.weight"""],
                   f"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w2.weight"""],
                   f"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w3.weight"""],
                   f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[f"""layers.{layer_i}.attention_norm.weight"""],
                   f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[f"""layers.{layer_i}.ffn_norm.weight"""],
               }
          else:
               # Sharded
               # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
               # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
               # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

               UpperCAmelCase_			       =  {
                   f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
                       f"""layers.{layer_i}.attention_norm.weight"""
                   ].clone(),
                   f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
                       f"""layers.{layer_i}.ffn_norm.weight"""
                   ].clone(),
               }
               UpperCAmelCase_			       =  permute(
                   torch.cat(
                       [
                           loaded[i][f"""layers.{layer_i}.attention.wq.weight"""].view(snake_case_   ,       snake_case_   ,       snake_case_     )
                           for i in range(snake_case_     )
                       ]   ,       dim=0   ,       ).reshape(snake_case_   ,       snake_case_     )     )
               UpperCAmelCase_			       =  permute(
                   torch.cat(
                       [
                           loaded[i][f"""layers.{layer_i}.attention.wk.weight"""].view(
                               snake_case_   ,       snake_case_   ,       snake_case_     )
                           for i in range(snake_case_     )
                       ]   ,       dim=0   ,       ).reshape(snake_case_   ,       snake_case_     )   ,       snake_case_   ,       snake_case_   ,       snake_case_   ,       )
               UpperCAmelCase_			       =  torch.cat(
                   [
                       loaded[i][f"""layers.{layer_i}.attention.wv.weight"""].view(
                           snake_case_   ,       snake_case_   ,       snake_case_     )
                       for i in range(snake_case_     )
                   ]   ,       dim=0   ,       ).reshape(snake_case_   ,       snake_case_     )

               UpperCAmelCase_			       =  torch.cat(
                   [loaded[i][f"""layers.{layer_i}.attention.wo.weight"""] for i in range(snake_case_     )]   ,       dim=1     )
               UpperCAmelCase_			       =  torch.cat(
                   [loaded[i][f"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(snake_case_     )]   ,       dim=0     )
               UpperCAmelCase_			       =  torch.cat(
                   [loaded[i][f"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(snake_case_     )]   ,       dim=1     )
               UpperCAmelCase_			       =  torch.cat(
                   [loaded[i][f"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(snake_case_     )]   ,       dim=0     )

          UpperCAmelCase_			       =  inv_freq
          for k, v in state_dict.items():
               UpperCAmelCase_			       =  filename
               param_count += v.numel()
          torch.save(snake_case_   ,       os.path.join(snake_case_   ,       snake_case_     )     )

     UpperCAmelCase_			       =  f"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
     if model_size == "7B":
          # Unsharded
          UpperCAmelCase_			       =  {
              "model.embed_tokens.weight": loaded["tok_embeddings.weight"],
              "model.norm.weight": loaded["norm.weight"],
              "lm_head.weight": loaded["output.weight"],
          }
     else:
          UpperCAmelCase_			       =  {
              "model.norm.weight": loaded[0]["norm.weight"],
              "model.embed_tokens.weight": torch.cat(
                  [loaded[i]["tok_embeddings.weight"] for i in range(snake_case_     )]   ,       dim=1     ),
              "lm_head.weight": torch.cat([loaded[i]["output.weight"] for i in range(snake_case_     )]   ,       dim=0     ),
          }

     for k, v in state_dict.items():
          UpperCAmelCase_			       =  filename
          param_count += v.numel()
     torch.save(snake_case_   ,       os.path.join(snake_case_   ,       snake_case_     )     )

     # Write configs
     UpperCAmelCase_			       =  {"total_size": param_count * 2}
     write_json(snake_case_   ,       os.path.join(snake_case_   ,       "pytorch_model.bin.index.json"     )     )
     UpperCAmelCase_			       =  params["ffn_dim_multiplier"] if "ffn_dim_multiplier" in params else 1
     UpperCAmelCase_			       =  params["multiple_of"] if "multiple_of" in params else 2_56
     UpperCAmelCase_			       =  LlamaConfig(
         hidden_size=snake_case_   ,       intermediate_size=compute_intermediate_size(snake_case_   ,       snake_case_   ,       snake_case_     )   ,       num_attention_heads=params["n_heads"]   ,       num_hidden_layers=params["n_layers"]   ,       rms_norm_eps=params["norm_eps"]   ,       num_key_value_heads=snake_case_   ,       )
     config.save_pretrained(snake_case_     )

     # Make space so we can load the model properly now.
     del state_dict
     del loaded
     gc.collect()

     print("Loading the checkpoint in a Llama model."     )
     UpperCAmelCase_			       =  LlamaForCausalLM.from_pretrained(snake_case_   ,       torch_dtype=torch.floataa   ,       low_cpu_mem_usage=snake_case_     )
     # Avoid saving this as part of the config.
     del model.config._name_or_path

     print("Saving in the Transformers format."     )
     model.save_pretrained(snake_case_   ,       safe_serialization=snake_case_     )
     shutil.rmtree(snake_case_     )





def       lowerCAmelCase_							(   snake_case_			:					Union[str, Any]   ,       snake_case_			:					Dict     )       -> Dict:


     '''simple docstring'''




     UpperCAmelCase_			       =  LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
     print(f"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""     )
     UpperCAmelCase_			       =  tokenizer_class(snake_case_     )
     tokenizer.save_pretrained(snake_case_     )





def       lowerCAmelCase_							(   )       -> Dict:


     '''simple docstring'''




     UpperCAmelCase_			       =  argparse.ArgumentParser()
     parser.add_argument(
         "--input_dir"   ,       help="Location of LLaMA weights, which contains tokenizer.model and model folders"   ,       )
     parser.add_argument(
         "--model_size"   ,       choices=["7B", "7Bf", "13B", "13Bf", "30B", "65B", "70B", "70Bf", "tokenizer_only"]   ,       )
     parser.add_argument(
         "--output_dir"   ,       help="Location to write HF model and tokenizer"   ,       )
     parser.add_argument("--safe_serialization"   ,       type=snake_case_   ,       help="Whether or not to save using `safetensors`."     )
     UpperCAmelCase_			       =  parser.parse_args()
     if args.model_size != "tokenizer_only":
          write_model(
              model_path=args.output_dir   ,       input_base_path=os.path.join(args.input_dir   ,       args.model_size     )   ,       model_size=args.model_size   ,       safe_serialization=args.safe_serialization   ,       )
     UpperCAmelCase_			       =  os.path.join(args.input_dir   ,       "tokenizer.model"     )
     write_tokenizer(args.output_dir   ,       snake_case_     )


if __name__ == "__main__":
    main()







import copy
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import ClassLabel, Features, Value
from .base import TaskTemplate







@dataclass(frozen=lowercase_     )
class 					lowercase							(			lowercase_     ):
				# `task` is not a ClassVar since we want it to be part of the `asdict` output for JSON serialization
				__SCREAMING_SNAKE_CASE   :					str			     =     field(default='''text-classification'''      ,    metadata={'''include_in_asdict_even_if_is_default''': True}     )
				__SCREAMING_SNAKE_CASE   :					ClassVar[Features]			     =     Features({'''text''': Value('''string'''     )}     )
				__SCREAMING_SNAKE_CASE   :					ClassVar[Features]			     =     Features({'''labels''': ClassLabel}     )
				__SCREAMING_SNAKE_CASE   :					str			     =     "text"
				__SCREAMING_SNAKE_CASE   :					str			     =     "labels"




				def        a				(   self ,  snake_case			):
											if self.label_column not in features:
																		raise ValueError(F'''Column {self.label_column} is not present in features.'''			)
											if not isinstance(features[self.label_column] ,  snake_case			):
																		raise ValueError(F'''Column {self.label_column} is not a ClassLabel.'''			)
											snake_case_     							=       copy.deepcopy(self			)
											snake_case_     							=       self.label_schema.copy()
											snake_case_     							=       features[self.label_column]
											snake_case_     							=       label_schema
											return task_template




				@property
				def        a				(   self			):
											return {
											    self.text_column: "text",
											    self.label_column: "labels",
											}




