




import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
				from transformers import LlamaTokenizerFast
except ImportError as e:
				warnings.warn(e)
				warnings.warn(
				    "The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"
				)
				a    				=      None
a    				=      {
    "7B": 11008,
    "13B": 13824,
    "30B": 17920,
    "65B": 22016,
    "70B": 28672,
}
a    				=      {
    "7B": 1,
    "7Bf": 1,
    "13B": 2,
    "13Bf": 2,
    "30B": 4,
    "65B": 8,
    "70B": 8,
    "70Bf": 8,
}




def 							_SCREAMING_SNAKE_CASE   (		snake_case      ,  snake_case=1      ,  snake_case=2_5_6							)    ->					Any:
						return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3							)							) + multiple_of - 1) // multiple_of)




def 							_SCREAMING_SNAKE_CASE   (		snake_case							)    ->					int:
						with open(UpperCamelCase__      ,  """r"""							) as f:
												return json.load(UpperCamelCase__							)




def 							_SCREAMING_SNAKE_CASE   (		snake_case      ,  snake_case							)    ->					Union[str, Any]:
						with open(UpperCamelCase__      ,  """w"""							) as f:
												json.dump(UpperCamelCase__      ,  UpperCamelCase__							)




def 							_SCREAMING_SNAKE_CASE   (		snake_case      ,  snake_case      ,  snake_case      ,  snake_case=True							)    ->					Any:
						os.makedirs(UpperCamelCase__      ,  exist_ok=UpperCamelCase__							)
						_UpperCAmelCase	     =      os.path.join(UpperCamelCase__      ,  """tmp"""							)
						os.makedirs(UpperCamelCase__      ,  exist_ok=UpperCamelCase__							)

						_UpperCAmelCase	     =      read_json(os.path.join(UpperCamelCase__      ,  """params.json"""							)							)
						_UpperCAmelCase	     =      NUM_SHARDS[model_size]
						_UpperCAmelCase	     =      params['''n_layers''']
						_UpperCAmelCase	     =      params['''n_heads''']
						_UpperCAmelCase	     =      n_heads // num_shards
						_UpperCAmelCase	     =      params['''dim''']
						_UpperCAmelCase	     =      dim // n_heads
						_UpperCAmelCase	     =      1_0_0_0_0.0
						_UpperCAmelCase	     =      1.0 / (base ** (torch.arange(0      ,  UpperCamelCase__      ,  2							).float() / dims_per_head))

						if "n_kv_heads" in params:
												_UpperCAmelCase	     =      params['''n_kv_heads''']  # for GQA / MQA
												_UpperCAmelCase	     =      n_heads_per_shard // num_key_value_heads
												_UpperCAmelCase	     =      dim // num_key_value_heads
						else:  # compatibility with other checkpoints
												_UpperCAmelCase	     =      n_heads
												_UpperCAmelCase	     =      n_heads_per_shard
												_UpperCAmelCase	     =      dim

						# permute for sliced rotary
						def permute(snake_case      ,  snake_case=n_heads      ,  snake_case=dim      ,  snake_case=dim							):
												return w.view(UpperCamelCase__      ,  dima // n_heads // 2      ,  2      ,  UpperCamelCase__							).transpose(1      ,  2							).reshape(UpperCamelCase__      ,  UpperCamelCase__							)

						print(f"Fetching all parameters from the checkpoint at {input_base_path}."							)
						# Load weights
						if model_size == "7B":
												# Not sharded
												# (The sharded implementation would also work, but this is simpler.)
												_UpperCAmelCase	     =      torch.load(os.path.join(UpperCamelCase__      ,  """consolidated.00.pth"""							)      ,  map_location="""cpu"""							)
						else:
												# Sharded
												_UpperCAmelCase	     =      [
												    torch.load(os.path.join(UpperCamelCase__      ,  f"consolidated.{i:02d}.pth"							)      ,  map_location="""cpu"""							)
												    for i in range(UpperCamelCase__							)
												]
						_UpperCAmelCase	     =      0
						_UpperCAmelCase	     =      {'''weight_map''': {}}
						for layer_i in range(UpperCamelCase__							):
												_UpperCAmelCase	     =      f"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
												if model_size == "7B":
																		# Unsharded
																		_UpperCAmelCase	     =      {
																		    f"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
																		        loaded[f"layers.{layer_i}.attention.wq.weight"]							),
																		    f"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
																		        loaded[f"layers.{layer_i}.attention.wk.weight"]							),
																		    f"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[f"layers.{layer_i}.attention.wv.weight"],
																		    f"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[f"layers.{layer_i}.attention.wo.weight"],
																		    f"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w1.weight"],
																		    f"model.layers.{layer_i}.mlp.down_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w2.weight"],
																		    f"model.layers.{layer_i}.mlp.up_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w3.weight"],
																		    f"model.layers.{layer_i}.input_layernorm.weight": loaded[f"layers.{layer_i}.attention_norm.weight"],
																		    f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[f"layers.{layer_i}.ffn_norm.weight"],
																		}
												else:
																		# Sharded
																		# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
																		# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
																		# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

																		_UpperCAmelCase	     =      {
																		    f"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
																		        f"layers.{layer_i}.attention_norm.weight"
																		    ].clone(),
																		    f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
																		        f"layers.{layer_i}.ffn_norm.weight"
																		    ].clone(),
																		}
																		_UpperCAmelCase	     =      permute(
																		    torch.cat(
																		        [
																		            loaded[i][f"layers.{layer_i}.attention.wq.weight"].view(UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__							)
																		            for i in range(UpperCamelCase__							)
																		        ]      ,  dim=0      ,  ).reshape(UpperCamelCase__      ,  UpperCamelCase__							)							)
																		_UpperCAmelCase	     =      permute(
																		    torch.cat(
																		        [
																		            loaded[i][f"layers.{layer_i}.attention.wk.weight"].view(
																		                UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__							)
																		            for i in range(UpperCamelCase__							)
																		        ]      ,  dim=0      ,  ).reshape(UpperCamelCase__      ,  UpperCamelCase__							)      ,  UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__      ,  )
																		_UpperCAmelCase	     =      torch.cat(
																		    [
																		        loaded[i][f"layers.{layer_i}.attention.wv.weight"].view(
																		            UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__							)
																		        for i in range(UpperCamelCase__							)
																		    ]      ,  dim=0      ,  ).reshape(UpperCamelCase__      ,  UpperCamelCase__							)

																		_UpperCAmelCase	     =      torch.cat(
																		    [loaded[i][f"layers.{layer_i}.attention.wo.weight"] for i in range(UpperCamelCase__							)]      ,  dim=1							)
																		_UpperCAmelCase	     =      torch.cat(
																		    [loaded[i][f"layers.{layer_i}.feed_forward.w1.weight"] for i in range(UpperCamelCase__							)]      ,  dim=0							)
																		_UpperCAmelCase	     =      torch.cat(
																		    [loaded[i][f"layers.{layer_i}.feed_forward.w2.weight"] for i in range(UpperCamelCase__							)]      ,  dim=1							)
																		_UpperCAmelCase	     =      torch.cat(
																		    [loaded[i][f"layers.{layer_i}.feed_forward.w3.weight"] for i in range(UpperCamelCase__							)]      ,  dim=0							)

												_UpperCAmelCase	     =      inv_freq
												for k, v in state_dict.items():
																		_UpperCAmelCase	     =      filename
																		param_count += v.numel()
												torch.save(UpperCamelCase__      ,  os.path.join(UpperCamelCase__      ,  UpperCamelCase__							)							)

						_UpperCAmelCase	     =      f"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
						if model_size == "7B":
												# Unsharded
												_UpperCAmelCase	     =      {
												    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
												    '''model.norm.weight''': loaded['''norm.weight'''],
												    '''lm_head.weight''': loaded['''output.weight'''],
												}
						else:
												_UpperCAmelCase	     =      {
												    '''model.norm.weight''': loaded[0]['''norm.weight'''],
												    '''model.embed_tokens.weight''': torch.cat(
												        [loaded[i]["""tok_embeddings.weight"""] for i in range(UpperCamelCase__							)]      ,  dim=1							),
												    '''lm_head.weight''': torch.cat([loaded[i]["""output.weight"""] for i in range(UpperCamelCase__							)]      ,  dim=0							),
												}

						for k, v in state_dict.items():
												_UpperCAmelCase	     =      filename
												param_count += v.numel()
						torch.save(UpperCamelCase__      ,  os.path.join(UpperCamelCase__      ,  UpperCamelCase__							)							)

						# Write configs
						_UpperCAmelCase	     =      {'''total_size''': param_count * 2}
						write_json(UpperCamelCase__      ,  os.path.join(UpperCamelCase__      ,  """pytorch_model.bin.index.json"""							)							)
						_UpperCAmelCase	     =      params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
						_UpperCAmelCase	     =      params['''multiple_of'''] if '''multiple_of''' in params else 2_5_6
						_UpperCAmelCase	     =      LlamaConfig(
						    hidden_size=UpperCamelCase__      ,  intermediate_size=compute_intermediate_size(UpperCamelCase__      ,  UpperCamelCase__      ,  UpperCamelCase__							)      ,  num_attention_heads=params["""n_heads"""]      ,  num_hidden_layers=params["""n_layers"""]      ,  rms_norm_eps=params["""norm_eps"""]      ,  num_key_value_heads=UpperCamelCase__      ,  )
						config.save_pretrained(UpperCamelCase__							)

						# Make space so we can load the model properly now.
						del state_dict
						del loaded
						gc.collect()

						print("""Loading the checkpoint in a Llama model."""							)
						_UpperCAmelCase	     =      LlamaForCausalLM.from_pretrained(UpperCamelCase__      ,  torch_dtype=torch.floataa      ,  low_cpu_mem_usage=UpperCamelCase__							)
						# Avoid saving this as part of the config.
						del model.config._name_or_path

						print("""Saving in the Transformers format."""							)
						model.save_pretrained(UpperCamelCase__      ,  safe_serialization=UpperCamelCase__							)
						shutil.rmtree(UpperCamelCase__							)




def 							_SCREAMING_SNAKE_CASE   (		snake_case      ,  snake_case							)    ->					Optional[Any]:
						# Initialize the tokenizer based on the `spm` model
						_UpperCAmelCase	     =      LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
						print(f"Saving a {tokenizer_class.__name__} to {tokenizer_path}."							)
						_UpperCAmelCase	     =      tokenizer_class(UpperCamelCase__							)
						tokenizer.save_pretrained(UpperCamelCase__							)




def 							_SCREAMING_SNAKE_CASE   (		)    ->					Optional[int]:
						_UpperCAmelCase	     =      argparse.ArgumentParser()
						parser.add_argument(
						    """--input_dir"""      ,  help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""      ,  )
						parser.add_argument(
						    """--model_size"""      ,  choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]      ,  )
						parser.add_argument(
						    """--output_dir"""      ,  help="""Location to write HF model and tokenizer"""      ,  )
						parser.add_argument("""--safe_serialization"""      ,  type=UpperCamelCase__      ,  help="""Whether or not to save using `safetensors`."""							)
						_UpperCAmelCase	     =      parser.parse_args()
						if args.model_size != "tokenizer_only":
												write_model(
												    model_path=args.output_dir      ,  input_base_path=os.path.join(args.input_dir      ,  args.model_size							)      ,  model_size=args.model_size      ,  safe_serialization=args.safe_serialization      ,  )
						_UpperCAmelCase	     =      os.path.join(args.input_dir      ,  """tokenizer.model"""							)
						write_tokenizer(args.output_dir      ,  UpperCamelCase__							)


if __name__ == "__main__":
				main()

def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__			,		UpperCamelCase__		):

      def get_matched_characters(UpperCamelCase__			,		UpperCamelCase__		) -> str:
            UpperCamelCase__     : Union[str, Any]											=					[]
            UpperCamelCase__     : Union[str, Any]											=					min(len(_stra		)			,		len(_stra		)		) // 2
            for i, l in enumerate(_stra		):
                  UpperCamelCase__     : Optional[int]											=					int(max(0			,		i - limit		)		)
                  UpperCamelCase__     : List[str]											=					int(min(i + limit + 1			,		len(_stra		)		)		)
                  if l in _stra[left:right]:
                        matched.append(UpperCamelCase__		)
                        UpperCamelCase__     : List[str]											=					f'''{_stra[0:_stra.index(UpperCamelCase__		)]} {_stra[_stra.index(UpperCamelCase__		) + 1:]}'''

            return "".join(UpperCamelCase__		)

      # matching characters
      UpperCamelCase__     : Any											=					get_matched_characters(UpperCamelCase__			,		UpperCamelCase__		)
      UpperCamelCase__     : Union[str, Any]											=					get_matched_characters(UpperCamelCase__			,		UpperCamelCase__		)
      UpperCamelCase__     : List[Any]											=					len(UpperCamelCase__		)

      # transposition
      UpperCamelCase__     : str											=					(
          len([(ca, ca) for ca, ca in zip(UpperCamelCase__			,		UpperCamelCase__		) if ca != ca]		) // 2
      )

      if not match_count:
            UpperCamelCase__     : Union[str, Any]											=					0.0
      else:
            UpperCamelCase__     : Optional[int]											=					(
                1
                / 3
                * (
                    match_count / len(UpperCamelCase__		)
                    + match_count / len(UpperCamelCase__		)
                    + (match_count - transpositions) / match_count
                )
            )

      # common prefix up to 4 characters
      UpperCamelCase__     : Dict											=					0
      for ca, ca in zip(stra[:4]			,		stra[:4]		):
            if ca == ca:
                  prefix_len += 1
            else:
                  break

      return jaro + 0.1 * prefix_len * (1 - jaro)


if __name__ == "__main__":
    import doctest

    doctest.testmod()
    print(jaro_winkler("hello", "world"))


