


"""simple docstring"""


import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
   from transformers import LlamaTokenizerFast
except ImportError as e:
   warnings.warn(e)
   warnings.warn(
       'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
   )
   __snake_case    					=					None
__snake_case    					=					{
    '7B': 1_1008,
    '13B': 1_3824,
    '30B': 1_7920,
    '65B': 2_2016,
    '70B': 2_8672,
}
__snake_case    					=					{
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}


def    _lowerCamelCase   (     lowerCamelCase__	:							Any       ,		lowerCamelCase__	:							Dict=1       ,		lowerCamelCase__	:							Union[str, Any]=2_56  ):
 return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3  )  ) + multiple_of - 1) // multiple_of)


def    _lowerCamelCase   (     lowerCamelCase__	:							Union[str, Any]  ):
 with open(lowerCamelCase__       ,		"""r"""  ) as f:
  return json.load(lowerCamelCase__  )


def    _lowerCamelCase   (     lowerCamelCase__	:							List[Any]       ,		lowerCamelCase__	:							Optional[int]  ):
 with open(lowerCamelCase__       ,		"""w"""  ) as f:
  json.dump(lowerCamelCase__       ,		lowerCamelCase__  )


def    _lowerCamelCase   (     lowerCamelCase__	:							int       ,		lowerCamelCase__	:							Any       ,		lowerCamelCase__	:							Tuple       ,		lowerCamelCase__	:							str=True  ):
 os.makedirs(lowerCamelCase__       ,		exist_ok=lowerCamelCase__  )
 lowercase__       :       List[Any]				=						os.path.join(lowerCamelCase__       ,		"""tmp"""  )
 os.makedirs(lowerCamelCase__       ,		exist_ok=lowerCamelCase__  )

 lowercase__       :       str				=						read_json(os.path.join(lowerCamelCase__       ,		"""params.json"""  )  )
 lowercase__       :       str				=						NUM_SHARDS[model_size]
 lowercase__       :       Union[str, Any]				=						params["""n_layers"""]
 lowercase__       :       str				=						params["""n_heads"""]
 lowercase__       :       Dict				=						n_heads // num_shards
 lowercase__       :       List[str]				=						params["""dim"""]
 lowercase__       :       Optional[int]				=						dim // n_heads
 lowercase__       :       str				=						1_0000.0
 lowercase__       :       Tuple				=						1.0 / (base ** (torch.arange(0       ,		lowerCamelCase__       ,		2  ).float() / dims_per_head))

 if "n_kv_heads" in params:
  lowercase__       :       Dict				=						params["""n_kv_heads"""]  # for GQA / MQA
  lowercase__       :       Dict				=						n_heads_per_shard // num_key_value_heads
  lowercase__       :       int				=						dim // num_key_value_heads
 else:  # compatibility with other checkpoints
  lowercase__       :       List[Any]				=						n_heads
  lowercase__       :       List[Any]				=						n_heads_per_shard
  lowercase__       :       Dict				=						dim

 # permute for sliced rotary
 def permute(lowerCamelCase__	:							Tuple       ,		lowerCamelCase__	:							str=n_heads       ,		lowerCamelCase__	:							List[str]=dim       ,		lowerCamelCase__	:							Tuple=dim  ):
  return w.view(lowerCamelCase__       ,		dima // n_heads // 2       ,		2       ,		lowerCamelCase__  ).transpose(1       ,		2  ).reshape(lowerCamelCase__       ,		lowerCamelCase__  )

 print(f'''Fetching all parameters from the checkpoint at {input_base_path}.'''  )
 # Load weights
 if model_size == "7B":
  # Not sharded
  # (The sharded implementation would also work, but this is simpler.)
  lowercase__       :       Any				=						torch.load(os.path.join(lowerCamelCase__       ,		"""consolidated.00.pth"""  )       ,		map_location="""cpu"""  )
 else:
  # Sharded
  lowercase__       :       Optional[Any]				=						[
      torch.load(os.path.join(lowerCamelCase__       ,		f'''consolidated.{i:02d}.pth'''  )       ,		map_location="""cpu"""  )
      for i in range(lowerCamelCase__  )
  ]
 lowercase__       :       Optional[int]				=						0
 lowercase__       :       Optional[Any]				=						{"""weight_map""": {}}
 for layer_i in range(lowerCamelCase__  ):
  lowercase__       :       Union[str, Any]				=						f'''pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'''
  if model_size == "7B":
   # Unsharded
   lowercase__       :       Union[str, Any]				=						{
       f'''model.layers.{layer_i}.self_attn.q_proj.weight''': permute(
           loaded[f'''layers.{layer_i}.attention.wq.weight''']  ),
       f'''model.layers.{layer_i}.self_attn.k_proj.weight''': permute(
           loaded[f'''layers.{layer_i}.attention.wk.weight''']  ),
       f'''model.layers.{layer_i}.self_attn.v_proj.weight''': loaded[f'''layers.{layer_i}.attention.wv.weight'''],
       f'''model.layers.{layer_i}.self_attn.o_proj.weight''': loaded[f'''layers.{layer_i}.attention.wo.weight'''],
       f'''model.layers.{layer_i}.mlp.gate_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w1.weight'''],
       f'''model.layers.{layer_i}.mlp.down_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w2.weight'''],
       f'''model.layers.{layer_i}.mlp.up_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w3.weight'''],
       f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[f'''layers.{layer_i}.attention_norm.weight'''],
       f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[f'''layers.{layer_i}.ffn_norm.weight'''],
   }
  else:
   # Sharded
   # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
   # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
   # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

   lowercase__       :       Optional[Any]				=						{
       f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[0][
           f'''layers.{layer_i}.attention_norm.weight'''
       ].clone(),
       f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[0][
           f'''layers.{layer_i}.ffn_norm.weight'''
       ].clone(),
   }
   lowercase__       :       Optional[Any]				=						permute(
       torch.cat(
           [
               loaded[i][f'''layers.{layer_i}.attention.wq.weight'''].view(lowerCamelCase__       ,		lowerCamelCase__       ,		lowerCamelCase__  )
               for i in range(lowerCamelCase__  )
           ]       ,		dim=0       ,		).reshape(lowerCamelCase__       ,		lowerCamelCase__  )  )
   lowercase__       :       Dict				=						permute(
       torch.cat(
           [
               loaded[i][f'''layers.{layer_i}.attention.wk.weight'''].view(
                   lowerCamelCase__       ,		lowerCamelCase__       ,		lowerCamelCase__  )
               for i in range(lowerCamelCase__  )
           ]       ,		dim=0       ,		).reshape(lowerCamelCase__       ,		lowerCamelCase__  )       ,		lowerCamelCase__       ,		lowerCamelCase__       ,		lowerCamelCase__       ,		)
   lowercase__       :       List[str]				=						torch.cat(
       [
           loaded[i][f'''layers.{layer_i}.attention.wv.weight'''].view(
               lowerCamelCase__       ,		lowerCamelCase__       ,		lowerCamelCase__  )
           for i in range(lowerCamelCase__  )
       ]       ,		dim=0       ,		).reshape(lowerCamelCase__       ,		lowerCamelCase__  )

   lowercase__       :       int				=						torch.cat(
       [loaded[i][f'''layers.{layer_i}.attention.wo.weight'''] for i in range(lowerCamelCase__  )]       ,		dim=1  )
   lowercase__       :       Optional[Any]				=						torch.cat(
       [loaded[i][f'''layers.{layer_i}.feed_forward.w1.weight'''] for i in range(lowerCamelCase__  )]       ,		dim=0  )
   lowercase__       :       Optional[int]				=						torch.cat(
       [loaded[i][f'''layers.{layer_i}.feed_forward.w2.weight'''] for i in range(lowerCamelCase__  )]       ,		dim=1  )
   lowercase__       :       List[str]				=						torch.cat(
       [loaded[i][f'''layers.{layer_i}.feed_forward.w3.weight'''] for i in range(lowerCamelCase__  )]       ,		dim=0  )

  lowercase__       :       int				=						inv_freq
  for k, v in state_dict.items():
   lowercase__       :       Any				=						filename
   param_count += v.numel()
  torch.save(lowerCamelCase__       ,		os.path.join(lowerCamelCase__       ,		lowerCamelCase__  )  )

 lowercase__       :       Any				=						f'''pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'''
 if model_size == "7B":
  # Unsharded
  lowercase__       :       List[str]				=						{
      """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
      """model.norm.weight""": loaded["""norm.weight"""],
      """lm_head.weight""": loaded["""output.weight"""],
  }
 else:
  lowercase__       :       List[Any]				=						{
      """model.norm.weight""": loaded[0]["""norm.weight"""],
      """model.embed_tokens.weight""": torch.cat(
          [loaded[i]["""tok_embeddings.weight"""] for i in range(lowerCamelCase__  )]       ,		dim=1  ),
      """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(lowerCamelCase__  )]       ,		dim=0  ),
  }

 for k, v in state_dict.items():
  lowercase__       :       List[Any]				=						filename
  param_count += v.numel()
 torch.save(lowerCamelCase__       ,		os.path.join(lowerCamelCase__       ,		lowerCamelCase__  )  )

 # Write configs
 lowercase__       :       List[str]				=						{"""total_size""": param_count * 2}
 write_json(lowerCamelCase__       ,		os.path.join(lowerCamelCase__       ,		"""pytorch_model.bin.index.json"""  )  )
 lowercase__       :       List[Any]				=						params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
 lowercase__       :       str				=						params["""multiple_of"""] if """multiple_of""" in params else 2_56
 lowercase__       :       Any				=						LlamaConfig(
     hidden_size=lowerCamelCase__       ,		intermediate_size=compute_intermediate_size(lowerCamelCase__       ,		lowerCamelCase__       ,		lowerCamelCase__  )       ,		num_attention_heads=params["""n_heads"""]       ,		num_hidden_layers=params["""n_layers"""]       ,		rms_norm_eps=params["""norm_eps"""]       ,		num_key_value_heads=lowerCamelCase__       ,		)
 config.save_pretrained(lowerCamelCase__  )

 # Make space so we can load the model properly now.
 del state_dict
 del loaded
 gc.collect()

 print("""Loading the checkpoint in a Llama model."""  )
 lowercase__       :       List[Any]				=						LlamaForCausalLM.from_pretrained(lowerCamelCase__       ,		torch_dtype=torch.floataa       ,		low_cpu_mem_usage=lowerCamelCase__  )
 # Avoid saving this as part of the config.
 del model.config._name_or_path

 print("""Saving in the Transformers format."""  )
 model.save_pretrained(lowerCamelCase__       ,		safe_serialization=lowerCamelCase__  )
 shutil.rmtree(lowerCamelCase__  )


def    _lowerCamelCase   (     lowerCamelCase__	:							Optional[int]       ,		lowerCamelCase__	:							Any  ):
 # Initialize the tokenizer based on the `spm` model
 lowercase__       :       Optional[Any]				=						LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
 print(f'''Saving a {tokenizer_class.__name__} to {tokenizer_path}.'''  )
 lowercase__       :       Optional[int]				=						tokenizer_class(lowerCamelCase__  )
 tokenizer.save_pretrained(lowerCamelCase__  )


def    _lowerCamelCase   (     ):
 lowercase__       :       List[str]				=						argparse.ArgumentParser()
 parser.add_argument(
     """--input_dir"""       ,		help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""       ,		)
 parser.add_argument(
     """--model_size"""       ,		choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]       ,		)
 parser.add_argument(
     """--output_dir"""       ,		help="""Location to write HF model and tokenizer"""       ,		)
 parser.add_argument("""--safe_serialization"""       ,		type=lowerCamelCase__       ,		help="""Whether or not to save using `safetensors`."""  )
 lowercase__       :       str				=						parser.parse_args()
 if args.model_size != "tokenizer_only":
  write_model(
      model_path=args.output_dir       ,		input_base_path=os.path.join(args.input_dir       ,		args.model_size  )       ,		model_size=args.model_size       ,		safe_serialization=args.safe_serialization       ,		)
 lowercase__       :       Tuple				=						os.path.join(args.input_dir       ,		"""tokenizer.model"""  )
 write_tokenizer(args.output_dir       ,		lowerCamelCase__  )


if __name__ == "__main__":
   main()



"""simple docstring"""


import argparse
import collections
import os
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
__snake_case    					=					'src/transformers'
__snake_case    					=					'docs/source/en'
__snake_case    					=					'.'


def    _lowerCamelCase   (     lowerCamelCase__	:							Union[str, Any]       ,		lowerCamelCase__	:							Optional[Any]       ,		lowerCamelCase__	:							Any  ):
 with open(lowerCamelCase__       ,		"""r"""       ,		encoding="""utf-8"""       ,		newline="""\n"""  ) as f:
  lowercase__       :       List[str]				=						f.readlines()
 # Find the start prompt.
 lowercase__       :       str				=						0
 while not lines[start_index].startswith(lowerCamelCase__  ):
  start_index += 1
 start_index += 1

 lowercase__       :       Optional[Any]				=						start_index
 while not lines[end_index].startswith(lowerCamelCase__  ):
  end_index += 1
 end_index -= 1

 while len(lines[start_index]  ) <= 1:
  start_index += 1
 while len(lines[end_index]  ) <= 1:
  end_index -= 1
 end_index += 1
 return "".join(lines[start_index:end_index]  ), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
__snake_case    					=					'Model|Encoder|Decoder|ForConditionalGeneration'
# Regexes that match TF/Flax/PT model names.
__snake_case    					=					re.compile(R'TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
__snake_case    					=					re.compile(R'Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
__snake_case    					=					re.compile(R'(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')


# This is to make sure the transformers module imported is the one in the repo.
__snake_case    					=					direct_transformers_import(TRANSFORMERS_PATH)


def    _lowerCamelCase   (     lowerCamelCase__	:							List[Any]  ):
 lowercase__       :       Union[str, Any]				=						re.finditer(""".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"""       ,		lowerCamelCase__  )
 return [m.group(0  ) for m in matches]


def    _lowerCamelCase   (     lowerCamelCase__	:							Optional[int]       ,		lowerCamelCase__	:							int  ):
 lowercase__       :       Any				=						2 if text == """✅""" or text == """❌""" else len(lowerCamelCase__  )
 lowercase__       :       int				=						(width - text_length) // 2
 lowercase__       :       List[str]				=						width - text_length - left_indent
 return " " * left_indent + text + " " * right_indent


def    _lowerCamelCase   (     ):
 lowercase__       :       List[str]				=						transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
 lowercase__       :       Optional[int]				=						{
     name: config_maping_names[code]
     for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
     if code in config_maping_names
 }
 lowercase__       :       str				=						{name: config.replace("""Config"""       ,		""""""  ) for name, config in model_name_to_config.items()}

 # Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
 lowercase__       :       Tuple				=						collections.defaultdict(lowerCamelCase__  )
 lowercase__       :       Optional[Any]				=						collections.defaultdict(lowerCamelCase__  )
 lowercase__       :       Optional[int]				=						collections.defaultdict(lowerCamelCase__  )
 lowercase__       :       Dict				=						collections.defaultdict(lowerCamelCase__  )
 lowercase__       :       Union[str, Any]				=						collections.defaultdict(lowerCamelCase__  )

 # Let's lookup through all transformers object (once).
 for attr_name in dir(lowerCamelCase__  ):
  lowercase__       :       Optional[Any]				=						None
  if attr_name.endswith("""Tokenizer"""  ):
   lowercase__       :       List[Any]				=						slow_tokenizers
   lowercase__       :       List[str]				=						attr_name[:-9]
  elif attr_name.endswith("""TokenizerFast"""  ):
   lowercase__       :       Tuple				=						fast_tokenizers
   lowercase__       :       str				=						attr_name[:-13]
  elif _re_tf_models.match(lowerCamelCase__  ) is not None:
   lowercase__       :       Tuple				=						tf_models
   lowercase__       :       List[Any]				=						_re_tf_models.match(lowerCamelCase__  ).groups()[0]
  elif _re_flax_models.match(lowerCamelCase__  ) is not None:
   lowercase__       :       List[Any]				=						flax_models
   lowercase__       :       List[Any]				=						_re_flax_models.match(lowerCamelCase__  ).groups()[0]
  elif _re_pt_models.match(lowerCamelCase__  ) is not None:
   lowercase__       :       Union[str, Any]				=						pt_models
   lowercase__       :       Optional[int]				=						_re_pt_models.match(lowerCamelCase__  ).groups()[0]

  if lookup_dict is not None:
   while len(lowerCamelCase__  ) > 0:
    if attr_name in model_name_to_prefix.values():
     lowercase__       :       Tuple				=						True
     break
    # Try again after removing the last word in the name
    lowercase__       :       Union[str, Any]				=						"""""".join(camel_case_split(lowerCamelCase__  )[:-1]  )

    # Let's build that table!
 lowercase__       :       Union[str, Any]				=						list(model_name_to_config.keys()  )
 model_names.sort(key=str.lower  )
 lowercase__       :       List[Any]				=						["""Model""", """Tokenizer slow""", """Tokenizer fast""", """PyTorch support""", """TensorFlow support""", """Flax Support"""]
 # We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
 lowercase__       :       str				=						[len(lowerCamelCase__  ) + 2 for c in columns]
 lowercase__       :       Tuple				=						max([len(lowerCamelCase__  ) for name in model_names]  ) + 2

 # Build the table per se
 lowercase__       :       List[Any]				=						"""|""" + """|""".join([_center_text(lowerCamelCase__       ,		lowerCamelCase__  ) for c, w in zip(lowerCamelCase__       ,		lowerCamelCase__  )]  ) + """|\n"""
 # Use ":-----:" format to center-aligned table cell texts
 table += "|" + "|".join([""":""" + """-""" * (w - 2) + """:""" for w in widths]  ) + "|\n"

 lowercase__       :       List[Any]				=						{True: """✅""", False: """❌"""}
 for name in model_names:
  lowercase__       :       int				=						model_name_to_prefix[name]
  lowercase__       :       List[str]				=						[
      name,
      check[slow_tokenizers[prefix]],
      check[fast_tokenizers[prefix]],
      check[pt_models[prefix]],
      check[tf_models[prefix]],
      check[flax_models[prefix]],
  ]
  table += "|" + "|".join([_center_text(lowerCamelCase__       ,		lowerCamelCase__  ) for l, w in zip(lowerCamelCase__       ,		lowerCamelCase__  )]  ) + "|\n"
 return table


def    _lowerCamelCase   (     lowerCamelCase__	:							Union[str, Any]=False  ):
 lowercase__	,       lowercase__	,       lowercase__	,       lowercase__       :       str				=						_find_text_in_file(
     filename=os.path.join(lowerCamelCase__       ,		"""index.md"""  )       ,		start_prompt="""<!--This table is updated automatically from the auto modules"""       ,		end_prompt="""<!-- End table-->"""       ,		)
 lowercase__       :       Union[str, Any]				=						get_model_table_from_auto_modules()

 if current_table != new_table:
  if overwrite:
   with open(os.path.join(lowerCamelCase__       ,		"""index.md"""  )       ,		"""w"""       ,		encoding="""utf-8"""       ,		newline="""\n"""  ) as f:
    f.writelines(lines[:start_index] + [new_table] + lines[end_index:]  )
  else:
   raise ValueError(
       """The model table in the `index.md` has not been updated. Run `make fix-copies` to fix this."""  )


if __name__ == "__main__":
   __snake_case    					=					argparse.ArgumentParser()
   parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
   __snake_case    					=					parser.parse_args()

   check_model_table(args.fix_and_overwrite)