
"""simple docstring"""




import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
					from transformers import LlamaTokenizerFast
except ImportError as e:
					warnings.warn(e)
					warnings.warn(
					    'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
					)
					UpperCAmelCase__   :					Tuple												=				None
UpperCAmelCase__   :					Any												=				{
    '7B': 1_1_0_0_8,
    '13B': 1_3_8_2_4,
    '30B': 1_7_9_2_0,
    '65B': 2_2_0_1_6,
    '70B': 2_8_6_7_2,
}
UpperCAmelCase__   :					List[str]												=				{
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}

def lowercase_						(     _snake_case     ,_snake_case=1     ,_snake_case=256		):
							return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3		)		) + multiple_of - 1) // multiple_of)

def lowercase_						(     _snake_case		):







							with open(__snake_case     ,"""r"""		) as f:
														return json.load(__snake_case		)

def lowercase_						(     _snake_case     ,_snake_case		):







							with open(__snake_case     ,"""w"""		) as f:
														json.dump(__snake_case     ,__snake_case		)

def lowercase_						(     _snake_case     ,_snake_case     ,_snake_case     ,_snake_case=True		):
							os.makedirs(__snake_case     ,exist_ok=__snake_case		)
							SCREAMING_SNAKE_CASE__		: Optional[int]     =     os.path.join(__snake_case     ,"""tmp"""		)
							os.makedirs(__snake_case     ,exist_ok=__snake_case		)

							SCREAMING_SNAKE_CASE__		: str     =     read_json(os.path.join(__snake_case     ,"""params.json"""		)		)
							SCREAMING_SNAKE_CASE__		: Dict     =     NUM_SHARDS[model_size]
							SCREAMING_SNAKE_CASE__		: List[str]     =     params["""n_layers"""]
							SCREAMING_SNAKE_CASE__		: str     =     params["""n_heads"""]
							SCREAMING_SNAKE_CASE__		: Optional[Any]     =     n_heads // num_shards
							SCREAMING_SNAKE_CASE__		: Optional[int]     =     params["""dim"""]
							SCREAMING_SNAKE_CASE__		: str     =     dim // n_heads
							SCREAMING_SNAKE_CASE__		: Union[str, Any]     =     10_000.0
							SCREAMING_SNAKE_CASE__		: List[Any]     =     1.0 / (base ** (torch.arange(0     ,__snake_case     ,2		).float() / dims_per_head))

							if "n_kv_heads" in params:
														SCREAMING_SNAKE_CASE__		: Tuple     =     params["""n_kv_heads"""]  # for GQA / MQA
														SCREAMING_SNAKE_CASE__		: Union[str, Any]     =     n_heads_per_shard // num_key_value_heads
														SCREAMING_SNAKE_CASE__		: int     =     dim // num_key_value_heads
							else:  # compatibility with other checkpoints
														SCREAMING_SNAKE_CASE__		: Dict     =     n_heads
														SCREAMING_SNAKE_CASE__		: Any     =     n_heads_per_shard
														SCREAMING_SNAKE_CASE__		: Union[str, Any]     =     dim

							# permute for sliced rotary
							def permute(_snake_case     ,_snake_case=n_heads     ,_snake_case=dim     ,_snake_case=dim		):
														return w.view(__snake_case     ,dima // n_heads // 2     ,2     ,__snake_case		).transpose(1     ,2		).reshape(__snake_case     ,__snake_case		)

							print(f'''Fetching all parameters from the checkpoint at {input_base_path}.'''		)
							# Load weights
							if model_size == "7B":
														# Not sharded
														# (The sharded implementation would also work, but this is simpler.)
														SCREAMING_SNAKE_CASE__		: Tuple     =     torch.load(os.path.join(__snake_case     ,"""consolidated.00.pth"""		)     ,map_location="""cpu"""		)
							else:
														# Sharded
														SCREAMING_SNAKE_CASE__		: Tuple     =     [
														    torch.load(os.path.join(__snake_case     ,f'''consolidated.{i:02d}.pth'''		)     ,map_location="""cpu"""		)
														    for i in range(__snake_case		)
														]
							SCREAMING_SNAKE_CASE__		: Optional[int]     =     0
							SCREAMING_SNAKE_CASE__		: Optional[Any]     =     {"""weight_map""": {}}
							for layer_i in range(__snake_case		):
														SCREAMING_SNAKE_CASE__		: Optional[Any]     =     f'''pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'''
														if model_size == "7B":
																					# Unsharded
																					SCREAMING_SNAKE_CASE__		: Any     =     {
																					    f'''model.layers.{layer_i}.self_attn.q_proj.weight''': permute(
																					        loaded[f'''layers.{layer_i}.attention.wq.weight''']		),
																					    f'''model.layers.{layer_i}.self_attn.k_proj.weight''': permute(
																					        loaded[f'''layers.{layer_i}.attention.wk.weight''']		),
																					    f'''model.layers.{layer_i}.self_attn.v_proj.weight''': loaded[f'''layers.{layer_i}.attention.wv.weight'''],
																					    f'''model.layers.{layer_i}.self_attn.o_proj.weight''': loaded[f'''layers.{layer_i}.attention.wo.weight'''],
																					    f'''model.layers.{layer_i}.mlp.gate_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w1.weight'''],
																					    f'''model.layers.{layer_i}.mlp.down_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w2.weight'''],
																					    f'''model.layers.{layer_i}.mlp.up_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w3.weight'''],
																					    f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[f'''layers.{layer_i}.attention_norm.weight'''],
																					    f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[f'''layers.{layer_i}.ffn_norm.weight'''],
																					}
														else:
																					# Sharded
																					# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
																					# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
																					# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

																					SCREAMING_SNAKE_CASE__		: Optional[Any]     =     {
																					    f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[0][
																					        f'''layers.{layer_i}.attention_norm.weight'''
																					    ].clone(),
																					    f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[0][
																					        f'''layers.{layer_i}.ffn_norm.weight'''
																					    ].clone(),
																					}
																					SCREAMING_SNAKE_CASE__		: Optional[Any]     =     permute(
																					    torch.cat(
																					        [
																					            loaded[i][f'''layers.{layer_i}.attention.wq.weight'''].view(__snake_case     ,__snake_case     ,__snake_case		)
																					            for i in range(__snake_case		)
																					        ]     ,dim=0     ,).reshape(__snake_case     ,__snake_case		)		)
																					SCREAMING_SNAKE_CASE__		: Any     =     permute(
																					    torch.cat(
																					        [
																					            loaded[i][f'''layers.{layer_i}.attention.wk.weight'''].view(
																					                __snake_case     ,__snake_case     ,__snake_case		)
																					            for i in range(__snake_case		)
																					        ]     ,dim=0     ,).reshape(__snake_case     ,__snake_case		)     ,__snake_case     ,__snake_case     ,__snake_case     ,)
																					SCREAMING_SNAKE_CASE__		: Any     =     torch.cat(
																					    [
																					        loaded[i][f'''layers.{layer_i}.attention.wv.weight'''].view(
																					            __snake_case     ,__snake_case     ,__snake_case		)
																					        for i in range(__snake_case		)
																					    ]     ,dim=0     ,).reshape(__snake_case     ,__snake_case		)

																					SCREAMING_SNAKE_CASE__		: Union[str, Any]     =     torch.cat(
																					    [loaded[i][f'''layers.{layer_i}.attention.wo.weight'''] for i in range(__snake_case		)]     ,dim=1		)
																					SCREAMING_SNAKE_CASE__		: Optional[int]     =     torch.cat(
																					    [loaded[i][f'''layers.{layer_i}.feed_forward.w1.weight'''] for i in range(__snake_case		)]     ,dim=0		)
																					SCREAMING_SNAKE_CASE__		: Dict     =     torch.cat(
																					    [loaded[i][f'''layers.{layer_i}.feed_forward.w2.weight'''] for i in range(__snake_case		)]     ,dim=1		)
																					SCREAMING_SNAKE_CASE__		: Optional[Any]     =     torch.cat(
																					    [loaded[i][f'''layers.{layer_i}.feed_forward.w3.weight'''] for i in range(__snake_case		)]     ,dim=0		)

														SCREAMING_SNAKE_CASE__		: Any     =     inv_freq
														for k, v in state_dict.items():
																					SCREAMING_SNAKE_CASE__		: int     =     filename
																					param_count += v.numel()
														torch.save(__snake_case     ,os.path.join(__snake_case     ,__snake_case		)		)

							SCREAMING_SNAKE_CASE__		: str     =     f'''pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'''
							if model_size == "7B":
														# Unsharded
														SCREAMING_SNAKE_CASE__		: Dict     =     {
														    """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
														    """model.norm.weight""": loaded["""norm.weight"""],
														    """lm_head.weight""": loaded["""output.weight"""],
														}
							else:
														SCREAMING_SNAKE_CASE__		: str     =     {
														    """model.norm.weight""": loaded[0]["""norm.weight"""],
														    """model.embed_tokens.weight""": torch.cat(
														        [loaded[i]["""tok_embeddings.weight"""] for i in range(__snake_case		)]     ,dim=1		),
														    """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(__snake_case		)]     ,dim=0		),
														}

							for k, v in state_dict.items():
														SCREAMING_SNAKE_CASE__		: int     =     filename
														param_count += v.numel()
							torch.save(__snake_case     ,os.path.join(__snake_case     ,__snake_case		)		)

							# Write configs
							SCREAMING_SNAKE_CASE__		: Tuple     =     {"""total_size""": param_count * 2}
							write_json(__snake_case     ,os.path.join(__snake_case     ,"""pytorch_model.bin.index.json"""		)		)
							SCREAMING_SNAKE_CASE__		: Any     =     params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
							SCREAMING_SNAKE_CASE__		: Union[str, Any]     =     params["""multiple_of"""] if """multiple_of""" in params else 256
							SCREAMING_SNAKE_CASE__		: Tuple     =     LlamaConfig(
							    hidden_size=__snake_case     ,intermediate_size=compute_intermediate_size(__snake_case     ,__snake_case     ,__snake_case		)     ,num_attention_heads=params["""n_heads"""]     ,num_hidden_layers=params["""n_layers"""]     ,rms_norm_eps=params["""norm_eps"""]     ,num_key_value_heads=__snake_case     ,)
							config.save_pretrained(__snake_case		)

							# Make space so we can load the model properly now.
							del state_dict
							del loaded
							gc.collect()

							print("""Loading the checkpoint in a Llama model."""		)
							SCREAMING_SNAKE_CASE__		: Optional[int]     =     LlamaForCausalLM.from_pretrained(__snake_case     ,torch_dtype=torch.floataa     ,low_cpu_mem_usage=__snake_case		)
							# Avoid saving this as part of the config.
							del model.config._name_or_path

							print("""Saving in the Transformers format."""		)
							model.save_pretrained(__snake_case     ,safe_serialization=__snake_case		)
							shutil.rmtree(__snake_case		)

def lowercase_						(     _snake_case     ,_snake_case		):
							SCREAMING_SNAKE_CASE__		: Optional[Any]     =     LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
							print(f'''Saving a {tokenizer_class.__name__} to {tokenizer_path}.'''		)
							SCREAMING_SNAKE_CASE__		: Union[str, Any]     =     tokenizer_class(__snake_case		)
							tokenizer.save_pretrained(__snake_case		)

def lowercase_						(     ):
							SCREAMING_SNAKE_CASE__		: int     =     argparse.ArgumentParser()
							parser.add_argument(
							    """--input_dir"""     ,help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""     ,)
							parser.add_argument(
							    """--model_size"""     ,choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]     ,)
							parser.add_argument(
							    """--output_dir"""     ,help="""Location to write HF model and tokenizer"""     ,)
							parser.add_argument("""--safe_serialization"""     ,type=__snake_case     ,help="""Whether or not to save using `safetensors`."""		)
							SCREAMING_SNAKE_CASE__		: Dict     =     parser.parse_args()
							if args.model_size != "tokenizer_only":
														write_model(
														    model_path=args.output_dir     ,input_base_path=os.path.join(args.input_dir     ,args.model_size		)     ,model_size=args.model_size     ,safe_serialization=args.safe_serialization     ,)
							SCREAMING_SNAKE_CASE__		: Optional[Any]     =     os.path.join(args.input_dir     ,"""tokenizer.model"""		)
							write_tokenizer(args.output_dir     ,__snake_case		)


if __name__ == "__main__":
					main()



"""simple docstring"""




import logging

from transformers import PretrainedConfig


UpperCAmelCase__   :					Union[str, Any]												=				logging.getLogger(__name__)


UpperCAmelCase__   :					Any												=				{
    'bertabs-finetuned-cnndm': 'https://huggingface.co/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization/resolve/main/config.json',
}



class 			lowerCAmelCase_		(a__						):


				"""simple docstring"""



				__UpperCamelCase   :		List[Any]       =     '''bertabs'''



				def __init__(self     , SCREAMING_SNAKE_CASE__=3_05_22     , SCREAMING_SNAKE_CASE__=5_12     , SCREAMING_SNAKE_CASE__=6     , SCREAMING_SNAKE_CASE__=5_12     , SCREAMING_SNAKE_CASE__=8     , SCREAMING_SNAKE_CASE__=5_12     , SCREAMING_SNAKE_CASE__=0.2     , SCREAMING_SNAKE_CASE__=6     , SCREAMING_SNAKE_CASE__=7_68     , SCREAMING_SNAKE_CASE__=8     , SCREAMING_SNAKE_CASE__=20_48     , SCREAMING_SNAKE_CASE__=0.2     , **SCREAMING_SNAKE_CASE__     , )      ->	Union[str, Any]:



											"""simple docstring"""




											super().__init__(**SCREAMING_SNAKE_CASE__					)

											SCREAMING_SNAKE_CASE__		: Optional[Any]     =     vocab_size
											SCREAMING_SNAKE_CASE__		: Any     =     max_pos

											SCREAMING_SNAKE_CASE__		: List[Any]     =     enc_layers
											SCREAMING_SNAKE_CASE__		: Tuple     =     enc_hidden_size
											SCREAMING_SNAKE_CASE__		: Dict     =     enc_heads
											SCREAMING_SNAKE_CASE__		: Optional[Any]     =     enc_ff_size
											SCREAMING_SNAKE_CASE__		: Dict     =     enc_dropout

											SCREAMING_SNAKE_CASE__		: Any     =     dec_layers
											SCREAMING_SNAKE_CASE__		: Union[str, Any]     =     dec_hidden_size
											SCREAMING_SNAKE_CASE__		: Any     =     dec_heads
											SCREAMING_SNAKE_CASE__		: List[Any]     =     dec_ff_size
											SCREAMING_SNAKE_CASE__		: str     =     dec_dropout




