
'''simple docstring'''



import argparse
import struct
import unittest




class     UpperCAmelCase_	:






						'''simple docstring'''






						def __init__(   self       ,  _lowercase ):

												"""simple docstring"""



												_lowerCAmelCase    		=					data

												# Initialize hash values
												_lowerCAmelCase    		=					[
												    0x6_A_0_9_E_6_6_7,
												    0xB_B_6_7_A_E_8_5,
												    0x3_C_6_E_F_3_7_2,
												    0xA_5_4_F_F_5_3_A,
												    0x5_1_0_E_5_2_7_F,
												    0x9_B_0_5_6_8_8_C,
												    0x1_F_8_3_D_9_A_B,
												    0x5_B_E_0_C_D_1_9,
												]

												# Initialize round constants
												_lowerCAmelCase    		=					[
												    0x4_2_8_A_2_F_9_8,
												    0x7_1_3_7_4_4_9_1,
												    0xB_5_C_0_F_B_C_F,
												    0xE_9_B_5_D_B_A_5,
												    0x3_9_5_6_C_2_5_B,
												    0x5_9_F_1_1_1_F_1,
												    0x9_2_3_F_8_2_A_4,
												    0xA_B_1_C_5_E_D_5,
												    0xD_8_0_7_A_A_9_8,
												    0x1_2_8_3_5_B_0_1,
												    0x2_4_3_1_8_5_B_E,
												    0x5_5_0_C_7_D_C_3,
												    0x7_2_B_E_5_D_7_4,
												    0x8_0_D_E_B_1_F_E,
												    0x9_B_D_C_0_6_A_7,
												    0xC_1_9_B_F_1_7_4,
												    0xE_4_9_B_6_9_C_1,
												    0xE_F_B_E_4_7_8_6,
												    0x0_F_C_1_9_D_C_6,
												    0x2_4_0_C_A_1_C_C,
												    0x2_D_E_9_2_C_6_F,
												    0x4_A_7_4_8_4_A_A,
												    0x5_C_B_0_A_9_D_C,
												    0x7_6_F_9_8_8_D_A,
												    0x9_8_3_E_5_1_5_2,
												    0xA_8_3_1_C_6_6_D,
												    0xB_0_0_3_2_7_C_8,
												    0xB_F_5_9_7_F_C_7,
												    0xC_6_E_0_0_B_F_3,
												    0xD_5_A_7_9_1_4_7,
												    0x0_6_C_A_6_3_5_1,
												    0x1_4_2_9_2_9_6_7,
												    0x2_7_B_7_0_A_8_5,
												    0x2_E_1_B_2_1_3_8,
												    0x4_D_2_C_6_D_F_C,
												    0x5_3_3_8_0_D_1_3,
												    0x6_5_0_A_7_3_5_4,
												    0x7_6_6_A_0_A_B_B,
												    0x8_1_C_2_C_9_2_E,
												    0x9_2_7_2_2_C_8_5,
												    0xA_2_B_F_E_8_A_1,
												    0xA_8_1_A_6_6_4_B,
												    0xC_2_4_B_8_B_7_0,
												    0xC_7_6_C_5_1_A_3,
												    0xD_1_9_2_E_8_1_9,
												    0xD_6_9_9_0_6_2_4,
												    0xF_4_0_E_3_5_8_5,
												    0x1_0_6_A_A_0_7_0,
												    0x1_9_A_4_C_1_1_6,
												    0x1_E_3_7_6_C_0_8,
												    0x2_7_4_8_7_7_4_C,
												    0x3_4_B_0_B_C_B_5,
												    0x3_9_1_C_0_C_B_3,
												    0x4_E_D_8_A_A_4_A,
												    0x5_B_9_C_C_A_4_F,
												    0x6_8_2_E_6_F_F_3,
												    0x7_4_8_F_8_2_E_E,
												    0x7_8_A_5_6_3_6_F,
												    0x8_4_C_8_7_8_1_4,
												    0x8_C_C_7_0_2_0_8,
												    0x9_0_B_E_F_F_F_A,
												    0xA_4_5_0_6_C_E_B,
												    0xB_E_F_9_A_3_F_7,
												    0xC_6_7_1_7_8_F_2,
												]

												_lowerCAmelCase    		=					self.preprocessing(self.data )
												self.final_hash()




						@staticmethod
						def    _lowercase      (   _lowercase ):

												"""simple docstring"""



												_lowerCAmelCase    		=					B"""\x80""" + (B"""\x00""" * (63 - (len(_lowercase ) + 8) % 64))
												_lowerCAmelCase    		=					struct.pack(""">Q"""       ,  (len(_lowercase ) * 8) )
												return data + padding + big_endian_integer




						def    _lowercase      (   self ):

												"""simple docstring"""



												_lowerCAmelCase    		=					[
												    self.preprocessed_data[x : x + 64]
												    for x in range(0       ,  len(self.preprocessed_data )       ,  64 )
												]

												for block in self.blocks:
																		# Convert the given block into a list of 4 byte integers
																		_lowerCAmelCase    		=					list(struct.unpack(""">16L"""       ,  _lowercase ) )
																		# add 48 0-ed integers
																		words += [0] * 48

																		_lowerCAmelCase		,       _lowerCAmelCase		,       _lowerCAmelCase		,       _lowerCAmelCase		,       _lowerCAmelCase		,       _lowerCAmelCase		,       _lowerCAmelCase		,       _lowerCAmelCase    		=					self.hashes

																		for index in range(0       ,  64 ):
																								if index > 15:
																														# modify the zero-ed indexes at the end of the array
																														_lowerCAmelCase    		=					(
																														    self.ror(words[index - 15]       ,  7 )
																														    ^ self.ror(words[index - 15]       ,  18 )
																														    ^ (words[index - 15] >> 3)
																														)
																														_lowerCAmelCase    		=					(
																														    self.ror(words[index - 2]       ,  17 )
																														    ^ self.ror(words[index - 2]       ,  19 )
																														    ^ (words[index - 2] >> 10)
																														)

																														_lowerCAmelCase    		=					(
																														    words[index - 16] + sa + words[index - 7] + sa
																														) % 0x1_0_0_0_0_0_0_0_0

																								# Compression
																								_lowerCAmelCase    		=					self.ror(_lowercase       ,  6 ) ^ self.ror(_lowercase       ,  11 ) ^ self.ror(_lowercase       ,  25 )
																								_lowerCAmelCase    		=					(e & f) ^ ((~e & 0xF_F_F_F_F_F_F_F) & g)
																								_lowerCAmelCase    		=					(
																								    h + sa + ch + self.round_constants[index] + words[index]
																								) % 0x1_0_0_0_0_0_0_0_0
																								_lowerCAmelCase    		=					self.ror(_lowercase       ,  2 ) ^ self.ror(_lowercase       ,  13 ) ^ self.ror(_lowercase       ,  22 )
																								_lowerCAmelCase    		=					(a & b) ^ (a & c) ^ (b & c)
																								_lowerCAmelCase    		=					(sa + maj) % 0x1_0_0_0_0_0_0_0_0

																								_lowerCAmelCase		,       _lowerCAmelCase		,       _lowerCAmelCase		,       _lowerCAmelCase		,       _lowerCAmelCase		,       _lowerCAmelCase		,       _lowerCAmelCase		,       _lowerCAmelCase    		=					(
																								    g,
																								    f,
																								    e,
																								    ((d + tempa) % 0x1_0_0_0_0_0_0_0_0),
																								    c,
																								    b,
																								    a,
																								    ((tempa + tempa) % 0x1_0_0_0_0_0_0_0_0),
																								)

																		_lowerCAmelCase    		=					[a, b, c, d, e, f, g, h]

																		# Modify final values
																		_lowerCAmelCase    		=					[
																		    ((element + mutated_hash_values[index]) % 0x1_0_0_0_0_0_0_0_0)
																		    for index, element in enumerate(self.hashes )
																		]

												_lowerCAmelCase    		=					"""""".join([hex(_lowercase )[2:].zfill(8 ) for value in self.hashes] )




						def    _lowercase      (   self       ,  _lowercase       ,  _lowercase ):

												"""simple docstring"""



												return 0xF_F_F_F_F_F_F_F & (value << (32 - rotations)) | (value >> rotations)






class     UpperCAmelCase_	(						unittest.TestCase					):






						'''simple docstring'''






						def    _lowercase      (   self ):

												"""simple docstring"""



												import hashlib

												_lowerCAmelCase    		=					bytes("""Test String"""       ,  """utf-8""" )
												self.assertEqual(SHAaaa(_lowercase ).hash       ,  hashlib.shaaaa(_lowercase ).hexdigest() )






def A ():
						import doctest

						doctest.testmod()

						_lowerCAmelCase    		=					argparse.ArgumentParser()
						parser.add_argument(
						    """-s"""					,      """--string"""					,      dest="""input_string"""					,      default="""Hello World!! Welcome to Cryptography"""					,      help="""Hash the string"""					,      )
						parser.add_argument(
						    """-f"""					,      """--file"""					,      dest="""input_file"""					,      help="""Hash contents of a file"""						)

						_lowerCAmelCase    		=					parser.parse_args()

						_lowerCAmelCase    		=					args.input_string

						# hash input should be a bytestring
						if args.input_file:
												with open(args.input_file					,      """rb"""						) as f:
																		_lowerCAmelCase    		=					f.read()
						else:
												_lowerCAmelCase    		=					bytes(__lowerCamelCase					,      """utf-8"""						)

						print(SHAaaa(__lowerCamelCase						).hash						)


if __name__ == "__main__":
						main()


'''simple docstring'''



import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
						from transformers import LlamaTokenizerFast
except ImportError as e:
						warnings.warn(e)
						warnings.warn(
						    """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
						)
						_lowercase          =						None
_lowercase          =						{
    """7B""": 11008,
    """13B""": 13824,
    """30B""": 17920,
    """65B""": 22016,
    """70B""": 28672,
}
_lowercase          =						{
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}


def A (__lowerCamelCase :int					,      __lowerCamelCase :Optional[Any]=1					,      __lowerCamelCase :List[Any]=256						):
						return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3						)						) + multiple_of - 1) // multiple_of)


def A (__lowerCamelCase :Any						):
						with open(__lowerCamelCase					,      """r"""						) as f:
												return json.load(__lowerCamelCase						)


def A (__lowerCamelCase :List[Any]					,      __lowerCamelCase :int						):
						with open(__lowerCamelCase					,      """w"""						) as f:
												json.dump(__lowerCamelCase					,      __lowerCamelCase						)


def A (__lowerCamelCase :Optional[Any]					,      __lowerCamelCase :Tuple					,      __lowerCamelCase :Optional[Any]					,      __lowerCamelCase :Tuple=True						):
						os.makedirs(__lowerCamelCase					,      exist_ok=__lowerCamelCase						)
						_lowerCAmelCase    		=					os.path.join(__lowerCamelCase					,      """tmp"""						)
						os.makedirs(__lowerCamelCase					,      exist_ok=__lowerCamelCase						)

						_lowerCAmelCase    		=					read_json(os.path.join(__lowerCamelCase					,      """params.json"""						)						)
						_lowerCAmelCase    		=					NUM_SHARDS[model_size]
						_lowerCAmelCase    		=					params["""n_layers"""]
						_lowerCAmelCase    		=					params["""n_heads"""]
						_lowerCAmelCase    		=					n_heads // num_shards
						_lowerCAmelCase    		=					params["""dim"""]
						_lowerCAmelCase    		=					dim // n_heads
						_lowerCAmelCase    		=					10_000.0
						_lowerCAmelCase    		=					1.0 / (base ** (torch.arange(0					,      __lowerCamelCase					,      2						).float() / dims_per_head))

						if "n_kv_heads" in params:
												_lowerCAmelCase    		=					params["""n_kv_heads"""]  # for GQA / MQA
												_lowerCAmelCase    		=					n_heads_per_shard // num_key_value_heads
												_lowerCAmelCase    		=					dim // num_key_value_heads
						else:  # compatibility with other checkpoints
												_lowerCAmelCase    		=					n_heads
												_lowerCAmelCase    		=					n_heads_per_shard
												_lowerCAmelCase    		=					dim

						# permute for sliced rotary
						def permute(__lowerCamelCase :Optional[int]					,      __lowerCamelCase :str=n_heads					,      __lowerCamelCase :str=dim					,      __lowerCamelCase :List[Any]=dim						):
												return w.view(__lowerCamelCase					,      dima // n_heads // 2					,      2					,      __lowerCamelCase						).transpose(1					,      2						).reshape(__lowerCamelCase					,      __lowerCamelCase						)

						print(f'Fetching all parameters from the checkpoint at {input_base_path}.'						)
						# Load weights
						if model_size == "7B":
												# Not sharded
												# (The sharded implementation would also work, but this is simpler.)
												_lowerCAmelCase    		=					torch.load(os.path.join(__lowerCamelCase					,      """consolidated.00.pth"""						)					,      map_location="""cpu"""						)
						else:
												# Sharded
												_lowerCAmelCase    		=					[
												    torch.load(os.path.join(__lowerCamelCase					,      f'consolidated.{i:02d}.pth'						)					,      map_location="""cpu"""						)
												    for i in range(__lowerCamelCase						)
												]
						_lowerCAmelCase    		=					0
						_lowerCAmelCase    		=					{"""weight_map""": {}}
						for layer_i in range(__lowerCamelCase						):
												_lowerCAmelCase    		=					f'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'
												if model_size == "7B":
																		# Unsharded
																		_lowerCAmelCase    		=					{
																		    f'model.layers.{layer_i}.self_attn.q_proj.weight': permute(
																		        loaded[f'layers.{layer_i}.attention.wq.weight']						),
																		    f'model.layers.{layer_i}.self_attn.k_proj.weight': permute(
																		        loaded[f'layers.{layer_i}.attention.wk.weight']						),
																		    f'model.layers.{layer_i}.self_attn.v_proj.weight': loaded[f'layers.{layer_i}.attention.wv.weight'],
																		    f'model.layers.{layer_i}.self_attn.o_proj.weight': loaded[f'layers.{layer_i}.attention.wo.weight'],
																		    f'model.layers.{layer_i}.mlp.gate_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w1.weight'],
																		    f'model.layers.{layer_i}.mlp.down_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w2.weight'],
																		    f'model.layers.{layer_i}.mlp.up_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w3.weight'],
																		    f'model.layers.{layer_i}.input_layernorm.weight': loaded[f'layers.{layer_i}.attention_norm.weight'],
																		    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[f'layers.{layer_i}.ffn_norm.weight'],
																		}
												else:
																		# Sharded
																		# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
																		# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
																		# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

																		_lowerCAmelCase    		=					{
																		    f'model.layers.{layer_i}.input_layernorm.weight': loaded[0][
																		        f'layers.{layer_i}.attention_norm.weight'
																		    ].clone(),
																		    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[0][
																		        f'layers.{layer_i}.ffn_norm.weight'
																		    ].clone(),
																		}
																		_lowerCAmelCase    		=					permute(
																		    torch.cat(
																		        [
																		            loaded[i][f'layers.{layer_i}.attention.wq.weight'].view(__lowerCamelCase					,      __lowerCamelCase					,      __lowerCamelCase						)
																		            for i in range(__lowerCamelCase						)
																		        ]					,      dim=0					,      ).reshape(__lowerCamelCase					,      __lowerCamelCase						)						)
																		_lowerCAmelCase    		=					permute(
																		    torch.cat(
																		        [
																		            loaded[i][f'layers.{layer_i}.attention.wk.weight'].view(
																		                __lowerCamelCase					,      __lowerCamelCase					,      __lowerCamelCase						)
																		            for i in range(__lowerCamelCase						)
																		        ]					,      dim=0					,      ).reshape(__lowerCamelCase					,      __lowerCamelCase						)					,      __lowerCamelCase					,      __lowerCamelCase					,      __lowerCamelCase					,      )
																		_lowerCAmelCase    		=					torch.cat(
																		    [
																		        loaded[i][f'layers.{layer_i}.attention.wv.weight'].view(
																		            __lowerCamelCase					,      __lowerCamelCase					,      __lowerCamelCase						)
																		        for i in range(__lowerCamelCase						)
																		    ]					,      dim=0					,      ).reshape(__lowerCamelCase					,      __lowerCamelCase						)

																		_lowerCAmelCase    		=					torch.cat(
																		    [loaded[i][f'layers.{layer_i}.attention.wo.weight'] for i in range(__lowerCamelCase						)]					,      dim=1						)
																		_lowerCAmelCase    		=					torch.cat(
																		    [loaded[i][f'layers.{layer_i}.feed_forward.w1.weight'] for i in range(__lowerCamelCase						)]					,      dim=0						)
																		_lowerCAmelCase    		=					torch.cat(
																		    [loaded[i][f'layers.{layer_i}.feed_forward.w2.weight'] for i in range(__lowerCamelCase						)]					,      dim=1						)
																		_lowerCAmelCase    		=					torch.cat(
																		    [loaded[i][f'layers.{layer_i}.feed_forward.w3.weight'] for i in range(__lowerCamelCase						)]					,      dim=0						)

												_lowerCAmelCase    		=					inv_freq
												for k, v in state_dict.items():
																		_lowerCAmelCase    		=					filename
																		param_count += v.numel()
												torch.save(__lowerCamelCase					,      os.path.join(__lowerCamelCase					,      __lowerCamelCase						)						)

						_lowerCAmelCase    		=					f'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'
						if model_size == "7B":
												# Unsharded
												_lowerCAmelCase    		=					{
												    """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
												    """model.norm.weight""": loaded["""norm.weight"""],
												    """lm_head.weight""": loaded["""output.weight"""],
												}
						else:
												_lowerCAmelCase    		=					{
												    """model.norm.weight""": loaded[0]["""norm.weight"""],
												    """model.embed_tokens.weight""": torch.cat(
												        [loaded[i]["""tok_embeddings.weight"""] for i in range(__lowerCamelCase						)]					,      dim=1						),
												    """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(__lowerCamelCase						)]					,      dim=0						),
												}

						for k, v in state_dict.items():
												_lowerCAmelCase    		=					filename
												param_count += v.numel()
						torch.save(__lowerCamelCase					,      os.path.join(__lowerCamelCase					,      __lowerCamelCase						)						)

						# Write configs
						_lowerCAmelCase    		=					{"""total_size""": param_count * 2}
						write_json(__lowerCamelCase					,      os.path.join(__lowerCamelCase					,      """pytorch_model.bin.index.json"""						)						)
						_lowerCAmelCase    		=					params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
						_lowerCAmelCase    		=					params["""multiple_of"""] if """multiple_of""" in params else 256
						_lowerCAmelCase    		=					LlamaConfig(
						    hidden_size=__lowerCamelCase					,      intermediate_size=compute_intermediate_size(__lowerCamelCase					,      __lowerCamelCase					,      __lowerCamelCase						)					,      num_attention_heads=params["""n_heads"""]					,      num_hidden_layers=params["""n_layers"""]					,      rms_norm_eps=params["""norm_eps"""]					,      num_key_value_heads=__lowerCamelCase					,      )
						config.save_pretrained(__lowerCamelCase						)

						# Make space so we can load the model properly now.
						del state_dict
						del loaded
						gc.collect()

						print("""Loading the checkpoint in a Llama model."""						)
						_lowerCAmelCase    		=					LlamaForCausalLM.from_pretrained(__lowerCamelCase					,      torch_dtype=torch.floataa					,      low_cpu_mem_usage=__lowerCamelCase						)
						# Avoid saving this as part of the config.
						del model.config._name_or_path

						print("""Saving in the Transformers format."""						)
						model.save_pretrained(__lowerCamelCase					,      safe_serialization=__lowerCamelCase						)
						shutil.rmtree(__lowerCamelCase						)


def A (__lowerCamelCase :Union[str, Any]					,      __lowerCamelCase :Union[str, Any]						):
						# Initialize the tokenizer based on the `spm` model
						_lowerCAmelCase    		=					LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
						print(f'Saving a {tokenizer_class.__name__} to {tokenizer_path}.'						)
						_lowerCAmelCase    		=					tokenizer_class(__lowerCamelCase						)
						tokenizer.save_pretrained(__lowerCamelCase						)


def A ():
						_lowerCAmelCase    		=					argparse.ArgumentParser()
						parser.add_argument(
						    """--input_dir"""					,      help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""					,      )
						parser.add_argument(
						    """--model_size"""					,      choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]					,      )
						parser.add_argument(
						    """--output_dir"""					,      help="""Location to write HF model and tokenizer"""					,      )
						parser.add_argument("""--safe_serialization"""					,      type=__lowerCamelCase					,      help="""Whether or not to save using `safetensors`."""						)
						_lowerCAmelCase    		=					parser.parse_args()
						if args.model_size != "tokenizer_only":
												write_model(
												    model_path=args.output_dir					,      input_base_path=os.path.join(args.input_dir					,      args.model_size						)					,      model_size=args.model_size					,      safe_serialization=args.safe_serialization					,      )
						_lowerCAmelCase    		=					os.path.join(args.input_dir					,      """tokenizer.model"""						)
						write_tokenizer(args.output_dir					,      __lowerCamelCase						)


if __name__ == "__main__":
						main()
