
import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
					from transformers import LlamaTokenizerFast
except ImportError as e:
					warnings.warn(e)
					warnings.warn(
					    '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
					)
					A   : Optional[int]			=				None
A   : List[str]			=				{
    '''7B''': 1_1_0_0_8,
    '''13B''': 1_3_8_2_4,
    '''30B''': 1_7_9_2_0,
    '''65B''': 2_2_0_1_6,
    '''70B''': 2_8_6_7_2,
}
A   : Tuple			=				{
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}







def        __lowerCamelCase (							__a       :List[Any]   ,					__a       :int=1   ,					__a       :Tuple=2_5_6   )		->    Optional[Any]:






		"""simple docstring"""

		return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3   )   ) + multiple_of - 1) // multiple_of)







def        __lowerCamelCase (							__a       :Dict   )		->    List[str]:






		"""simple docstring"""

		with open(__a   ,					"""r"""   ) as f:
				return json.load(__a   )







def        __lowerCamelCase (							__a       :int   ,					__a       :Tuple   )		->    Tuple:






		"""simple docstring"""

		with open(__a   ,					"""w"""   ) as f:
				json.dump(__a   ,					__a   )







def        __lowerCamelCase (							__a       :List[Any]   ,					__a       :int   ,					__a       :Dict   ,					__a       :Tuple=True   )		->    Union[str, Any]:






		"""simple docstring"""

		os.makedirs(__a   ,					exist_ok=__a   )
		A__  				=       os.path.join(__a   ,					"""tmp"""   )
		os.makedirs(__a   ,					exist_ok=__a   )

		A__  				=       read_json(os.path.join(__a   ,					"""params.json"""   )   )
		A__  				=       NUM_SHARDS[model_size]
		A__  				=       params["""n_layers"""]
		A__  				=       params["""n_heads"""]
		A__  				=       n_heads // num_shards
		A__  				=       params["""dim"""]
		A__  				=       dim // n_heads
		A__  				=       10000.0
		A__  				=       1.0 / (base ** (torch.arange(0   ,					__a   ,					2   ).float() / dims_per_head))

		if "n_kv_heads" in params:
				A__  				=       params["""n_kv_heads"""]  # for GQA / MQA
				A__  				=       n_heads_per_shard // num_key_value_heads
				A__  				=       dim // num_key_value_heads
		else:  # compatibility with other checkpoints
				A__  				=       n_heads
				A__  				=       n_heads_per_shard
				A__  				=       dim

		# permute for sliced rotary
		def permute(__a       :str   ,					__a       :Optional[int]=n_heads   ,					__a       :Dict=dim   ,					__a       :Dict=dim   ):
				return w.view(__a   ,					dima // n_heads // 2   ,					2   ,					__a   ).transpose(1   ,					2   ).reshape(__a   ,					__a   )

		print(F'Fetching all parameters from the checkpoint at {input_base_path}.'   )
		# Load weights
		if model_size == "7B":
				# Not sharded
				# (The sharded implementation would also work, but this is simpler.)
				A__  				=       torch.load(os.path.join(__a   ,					"""consolidated.00.pth"""   )   ,					map_location="""cpu"""   )
		else:
				# Sharded
				A__  				=       [
				    torch.load(os.path.join(__a   ,					F'consolidated.{i:02d}.pth'   )   ,					map_location="""cpu"""   )
				    for i in range(__a   )
				]
		A__  				=       0
		A__  				=       {"""weight_map""": {}}
		for layer_i in range(__a   ):
				A__  				=       F'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'
				if model_size == "7B":
						# Unsharded
						A__  				=       {
						    F'model.layers.{layer_i}.self_attn.q_proj.weight': permute(
						        loaded[F'layers.{layer_i}.attention.wq.weight']   ),
						    F'model.layers.{layer_i}.self_attn.k_proj.weight': permute(
						        loaded[F'layers.{layer_i}.attention.wk.weight']   ),
						    F'model.layers.{layer_i}.self_attn.v_proj.weight': loaded[F'layers.{layer_i}.attention.wv.weight'],
						    F'model.layers.{layer_i}.self_attn.o_proj.weight': loaded[F'layers.{layer_i}.attention.wo.weight'],
						    F'model.layers.{layer_i}.mlp.gate_proj.weight': loaded[F'layers.{layer_i}.feed_forward.w1.weight'],
						    F'model.layers.{layer_i}.mlp.down_proj.weight': loaded[F'layers.{layer_i}.feed_forward.w2.weight'],
						    F'model.layers.{layer_i}.mlp.up_proj.weight': loaded[F'layers.{layer_i}.feed_forward.w3.weight'],
						    F'model.layers.{layer_i}.input_layernorm.weight': loaded[F'layers.{layer_i}.attention_norm.weight'],
						    F'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[F'layers.{layer_i}.ffn_norm.weight'],
						}
				else:
						# Sharded
						# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
						# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
						# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

						A__  				=       {
						    F'model.layers.{layer_i}.input_layernorm.weight': loaded[0][
						        F'layers.{layer_i}.attention_norm.weight'
						    ].clone(),
						    F'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[0][
						        F'layers.{layer_i}.ffn_norm.weight'
						    ].clone(),
						}
						A__  				=       permute(
						    torch.cat(
						        [
						            loaded[i][F'layers.{layer_i}.attention.wq.weight'].view(__a   ,					__a   ,					__a   )
						            for i in range(__a   )
						        ]   ,					dim=0   ,					).reshape(__a   ,					__a   )   )
						A__  				=       permute(
						    torch.cat(
						        [
						            loaded[i][F'layers.{layer_i}.attention.wk.weight'].view(
						                __a   ,					__a   ,					__a   )
						            for i in range(__a   )
						        ]   ,					dim=0   ,					).reshape(__a   ,					__a   )   ,					__a   ,					__a   ,					__a   ,					)
						A__  				=       torch.cat(
						    [
						        loaded[i][F'layers.{layer_i}.attention.wv.weight'].view(
						            __a   ,					__a   ,					__a   )
						        for i in range(__a   )
						    ]   ,					dim=0   ,					).reshape(__a   ,					__a   )

						A__  				=       torch.cat(
						    [loaded[i][F'layers.{layer_i}.attention.wo.weight'] for i in range(__a   )]   ,					dim=1   )
						A__  				=       torch.cat(
						    [loaded[i][F'layers.{layer_i}.feed_forward.w1.weight'] for i in range(__a   )]   ,					dim=0   )
						A__  				=       torch.cat(
						    [loaded[i][F'layers.{layer_i}.feed_forward.w2.weight'] for i in range(__a   )]   ,					dim=1   )
						A__  				=       torch.cat(
						    [loaded[i][F'layers.{layer_i}.feed_forward.w3.weight'] for i in range(__a   )]   ,					dim=0   )

				A__  				=       inv_freq
				for k, v in state_dict.items():
						A__  				=       filename
						param_count += v.numel()
				torch.save(__a   ,					os.path.join(__a   ,					__a   )   )

		A__  				=       F'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'
		if model_size == "7B":
				# Unsharded
				A__  				=       {
				    """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
				    """model.norm.weight""": loaded["""norm.weight"""],
				    """lm_head.weight""": loaded["""output.weight"""],
				}
		else:
				A__  				=       {
				    """model.norm.weight""": loaded[0]["""norm.weight"""],
				    """model.embed_tokens.weight""": torch.cat(
				        [loaded[i]["""tok_embeddings.weight"""] for i in range(__a   )]   ,					dim=1   ),
				    """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(__a   )]   ,					dim=0   ),
				}

		for k, v in state_dict.items():
				A__  				=       filename
				param_count += v.numel()
		torch.save(__a   ,					os.path.join(__a   ,					__a   )   )

		# Write configs
		A__  				=       {"""total_size""": param_count * 2}
		write_json(__a   ,					os.path.join(__a   ,					"""pytorch_model.bin.index.json"""   )   )
		A__  				=       params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
		A__  				=       params["""multiple_of"""] if """multiple_of""" in params else 2_5_6
		A__  				=       LlamaConfig(
		    hidden_size=__a   ,					intermediate_size=compute_intermediate_size(__a   ,					__a   ,					__a   )   ,					num_attention_heads=params["""n_heads"""]   ,					num_hidden_layers=params["""n_layers"""]   ,					rms_norm_eps=params["""norm_eps"""]   ,					num_key_value_heads=__a   ,					)
		config.save_pretrained(__a   )

		# Make space so we can load the model properly now.
		del state_dict
		del loaded
		gc.collect()

		print("""Loading the checkpoint in a Llama model."""   )
		A__  				=       LlamaForCausalLM.from_pretrained(__a   ,					torch_dtype=torch.floataa   ,					low_cpu_mem_usage=__a   )
		# Avoid saving this as part of the config.
		del model.config._name_or_path

		print("""Saving in the Transformers format."""   )
		model.save_pretrained(__a   ,					safe_serialization=__a   )
		shutil.rmtree(__a   )







def        __lowerCamelCase (							__a       :List[str]   ,					__a       :Tuple   )		->    Tuple:






		"""simple docstring"""

		A__  				=       LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
		print(F'Saving a {tokenizer_class.__name__} to {tokenizer_path}.'   )
		A__  				=       tokenizer_class(__a   )
		tokenizer.save_pretrained(__a   )







def        __lowerCamelCase (							)		->    List[str]:






		"""simple docstring"""

		A__  				=       argparse.ArgumentParser()
		parser.add_argument(
		    """--input_dir"""   ,					help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""   ,					)
		parser.add_argument(
		    """--model_size"""   ,					choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]   ,					)
		parser.add_argument(
		    """--output_dir"""   ,					help="""Location to write HF model and tokenizer"""   ,					)
		parser.add_argument("""--safe_serialization"""   ,					type=__a   ,					help="""Whether or not to save using `safetensors`."""   )
		A__  				=       parser.parse_args()
		if args.model_size != "tokenizer_only":
				write_model(
				    model_path=args.output_dir   ,					input_base_path=os.path.join(args.input_dir   ,					args.model_size   )   ,					model_size=args.model_size   ,					safe_serialization=args.safe_serialization   ,					)
		A__  				=       os.path.join(args.input_dir   ,					"""tokenizer.model"""   )
		write_tokenizer(args.output_dir   ,					__a   )


if __name__ == "__main__":
					main()










import itertools
import os
import random
import tempfile
import unittest

import numpy as np
from datasets import load_dataset

from transformers import is_speech_available
from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_speech_available():
					from transformers import WhisperFeatureExtractor

if is_torch_available():
					import torch

A   : Dict			=				random.Random()







def        __lowerCamelCase (							__a       :Dict   ,					__a       :str=1.0   ,					__a       :List[Any]=None   ,					__a       :List[str]=None   )		->    Optional[int]:






		"""simple docstring"""

		if rng is None:
				A__  				=       global_rng

		A__  				=       []
		for batch_idx in range(shape[0]   ):
				values.append([]   )
				for _ in range(shape[1]   ):
						values[-1].append(rng.random() * scale   )

		return values


@require_torch
@require_torchaudio
class   A      (unittest.TestCase   ):
			'''simple docstring'''




			def __init__(      self       :   Dict							,     __lowerCAmelCase       :   Any							,     __lowerCAmelCase       :   str=7							,     __lowerCAmelCase       :   List[Any]=4_00							,     __lowerCAmelCase       :   Optional[Any]=20_00							,     __lowerCAmelCase       :   Dict=10							,     __lowerCAmelCase       :   Union[str, Any]=1_60							,     __lowerCAmelCase       :   List[Any]=8							,     __lowerCAmelCase       :   List[Any]=0.0							,     __lowerCAmelCase       :   int=40_00							,     __lowerCAmelCase       :   Any=False							,     __lowerCAmelCase       :   List[str]=True							,     )     ->     Optional[int]:







					"""simple docstring"""






					A__  				=       parent
					A__  				=       batch_size
					A__  				=       min_seq_length
					A__  				=       max_seq_length
					A__  				=       (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
					A__  				=       padding_value
					A__  				=       sampling_rate
					A__  				=       return_attention_mask
					A__  				=       do_normalize
					A__  				=       feature_size
					A__  				=       chunk_length
					A__  				=       hop_length






			def        a_	(      self       :   Optional[Any]       )     ->     Optional[int]:







					"""simple docstring"""






					return {
					    "feature_size": self.feature_size,
					    "hop_length": self.hop_length,
					    "chunk_length": self.chunk_length,
					    "padding_value": self.padding_value,
					    "sampling_rate": self.sampling_rate,
					    "return_attention_mask": self.return_attention_mask,
					    "do_normalize": self.do_normalize,
					}







			def        a_	(      self       :   List[str]							,     __lowerCAmelCase       :   List[Any]=False							,     __lowerCAmelCase       :   int=False       )     ->     str:







					"""simple docstring"""






					def _flatten(__lowerCAmelCase       :   Optional[int]       ):
							return list(itertools.chain(*__lowerCAmelCase       )       )

					if equal_length:
							A__  				=       [floats_list((self.max_seq_length, self.feature_size)       ) for _ in range(self.batch_size       )]
					else:
							# make sure that inputs increase in size
							A__  				=       [
							    floats_list((x, self.feature_size)       )
							    for x in range(self.min_seq_length							,     self.max_seq_length							,     self.seq_length_diff       )
							]
					if numpify:
							A__  				=       [np.asarray(__lowerCAmelCase       ) for x in speech_inputs]
					return speech_inputs


@require_torch
@require_torchaudio
class   A      (SCREAMING_SNAKE_CASE       ,					unittest.TestCase   ):
			'''simple docstring'''




			__lowerCamelCase     :	int          =   WhisperFeatureExtractor if is_speech_available() else None






			def        a_	(      self       :   Any       )     ->     str:







					"""simple docstring"""






					A__  				=       WhisperFeatureExtractionTester(self       )






			def        a_	(      self       :   Dict       )     ->     Tuple:







					"""simple docstring"""






					A__  				=       self.feature_extraction_class(**self.feat_extract_dict       )

					with tempfile.TemporaryDirectory() as tmpdirname:
							A__  				=       feat_extract_first.save_pretrained(__lowerCAmelCase       )[0]
							check_json_file_has_correct_format(__lowerCAmelCase       )
							A__  				=       self.feature_extraction_class.from_pretrained(__lowerCAmelCase       )

					A__  				=       feat_extract_first.to_dict()
					A__  				=       feat_extract_second.to_dict()
					A__  				=       feat_extract_first.mel_filters
					A__  				=       feat_extract_second.mel_filters
					self.assertTrue(np.allclose(__lowerCAmelCase							,     __lowerCAmelCase       )       )
					self.assertEqual(__lowerCAmelCase							,     __lowerCAmelCase       )






			def        a_	(      self       :   Dict       )     ->     str:







					"""simple docstring"""






					A__  				=       self.feature_extraction_class(**self.feat_extract_dict       )

					with tempfile.TemporaryDirectory() as tmpdirname:
							A__  				=       os.path.join(__lowerCAmelCase							,     """feat_extract.json"""       )
							feat_extract_first.to_json_file(__lowerCAmelCase       )
							A__  				=       self.feature_extraction_class.from_json_file(__lowerCAmelCase       )

					A__  				=       feat_extract_first.to_dict()
					A__  				=       feat_extract_second.to_dict()
					A__  				=       feat_extract_first.mel_filters
					A__  				=       feat_extract_second.mel_filters
					self.assertTrue(np.allclose(__lowerCAmelCase							,     __lowerCAmelCase       )       )
					self.assertEqual(__lowerCAmelCase							,     __lowerCAmelCase       )






			def        a_	(      self       :   Union[str, Any]       )     ->     List[Any]:







					"""simple docstring"""






					A__  				=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()       )
					# create three inputs of length 800, 1000, and 1200
					A__  				=       [floats_list((1, x)       )[0] for x in range(8_00							,     14_00							,     2_00       )]
					A__  				=       [np.asarray(__lowerCAmelCase       ) for speech_input in speech_inputs]

					# Test feature size
					A__  				=       feature_extractor(__lowerCAmelCase							,     padding="""max_length"""							,     return_tensors="""np"""       ).input_features
					self.assertTrue(input_features.ndim == 3       )
					self.assertTrue(input_features.shape[-1] == feature_extractor.nb_max_frames       )
					self.assertTrue(input_features.shape[-2] == feature_extractor.feature_size       )

					# Test not batched input
					A__  				=       feature_extractor(speech_inputs[0]							,     return_tensors="""np"""       ).input_features
					A__  				=       feature_extractor(np_speech_inputs[0]							,     return_tensors="""np"""       ).input_features
					self.assertTrue(np.allclose(__lowerCAmelCase							,     __lowerCAmelCase							,     atol=1e-3       )       )

					# Test batched
					A__  				=       feature_extractor(__lowerCAmelCase							,     return_tensors="""np"""       ).input_features
					A__  				=       feature_extractor(__lowerCAmelCase							,     return_tensors="""np"""       ).input_features
					for enc_seq_a, enc_seq_a in zip(__lowerCAmelCase							,     __lowerCAmelCase       ):
							self.assertTrue(np.allclose(__lowerCAmelCase							,     __lowerCAmelCase							,     atol=1e-3       )       )

					# Test 2-D numpy arrays are batched.
					A__  				=       [floats_list((1, x)       )[0] for x in (8_00, 8_00, 8_00)]
					A__  				=       np.asarray(__lowerCAmelCase       )
					A__  				=       feature_extractor(__lowerCAmelCase							,     return_tensors="""np"""       ).input_features
					A__  				=       feature_extractor(__lowerCAmelCase							,     return_tensors="""np"""       ).input_features
					for enc_seq_a, enc_seq_a in zip(__lowerCAmelCase							,     __lowerCAmelCase       ):
							self.assertTrue(np.allclose(__lowerCAmelCase							,     __lowerCAmelCase							,     atol=1e-3       )       )

					# Test truncation required
					A__  				=       [floats_list((1, x)       )[0] for x in range(2_00							,     (feature_extractor.n_samples + 5_00)							,     2_00       )]
					A__  				=       [np.asarray(__lowerCAmelCase       ) for speech_input in speech_inputs]

					A__  				=       [x[: feature_extractor.n_samples] for x in speech_inputs]
					A__  				=       [np.asarray(__lowerCAmelCase       ) for speech_input in speech_inputs_truncated]

					A__  				=       feature_extractor(__lowerCAmelCase							,     return_tensors="""np"""       ).input_features
					A__  				=       feature_extractor(__lowerCAmelCase							,     return_tensors="""np"""       ).input_features
					for enc_seq_a, enc_seq_a in zip(__lowerCAmelCase							,     __lowerCAmelCase       ):
							self.assertTrue(np.allclose(__lowerCAmelCase							,     __lowerCAmelCase							,     atol=1e-3       )       )






			def        a_	(      self       :   Dict       )     ->     Union[str, Any]:







					"""simple docstring"""






					import torch

					A__  				=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()       )
					A__  				=       np.random.rand(1_00							,     32       ).astype(np.floataa       )
					A__  				=       np_speech_inputs.tolist()

					for inputs in [py_speech_inputs, np_speech_inputs]:
							A__  				=       feature_extractor.pad([{"""input_features""": inputs}]							,     return_tensors="""np"""       )
							self.assertTrue(np_processed.input_features.dtype == np.floataa       )
							A__  				=       feature_extractor.pad([{"""input_features""": inputs}]							,     return_tensors="""pt"""       )
							self.assertTrue(pt_processed.input_features.dtype == torch.floataa       )






			def        a_	(      self       :   List[Any]							,     __lowerCAmelCase       :   Any       )     ->     Dict:







					"""simple docstring"""






					A__  				=       load_dataset("""hf-internal-testing/librispeech_asr_dummy"""							,     """clean"""							,     split="""validation"""       )
					# automatic decoding with librispeech
					A__  				=       ds.sort("""id"""       ).select(range(__lowerCAmelCase       )       )[:num_samples]["""audio"""]

					return [x["array"] for x in speech_samples]






			def        a_	(      self       :   Dict       )     ->     Any:







					"""simple docstring"""






					A__  				=       torch.tensor(
					    [
					        0.1_1_9_3, -0.0_9_4_6, -0.1_0_9_8, -0.0_1_9_6, 0.0_2_2_5, -0.0_6_9_0, -0.1_7_3_6, 0.0_9_5_1,
					        0.0_9_7_1, -0.0_8_1_7, -0.0_7_0_2, 0.0_1_6_2, 0.0_2_6_0, 0.0_0_1_7, -0.0_1_9_2, -0.1_6_7_8,
					        0.0_7_0_9, -0.1_8_6_7, -0.0_6_5_5, -0.0_2_7_4, -0.0_2_3_4, -0.1_8_8_4, -0.0_5_1_6, -0.0_5_5_4,
					        -0.0_2_7_4, -0.1_4_2_5, -0.1_4_2_3, 0.0_8_3_7, 0.0_3_7_7, -0.0_8_5_4
					    ]       )
					# fmt: on

					A__  				=       self._load_datasamples(1       )
					A__  				=       WhisperFeatureExtractor()
					A__  				=       feature_extractor(__lowerCAmelCase							,     return_tensors="""pt"""       ).input_features
					self.assertEqual(input_features.shape							,     (1, 80, 30_00)       )
					self.assertTrue(torch.allclose(input_features[0, 0, :30]							,     __lowerCAmelCase							,     atol=1e-4       )       )







			def        a_	(      self       :   Dict       )     ->     Tuple:







					"""simple docstring"""






					A__  				=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()       )
					A__  				=       self._load_datasamples(1       )[0]
					A__  				=       ((audio - audio.min()) / (audio.max() - audio.min())) * 6_55_35  # Rescale to [0, 65535] to show issue
					A__  				=       feat_extract.zero_mean_unit_var_norm([audio]							,     attention_mask=__lowerCAmelCase       )[0]

					self.assertTrue(np.all(np.mean(__lowerCAmelCase       ) < 1e-3       )       )
					self.assertTrue(np.all(np.abs(np.var(__lowerCAmelCase       ) - 1       ) < 1e-3       )       )



