






"""simple docstring"""







import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
	from transformers import LlamaTokenizerFast
except ImportError as e:
	warnings.warn(e)
	warnings.warn(
	    'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
	)
	__a       :	Union[str, Any]             = None
__a       :	Union[str, Any]             = {
    '7B': 1_1008,
    '13B': 1_3824,
    '30B': 1_7920,
    '65B': 2_2016,
    '70B': 2_8672,
}
__a       :	List[Any]             = {
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}





def        SCREAMING_SNAKE_CASE       (      lowerCamelCase_ ,     lowerCamelCase_=1 ,     lowerCamelCase_=256):
					return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3)) + multiple_of - 1) // multiple_of)





def        SCREAMING_SNAKE_CASE       (      lowerCamelCase_):
					with open(lowerCamelCase_ ,     '''r''') as f:
										return json.load(lowerCamelCase_)





def        SCREAMING_SNAKE_CASE       (      lowerCamelCase_ ,     lowerCamelCase_):
					with open(lowerCamelCase_ ,     '''w''') as f:
										json.dump(lowerCamelCase_ ,     lowerCamelCase_)





def        SCREAMING_SNAKE_CASE       (      lowerCamelCase_ ,     lowerCamelCase_ ,     lowerCamelCase_ ,     lowerCamelCase_=True):
					os.makedirs(lowerCamelCase_ ,     exist_ok=lowerCamelCase_)
					a__          =  os.path.join(lowerCamelCase_ ,     '''tmp''')
					os.makedirs(lowerCamelCase_ ,     exist_ok=lowerCamelCase_)

					a__          =  read_json(os.path.join(lowerCamelCase_ ,     '''params.json'''))
					a__          =  NUM_SHARDS[model_size]
					a__          =  params['''n_layers''']
					a__          =  params['''n_heads''']
					a__          =  n_heads // num_shards
					a__          =  params['''dim''']
					a__          =  dim // n_heads
					a__          =  10000.0
					a__          =  1.0 / (base ** (torch.arange(0 ,     lowerCamelCase_ ,     2).float() / dims_per_head))

					if "n_kv_heads" in params:
										a__          =  params['''n_kv_heads''']  # for GQA / MQA
										a__          =  n_heads_per_shard // num_key_value_heads
										a__          =  dim // num_key_value_heads
					else:  # compatibility with other checkpoints
										a__          =  n_heads
										a__          =  n_heads_per_shard
										a__          =  dim

					# permute for sliced rotary
					def permute(lowerCamelCase_ ,     lowerCamelCase_=n_heads ,     lowerCamelCase_=dim ,     lowerCamelCase_=dim):
										return w.view(lowerCamelCase_ ,     dima // n_heads // 2 ,     2 ,     lowerCamelCase_).transpose(1 ,     2).reshape(lowerCamelCase_ ,     lowerCamelCase_)

					print(f'Fetching all parameters from the checkpoint at {input_base_path}.')
					# Load weights
					if model_size == "7B":
										# Not sharded
										# (The sharded implementation would also work, but this is simpler.)
										a__          =  torch.load(os.path.join(lowerCamelCase_ ,     '''consolidated.00.pth''') ,     map_location='''cpu''')
					else:
										# Sharded
										a__          =  [
										    torch.load(os.path.join(lowerCamelCase_ ,     f'consolidated.{i:02d}.pth') ,     map_location='''cpu''')
										    for i in range(lowerCamelCase_)
										]
					a__          =  0
					a__          =  {'''weight_map''': {}}
					for layer_i in range(lowerCamelCase_):
										a__          =  f'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'
										if model_size == "7B":
															# Unsharded
															a__          =  {
															    f'model.layers.{layer_i}.self_attn.q_proj.weight': permute(
															        loaded[f'layers.{layer_i}.attention.wq.weight']),
															    f'model.layers.{layer_i}.self_attn.k_proj.weight': permute(
															        loaded[f'layers.{layer_i}.attention.wk.weight']),
															    f'model.layers.{layer_i}.self_attn.v_proj.weight': loaded[f'layers.{layer_i}.attention.wv.weight'],
															    f'model.layers.{layer_i}.self_attn.o_proj.weight': loaded[f'layers.{layer_i}.attention.wo.weight'],
															    f'model.layers.{layer_i}.mlp.gate_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w1.weight'],
															    f'model.layers.{layer_i}.mlp.down_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w2.weight'],
															    f'model.layers.{layer_i}.mlp.up_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w3.weight'],
															    f'model.layers.{layer_i}.input_layernorm.weight': loaded[f'layers.{layer_i}.attention_norm.weight'],
															    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[f'layers.{layer_i}.ffn_norm.weight'],
															}
										else:
															# Sharded
															# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
															# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
															# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

															a__          =  {
															    f'model.layers.{layer_i}.input_layernorm.weight': loaded[0][
															        f'layers.{layer_i}.attention_norm.weight'
															    ].clone(),
															    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[0][
															        f'layers.{layer_i}.ffn_norm.weight'
															    ].clone(),
															}
															a__          =  permute(
															    torch.cat(
															        [
															            loaded[i][f'layers.{layer_i}.attention.wq.weight'].view(lowerCamelCase_ ,     lowerCamelCase_ ,     lowerCamelCase_)
															            for i in range(lowerCamelCase_)
															        ] ,     dim=0 ,     ).reshape(lowerCamelCase_ ,     lowerCamelCase_))
															a__          =  permute(
															    torch.cat(
															        [
															            loaded[i][f'layers.{layer_i}.attention.wk.weight'].view(
															                lowerCamelCase_ ,     lowerCamelCase_ ,     lowerCamelCase_)
															            for i in range(lowerCamelCase_)
															        ] ,     dim=0 ,     ).reshape(lowerCamelCase_ ,     lowerCamelCase_) ,     lowerCamelCase_ ,     lowerCamelCase_ ,     lowerCamelCase_ ,     )
															a__          =  torch.cat(
															    [
															        loaded[i][f'layers.{layer_i}.attention.wv.weight'].view(
															            lowerCamelCase_ ,     lowerCamelCase_ ,     lowerCamelCase_)
															        for i in range(lowerCamelCase_)
															    ] ,     dim=0 ,     ).reshape(lowerCamelCase_ ,     lowerCamelCase_)

															a__          =  torch.cat(
															    [loaded[i][f'layers.{layer_i}.attention.wo.weight'] for i in range(lowerCamelCase_)] ,     dim=1)
															a__          =  torch.cat(
															    [loaded[i][f'layers.{layer_i}.feed_forward.w1.weight'] for i in range(lowerCamelCase_)] ,     dim=0)
															a__          =  torch.cat(
															    [loaded[i][f'layers.{layer_i}.feed_forward.w2.weight'] for i in range(lowerCamelCase_)] ,     dim=1)
															a__          =  torch.cat(
															    [loaded[i][f'layers.{layer_i}.feed_forward.w3.weight'] for i in range(lowerCamelCase_)] ,     dim=0)

										a__          =  inv_freq
										for k, v in state_dict.items():
															a__          =  filename
															param_count += v.numel()
										torch.save(lowerCamelCase_ ,     os.path.join(lowerCamelCase_ ,     lowerCamelCase_))

					a__          =  f'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'
					if model_size == "7B":
										# Unsharded
										a__          =  {
										    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
										    '''model.norm.weight''': loaded['''norm.weight'''],
										    '''lm_head.weight''': loaded['''output.weight'''],
										}
					else:
										a__          =  {
										    '''model.norm.weight''': loaded[0]['''norm.weight'''],
										    '''model.embed_tokens.weight''': torch.cat(
										        [loaded[i]['''tok_embeddings.weight'''] for i in range(lowerCamelCase_)] ,     dim=1),
										    '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(lowerCamelCase_)] ,     dim=0),
										}

					for k, v in state_dict.items():
										a__          =  filename
										param_count += v.numel()
					torch.save(lowerCamelCase_ ,     os.path.join(lowerCamelCase_ ,     lowerCamelCase_))

					# Write configs
					a__          =  {'''total_size''': param_count * 2}
					write_json(lowerCamelCase_ ,     os.path.join(lowerCamelCase_ ,     '''pytorch_model.bin.index.json'''))
					a__          =  params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
					a__          =  params['''multiple_of'''] if '''multiple_of''' in params else 256
					a__          =  LlamaConfig(
					    hidden_size=lowerCamelCase_ ,     intermediate_size=compute_intermediate_size(lowerCamelCase_ ,     lowerCamelCase_ ,     lowerCamelCase_) ,     num_attention_heads=params['''n_heads'''] ,     num_hidden_layers=params['''n_layers'''] ,     rms_norm_eps=params['''norm_eps'''] ,     num_key_value_heads=lowerCamelCase_ ,     )
					config.save_pretrained(lowerCamelCase_)

					# Make space so we can load the model properly now.
					del state_dict
					del loaded
					gc.collect()

					print('''Loading the checkpoint in a Llama model.''')
					a__          =  LlamaForCausalLM.from_pretrained(lowerCamelCase_ ,     torch_dtype=torch.floataa ,     low_cpu_mem_usage=lowerCamelCase_)
					# Avoid saving this as part of the config.
					del model.config._name_or_path

					print('''Saving in the Transformers format.''')
					model.save_pretrained(lowerCamelCase_ ,     safe_serialization=lowerCamelCase_)
					shutil.rmtree(lowerCamelCase_)





def        SCREAMING_SNAKE_CASE       (      lowerCamelCase_ ,     lowerCamelCase_):
					# Initialize the tokenizer based on the `spm` model
					a__          =  LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
					print(f'Saving a {tokenizer_class.__name__} to {tokenizer_path}.')
					a__          =  tokenizer_class(lowerCamelCase_)
					tokenizer.save_pretrained(lowerCamelCase_)





def        SCREAMING_SNAKE_CASE       (      ):
					a__          =  argparse.ArgumentParser()
					parser.add_argument(
					    '''--input_dir''' ,     help='''Location of LLaMA weights, which contains tokenizer.model and model folders''' ,     )
					parser.add_argument(
					    '''--model_size''' ,     choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only'''] ,     )
					parser.add_argument(
					    '''--output_dir''' ,     help='''Location to write HF model and tokenizer''' ,     )
					parser.add_argument('''--safe_serialization''' ,     type=lowerCamelCase_ ,     help='''Whether or not to save using `safetensors`.''')
					a__          =  parser.parse_args()
					if args.model_size != "tokenizer_only":
										write_model(
										    model_path=args.output_dir ,     input_base_path=os.path.join(args.input_dir ,     args.model_size) ,     model_size=args.model_size ,     safe_serialization=args.safe_serialization ,     )
					a__          =  os.path.join(args.input_dir ,     '''tokenizer.model''')
					write_tokenizer(args.output_dir ,     lowerCamelCase_)


if __name__ == "__main__":
	main()










"""simple docstring"""







from typing import List, Optional

import numpy as np

from ...processing_utils import ProcessorMixin
from ...utils import to_numpy







class        _SCREAMING_SNAKE_CASE						(  __snake_case							):





							"""simple docstring"""





							_SCREAMING_SNAKE_CASE    ='EncodecFeatureExtractor'
							_SCREAMING_SNAKE_CASE    =('T5Tokenizer', 'T5TokenizerFast')




							def __init__(   self:     List[Any]				,		__A:     Any				,		__A:     Dict    ):

												'''simple docstring'''

												super().__init__(__A				,		__A    )
												a__          =  self.feature_extractor
												a__          =  False




							def  lowercase		(   self:     Union[str, Any]				,		__A:     List[Any]=None				,		__A:     Optional[Any]=None				,		__A:     List[Any]=True    ):

												'''simple docstring'''

												return self.tokenizer.get_decoder_prompt_ids(task=__A				,		language=__A				,		no_timestamps=__A    )




							def __call__(   self:     Union[str, Any]				,		*__A:     int				,		**__A:     Dict    ):

												'''simple docstring'''

												if self._in_target_context_manager:
																	return self.current_processor(*__A				,		**__A    )

												a__          =  kwargs.pop('''audio'''				,		__A    )
												a__          =  kwargs.pop('''sampling_rate'''				,		__A    )
												a__          =  kwargs.pop('''text'''				,		__A    )
												if len(__A    ) > 0:
																	a__          =  args[0]
																	a__          =  args[1:]

												if audio is None and text is None:
																	raise ValueError('''You need to specify either an `audio` or `text` input to process.'''    )

												if text is not None:
																	a__          =  self.tokenizer(__A				,		**__A    )

												if audio is not None:
																	a__          =  self.feature_extractor(__A				,		*__A				,		sampling_rate=__A				,		**__A    )

												if audio is None:
																	return inputs

												elif text is None:
																	return audio_inputs

												else:
																	a__          =  audio_inputs['''input_values''']
																	if "padding_mask" in audio_inputs:
																						a__          =  audio_inputs['''padding_mask''']
																	return inputs




							def  lowercase		(   self:     Union[str, Any]				,		*__A:     List[str]				,		**__A:     Tuple    ):

												'''simple docstring'''

												a__          =  kwargs.pop('''audio'''				,		__A    )
												a__          =  kwargs.pop('''padding_mask'''				,		__A    )

												if len(__A    ) > 0:
																	a__          =  args[0]
																	a__          =  args[1:]

												if audio_values is not None:
																	return self._decode_audio(__A				,		padding_mask=__A    )
												else:
																	return self.tokenizer.batch_decode(*__A				,		**__A    )




							def  lowercase		(   self:     Union[str, Any]				,		*__A:     Optional[int]				,		**__A:     Optional[Any]    ):

												'''simple docstring'''

												return self.tokenizer.decode(*__A				,		**__A    )





							def  lowercase		(   self:     Union[str, Any]				,		__A:     Dict				,		__A:     Optional = None    ):

												'''simple docstring'''

												a__          =  to_numpy(__A    )
												a__       ,a__       ,a__          =  audio_values.shape

												if padding_mask is None:
																	return list(__A    )

												a__          =  to_numpy(__A    )

												# match the sequence length of the padding mask to the generated audio arrays by padding with the **non-padding**
												# token (so that the generated audio values are **not** treated as padded tokens)
												a__          =  seq_len - padding_mask.shape[-1]
												a__          =  1 - self.feature_extractor.padding_value
												a__          =  np.pad(__A				,		((0, 0), (0, difference))				,		'''constant'''				,		constant_values=__A    )

												a__          =  audio_values.tolist()
												for i in range(__A    ):
																	a__          =  np.asarray(audio_values[i]    )[
																	    padding_mask[i][None, :] != self.feature_extractor.padding_value
																	]
																	a__          =  sliced_audio.reshape(__A				,		-1    )

												return audio_values


