






import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
				from transformers import LlamaTokenizerFast
except ImportError as e:
				warnings.warn(e)
				warnings.warn(
				    'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
				)
				a_		:    List[Any]   							=     None
a_		:    str   							=     {
    '7B': 1_10_08,
    '13B': 1_38_24,
    '30B': 1_79_20,
    '65B': 2_20_16,
    '70B': 2_86_72,
}
a_		:    str   							=     {
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}






def 					lowerCamelCase__					(_UpperCAmelCase				,							_UpperCAmelCase=1				,							_UpperCAmelCase=256):
			return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3)) + multiple_of - 1) // multiple_of)






def 					lowerCamelCase__					(_UpperCAmelCase):
			with open(_UpperCAmelCase				,							'r') as f:
						return json.load(_UpperCAmelCase)






def 					lowerCamelCase__					(_UpperCAmelCase				,							_UpperCAmelCase):
			with open(_UpperCAmelCase				,							'w') as f:
						json.dump(_UpperCAmelCase				,							_UpperCAmelCase)






def 					lowerCamelCase__					(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase=True):
			os.makedirs(_UpperCAmelCase				,							exist_ok=_UpperCAmelCase)
			SCREAMING_SNAKE_CASE							=			os.path.join(_UpperCAmelCase				,							'tmp')
			os.makedirs(_UpperCAmelCase				,							exist_ok=_UpperCAmelCase)

			SCREAMING_SNAKE_CASE							=			read_json(os.path.join(_UpperCAmelCase				,							'params.json'))
			SCREAMING_SNAKE_CASE							=			NUM_SHARDS[model_size]
			SCREAMING_SNAKE_CASE							=			params['n_layers']
			SCREAMING_SNAKE_CASE							=			params['n_heads']
			SCREAMING_SNAKE_CASE							=			n_heads // num_shards
			SCREAMING_SNAKE_CASE							=			params['dim']
			SCREAMING_SNAKE_CASE							=			dim // n_heads
			SCREAMING_SNAKE_CASE							=			1_00_00.0
			SCREAMING_SNAKE_CASE							=			1.0 / (base ** (torch.arange(0				,							_UpperCAmelCase				,							2).float() / dims_per_head))

			if "n_kv_heads" in params:
						SCREAMING_SNAKE_CASE							=			params['n_kv_heads']  # for GQA / MQA
						SCREAMING_SNAKE_CASE							=			n_heads_per_shard // num_key_value_heads
						SCREAMING_SNAKE_CASE							=			dim // num_key_value_heads
			else:  # compatibility with other checkpoints
						SCREAMING_SNAKE_CASE							=			n_heads
						SCREAMING_SNAKE_CASE							=			n_heads_per_shard
						SCREAMING_SNAKE_CASE							=			dim

			# permute for sliced rotary
			def permute(_UpperCAmelCase				,							_UpperCAmelCase=n_heads				,							_UpperCAmelCase=dim				,							_UpperCAmelCase=dim):
						return w.view(_UpperCAmelCase				,							dima // n_heads // 2				,							2				,							_UpperCAmelCase).transpose(1				,							2).reshape(_UpperCAmelCase				,							_UpperCAmelCase)

			print(F'''Fetching all parameters from the checkpoint at {input_base_path}.''')
			# Load weights
			if model_size == "7B":
						# Not sharded
						# (The sharded implementation would also work, but this is simpler.)
						SCREAMING_SNAKE_CASE							=			torch.load(os.path.join(_UpperCAmelCase				,							'consolidated.00.pth')				,							map_location='cpu')
			else:
						# Sharded
						SCREAMING_SNAKE_CASE							=			[
						    torch.load(os.path.join(_UpperCAmelCase				,							F'''consolidated.{i:02d}.pth''')				,							map_location='cpu')
						    for i in range(_UpperCAmelCase)
						]
			SCREAMING_SNAKE_CASE							=			0
			SCREAMING_SNAKE_CASE							=			{'weight_map': {}}
			for layer_i in range(_UpperCAmelCase):
						SCREAMING_SNAKE_CASE							=			F'''pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'''
						if model_size == "7B":
									# Unsharded
									SCREAMING_SNAKE_CASE							=			{
									    F'''model.layers.{layer_i}.self_attn.q_proj.weight''': permute(
									        loaded[F'''layers.{layer_i}.attention.wq.weight''']),
									    F'''model.layers.{layer_i}.self_attn.k_proj.weight''': permute(
									        loaded[F'''layers.{layer_i}.attention.wk.weight''']),
									    F'''model.layers.{layer_i}.self_attn.v_proj.weight''': loaded[F'''layers.{layer_i}.attention.wv.weight'''],
									    F'''model.layers.{layer_i}.self_attn.o_proj.weight''': loaded[F'''layers.{layer_i}.attention.wo.weight'''],
									    F'''model.layers.{layer_i}.mlp.gate_proj.weight''': loaded[F'''layers.{layer_i}.feed_forward.w1.weight'''],
									    F'''model.layers.{layer_i}.mlp.down_proj.weight''': loaded[F'''layers.{layer_i}.feed_forward.w2.weight'''],
									    F'''model.layers.{layer_i}.mlp.up_proj.weight''': loaded[F'''layers.{layer_i}.feed_forward.w3.weight'''],
									    F'''model.layers.{layer_i}.input_layernorm.weight''': loaded[F'''layers.{layer_i}.attention_norm.weight'''],
									    F'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[F'''layers.{layer_i}.ffn_norm.weight'''],
									}
						else:
									# Sharded
									# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
									# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
									# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

									SCREAMING_SNAKE_CASE							=			{
									    F'''model.layers.{layer_i}.input_layernorm.weight''': loaded[0][
									        F'''layers.{layer_i}.attention_norm.weight'''
									    ].clone(),
									    F'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[0][
									        F'''layers.{layer_i}.ffn_norm.weight'''
									    ].clone(),
									}
									SCREAMING_SNAKE_CASE							=			permute(
									    torch.cat(
									        [
									            loaded[i][F'''layers.{layer_i}.attention.wq.weight'''].view(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase)
									            for i in range(_UpperCAmelCase)
									        ]				,							dim=0				,							).reshape(_UpperCAmelCase				,							_UpperCAmelCase))
									SCREAMING_SNAKE_CASE							=			permute(
									    torch.cat(
									        [
									            loaded[i][F'''layers.{layer_i}.attention.wk.weight'''].view(
									                _UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase)
									            for i in range(_UpperCAmelCase)
									        ]				,							dim=0				,							).reshape(_UpperCAmelCase				,							_UpperCAmelCase)				,							_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase				,							)
									SCREAMING_SNAKE_CASE							=			torch.cat(
									    [
									        loaded[i][F'''layers.{layer_i}.attention.wv.weight'''].view(
									            _UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase)
									        for i in range(_UpperCAmelCase)
									    ]				,							dim=0				,							).reshape(_UpperCAmelCase				,							_UpperCAmelCase)

									SCREAMING_SNAKE_CASE							=			torch.cat(
									    [loaded[i][F'''layers.{layer_i}.attention.wo.weight'''] for i in range(_UpperCAmelCase)]				,							dim=1)
									SCREAMING_SNAKE_CASE							=			torch.cat(
									    [loaded[i][F'''layers.{layer_i}.feed_forward.w1.weight'''] for i in range(_UpperCAmelCase)]				,							dim=0)
									SCREAMING_SNAKE_CASE							=			torch.cat(
									    [loaded[i][F'''layers.{layer_i}.feed_forward.w2.weight'''] for i in range(_UpperCAmelCase)]				,							dim=1)
									SCREAMING_SNAKE_CASE							=			torch.cat(
									    [loaded[i][F'''layers.{layer_i}.feed_forward.w3.weight'''] for i in range(_UpperCAmelCase)]				,							dim=0)

						SCREAMING_SNAKE_CASE							=			inv_freq
						for k, v in state_dict.items():
									SCREAMING_SNAKE_CASE							=			filename
									param_count += v.numel()
						torch.save(_UpperCAmelCase				,							os.path.join(_UpperCAmelCase				,							_UpperCAmelCase))

			SCREAMING_SNAKE_CASE							=			F'''pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'''
			if model_size == "7B":
						# Unsharded
						SCREAMING_SNAKE_CASE							=			{
						    'model.embed_tokens.weight': loaded['tok_embeddings.weight'],
						    'model.norm.weight': loaded['norm.weight'],
						    'lm_head.weight': loaded['output.weight'],
						}
			else:
						SCREAMING_SNAKE_CASE							=			{
						    'model.norm.weight': loaded[0]['norm.weight'],
						    'model.embed_tokens.weight': torch.cat(
						        [loaded[i]['tok_embeddings.weight'] for i in range(_UpperCAmelCase)]				,							dim=1),
						    'lm_head.weight': torch.cat([loaded[i]['output.weight'] for i in range(_UpperCAmelCase)]				,							dim=0),
						}

			for k, v in state_dict.items():
						SCREAMING_SNAKE_CASE							=			filename
						param_count += v.numel()
			torch.save(_UpperCAmelCase				,							os.path.join(_UpperCAmelCase				,							_UpperCAmelCase))

			# Write configs
			SCREAMING_SNAKE_CASE							=			{'total_size': param_count * 2}
			write_json(_UpperCAmelCase				,							os.path.join(_UpperCAmelCase				,							'pytorch_model.bin.index.json'))
			SCREAMING_SNAKE_CASE							=			params['ffn_dim_multiplier'] if 'ffn_dim_multiplier' in params else 1
			SCREAMING_SNAKE_CASE							=			params['multiple_of'] if 'multiple_of' in params else 256
			SCREAMING_SNAKE_CASE							=			LlamaConfig(
			    hidden_size=_UpperCAmelCase				,							intermediate_size=compute_intermediate_size(_UpperCAmelCase				,							_UpperCAmelCase				,							_UpperCAmelCase)				,							num_attention_heads=params['n_heads']				,							num_hidden_layers=params['n_layers']				,							rms_norm_eps=params['norm_eps']				,							num_key_value_heads=_UpperCAmelCase				,							)
			config.save_pretrained(_UpperCAmelCase)

			# Make space so we can load the model properly now.
			del state_dict
			del loaded
			gc.collect()

			print('Loading the checkpoint in a Llama model.')
			SCREAMING_SNAKE_CASE							=			LlamaForCausalLM.from_pretrained(_UpperCAmelCase				,							torch_dtype=torch.floataa				,							low_cpu_mem_usage=_UpperCAmelCase)
			# Avoid saving this as part of the config.
			del model.config._name_or_path

			print('Saving in the Transformers format.')
			model.save_pretrained(_UpperCAmelCase				,							safe_serialization=_UpperCAmelCase)
			shutil.rmtree(_UpperCAmelCase)






def 					lowerCamelCase__					(_UpperCAmelCase				,							_UpperCAmelCase):
			# Initialize the tokenizer based on the `spm` model
			SCREAMING_SNAKE_CASE							=			LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
			print(F'''Saving a {tokenizer_class.__name__} to {tokenizer_path}.''')
			SCREAMING_SNAKE_CASE							=			tokenizer_class(_UpperCAmelCase)
			tokenizer.save_pretrained(_UpperCAmelCase)






def 					lowerCamelCase__					():
			SCREAMING_SNAKE_CASE							=			argparse.ArgumentParser()
			parser.add_argument(
			    '--input_dir'				,							help='Location of LLaMA weights, which contains tokenizer.model and model folders'				,							)
			parser.add_argument(
			    '--model_size'				,							choices=['7B', '7Bf', '13B', '13Bf', '30B', '65B', '70B', '70Bf', 'tokenizer_only']				,							)
			parser.add_argument(
			    '--output_dir'				,							help='Location to write HF model and tokenizer'				,							)
			parser.add_argument('--safe_serialization'				,							type=_UpperCAmelCase				,							help='Whether or not to save using `safetensors`.')
			SCREAMING_SNAKE_CASE							=			parser.parse_args()
			if args.model_size != "tokenizer_only":
						write_model(
						    model_path=args.output_dir				,							input_base_path=os.path.join(args.input_dir				,							args.model_size)				,							model_size=args.model_size				,							safe_serialization=args.safe_serialization				,							)
			SCREAMING_SNAKE_CASE							=			os.path.join(args.input_dir				,							'tokenizer.model')
			write_tokenizer(args.output_dir				,							_UpperCAmelCase)


if __name__ == "__main__":
				main()






'''simple docstring'''
import json
import os
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase__											=logging.get_logger(__name__)


lowercase__											={
    'vocab_file': 'vocab.json',
    'tokenizer_config_file': 'tokenizer_config.json',
    'merges_file': 'merges.txt',
}

lowercase__											={
    'vocab_file': {
        'facebook/s2t-wav2vec2-large-en-de': (
            'https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/vocab.json'
        ),
    },
    'tokenizer_config_file': {
        'facebook/s2t-wav2vec2-large-en-de': (
            'https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/tokenizer_config.json'
        ),
    },
    'merges_file': {
        'facebook/s2t-wav2vec2-large-en-de': (
            'https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/merges.txt'
        ),
    },
}

lowercase__											='</w>'
lowercase__											='@@ '






def      UpperCamelCase_				(      A__						):
	a_      				=						set()
	a_      				=						word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)						)
		a_      				=						char
	return pairs


# Speech2Text2 has no max input length
lowercase__											={'facebook/s2t-wav2vec2-large-en-de': 10_24}







class   a_ (      UpperCamelCase__					):
			lowerCamelCase__	:			List[str]       =    VOCAB_FILES_NAMES
			lowerCamelCase__	:			Optional[Any]       =    PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase__	:			List[Any]       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase__	:			int       =    ['input_ids', 'attention_mask']




			def __init__(       self					,   UpperCAmelCase					,   UpperCAmelCase="<s>"					,   UpperCAmelCase="<pad>"					,   UpperCAmelCase="</s>"					,   UpperCAmelCase="<unk>"					,   UpperCAmelCase=False					,   UpperCAmelCase=None					,   **UpperCAmelCase					,   ):
				super().__init__(
				    unk_token=UpperCAmelCase					,   bos_token=UpperCAmelCase					,   eos_token=UpperCAmelCase					,   pad_token=UpperCAmelCase					,   do_lower_case=UpperCAmelCase					,   **UpperCAmelCase					,   )

				a_      				=						do_lower_case

				with open(UpperCAmelCase					,   encoding="""utf-8"""	) as vocab_handle:
					a_      				=						json.load(UpperCAmelCase	)
				a_      				=						{v: k for k, v in self.encoder.items()}

				if merges_file is None:
					logger.info(f'''No merges files provided. {self.__class__.__name__} can only be used for decoding.'''	)

					a_      				=						None
					a_      				=						None
				else:
					with open(UpperCAmelCase					,   encoding="""utf-8"""	) as merges_handle:
						a_      				=						merges_handle.read().split("""\n"""	)[:-1]

					a_      				=						[tuple(merge.split()[:2]	) for merge in merges]
					a_      				=						dict(zip(UpperCAmelCase					,   range(len(UpperCAmelCase	)	)	)	)
					a_      				=						{}




			@property
			def 					lowerCAmelCase__					(       self	):
				return len(self.decoder	)




			def 					lowerCAmelCase__					(       self	):
				return dict(self.encoder					,   **self.added_tokens_encoder	)




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase	):
				a_      				=						tuple(token[:-1]	) + (token[-1] + BPE_TOKEN_MERGES,)
				if token in self.cache:
					return self.cache[token]
				a_      				=						get_pairs(UpperCAmelCase	)

				if not pairs:
					return token

				while True:
					a_      				=						min(UpperCAmelCase					,   key=lambda UpperCAmelCase	: self.bpe_ranks.get(UpperCAmelCase					,   float("""inf"""	)	)	)
					if bigram not in self.bpe_ranks:
						break
					a_				,  a_      				=						bigram
					a_      				=						[]
					a_      				=						0
					while i < len(UpperCAmelCase	):
						try:
							a_      				=						word.index(UpperCAmelCase					,   UpperCAmelCase	)
						except ValueError:
							new_word.extend(word[i:]	)
							break
						else:
							new_word.extend(word[i:j]	)
							a_      				=						j

						if word[i] == first and i < len(UpperCAmelCase	) - 1 and word[i + 1] == second:
							new_word.append(first + second	)
							i += 2
						else:
							new_word.append(word[i]	)
							i += 1
					a_      				=						tuple(UpperCAmelCase	)
					a_      				=						new_word
					if len(UpperCAmelCase	) == 1:
						break
					else:
						a_      				=						get_pairs(UpperCAmelCase	)
				a_      				=						""" """.join(UpperCAmelCase	)
				if word == "\n  " + BPE_TOKEN_MERGES:
					a_      				=						"""\n""" + BPE_TOKEN_MERGES

				if word.endswith(UpperCAmelCase	):
					a_      				=						word.replace(UpperCAmelCase					,   """"""	)

				a_      				=						word.replace(""" """					,   UpperCAmelCase	)
				a_      				=						word
				return word




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase	):

				if self.bpe_ranks is None:
					raise ValueError(
					    """This tokenizer was instantiated without a `merges.txt` file, so"""
					    """ that it can only be used for decoding, not for encoding."""
					    """Make sure to provide `merges.txt` file at instantiation to enable """
					    """encoding."""	)

				if self.do_lower_case:
					a_      				=						text.lower()

				a_      				=						text.split()

				a_      				=						[]
				for token in text:
					if token:
						split_tokens.extend(list(self.bpe(UpperCAmelCase	).split(""" """	)	)	)

				return split_tokens




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase	):
				return self.encoder.get(UpperCAmelCase					,   self.encoder.get(self.unk_token	)	)




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase	):
				a_      				=						self.decoder.get(UpperCAmelCase					,   self.unk_token	)
				return result




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase	):
				a_      				=						""" """.join(UpperCAmelCase	)

				# make sure @@ tokens are concatenated
				a_      				=						"""""".join(string.split(UpperCAmelCase	)	)

				return string






			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase					,   UpperCAmelCase = None	):
				if not os.path.isdir(UpperCAmelCase	):
					logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''	)
					return
				a_      				=						os.path.join(
				    UpperCAmelCase					,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)
				a_      				=						os.path.join(
				    UpperCAmelCase					,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]	)

				with open(UpperCAmelCase					,   """w"""					,   encoding="""utf-8"""	) as f:
					f.write(json.dumps(self.encoder					,   indent=2					,   sort_keys=UpperCAmelCase					,   ensure_ascii=UpperCAmelCase	) + """\n"""	)

				a_      				=						0
				if self.bpe_ranks is None:
					return (vocab_file,)

				with open(UpperCAmelCase					,   """w"""					,   encoding="""utf-8"""	) as writer:
					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,   key=lambda UpperCAmelCase	: kv[1]	):
						if index != token_index:
							logger.warning(
							    f'''Saving vocabulary to {merges_file}: BPE merge indices are not consecutive.'''
							    """ Please check that the tokenizer is not corrupted!"""	)
							a_      				=						token_index
						writer.write(""" """.join(UpperCAmelCase	) + """\n"""	)
						index += 1

				return (vocab_file, merges_file)
