




'''simple docstring'''


# Lint as: python3
import os
import re
import urllib.parse
from pathlib import Path
from typing import Callable, List, Optional, Union
from zipfile import ZipFile

from ..utils.file_utils import cached_path, hf_github_url
from ..utils.logging import get_logger
from ..utils.version import Version


_snake_case   :   List[Any]     							=	get_logger(__name__)

class     A					:
							lowercase_           =       'dummy_data'
							lowercase_           =       'datasets'
							lowercase_           =       False







							def __init__(						self :						Dict							,    lowerCAmelCase_ :						str							,    lowerCAmelCase_ :						str							,    lowerCAmelCase_ :						Union[Version, str]							,    lowerCAmelCase_ :						Optional[str] = None							,    lowerCAmelCase_ :						bool = False							,    lowerCAmelCase_ :						bool = True							,    lowerCAmelCase_ :						Optional[List[Callable]] = None							,    )    ->							Any:


														"""simple docstring"""


														_a			  =       0
														_a			  =       dataset_name
														_a			  =       cache_dir
														_a			  =       use_local_dummy_data
														_a			  =       config
														# download_callbacks take a single url as input
														_a			  =       download_callbacks or []
														# if False, it doesn't load existing files and it returns the paths of the dummy files relative
														# to the dummy_data zip file root
														_a			  =       load_existing_dummy_data

														# TODO(PVP, QL) might need to make this more general
														_a			  =       str(lowerCAmelCase_      )
														# to be downloaded
														_a			  =       None
														_a			  =       None







							@property
							def      __lowerCAmelCase    (						self :						str      )    ->							Any:


														"""simple docstring"""


														if self._dummy_file is None:
																					_a			  =       self.download_dummy_data()
														return self._dummy_file







							@property
							def      __lowerCAmelCase    (						self :						List[str]      )    ->							List[Any]:


														"""simple docstring"""


														if self.config is not None:
																					# structure is dummy / config_name / version_name
																					return os.path.join('''dummy'''							,    self.config.name							,    self.version_name      )
														# structure is dummy / version_name
														return os.path.join('''dummy'''							,    self.version_name      )







							@property
							def      __lowerCAmelCase    (						self :						Any      )    ->							str:


														"""simple docstring"""


														return os.path.join(self.dummy_data_folder							,    '''dummy_data.zip'''      )







							def      __lowerCAmelCase    (						self :						str      )    ->							int:


														"""simple docstring"""


														_a			  =       (
														    self.local_path_to_dummy_data if self.use_local_dummy_data is True else self.github_path_to_dummy_data
														)

														_a			  =       cached_path(
														    lowerCAmelCase_							,    cache_dir=self.cache_dir							,    extract_compressed_file=lowerCAmelCase_							,    force_extract=lowerCAmelCase_      )

														return os.path.join(lowerCAmelCase_							,    self.dummy_file_name      )







							@property
							def      __lowerCAmelCase    (						self :						Tuple      )    ->							str:


														"""simple docstring"""


														return os.path.join(self.datasets_scripts_dir							,    self.dataset_name							,    self.dummy_zip_file      )







							@property
							def      __lowerCAmelCase    (						self :						Union[str, Any]      )    ->							Union[str, Any]:


														"""simple docstring"""


														if self._bucket_url is None:
																					_a			  =       hf_github_url(self.dataset_name							,    self.dummy_zip_file.replace(os.sep							,    '''/'''      )      )
														return self._bucket_url







							@property
							def      __lowerCAmelCase    (						self :						int      )    ->							str:


														"""simple docstring"""


														if os.path.isdir(self.dummy_file      ):
																					return self.dummy_file
														# else cut off path to file -> example `xsum`.
														return "/".join(self.dummy_file.replace(os.sep							,    '''/'''      ).split('''/'''      )[:-1]      )







							def      __lowerCAmelCase    (						self :						Dict							,    lowerCAmelCase_ :						Optional[Any]							,    *lowerCAmelCase_ :						List[Any]      )    ->							Optional[int]:


														"""simple docstring"""


														if self.load_existing_dummy_data:
																					# dummy data is downloaded and tested
																					_a			  =       self.dummy_file
														else:
																					# dummy data cannot be downloaded and only the path to dummy file is returned
																					_a			  =       self.dummy_file_name

														# special case when data_url is a dict
														if isinstance(lowerCAmelCase_							,    lowerCAmelCase_      ):
																					return self.create_dummy_data_dict(lowerCAmelCase_							,    lowerCAmelCase_      )
														elif isinstance(lowerCAmelCase_							,    (list, tuple)      ):
																					return self.create_dummy_data_list(lowerCAmelCase_							,    lowerCAmelCase_      )
														else:
																					return self.create_dummy_data_single(lowerCAmelCase_							,    lowerCAmelCase_      )







							def      __lowerCAmelCase    (						self :						Tuple							,    lowerCAmelCase_ :						Optional[int]							,    *lowerCAmelCase_ :						int      )    ->							List[str]:


														"""simple docstring"""


														return self.download_and_extract(lowerCAmelCase_      )







							def      __lowerCAmelCase    (						self :						int							,    lowerCAmelCase_ :						List[str]							,    lowerCAmelCase_ :						str      )    ->							str:


														"""simple docstring"""


														return self.download_and_extract(lowerCAmelCase_      )







							def      __lowerCAmelCase    (						self :						Tuple							,    lowerCAmelCase_ :						List[Any]							,    *lowerCAmelCase_ :						List[Any]							,    **lowerCAmelCase_ :						Any      )    ->							Optional[int]:


														"""simple docstring"""


														return path







							def      __lowerCAmelCase    (						self :						Union[str, Any]      )    ->							Union[str, Any]:


														"""simple docstring"""


														return {}







							def      __lowerCAmelCase    (						self :						str							,    lowerCAmelCase_ :						Dict							,    lowerCAmelCase_ :						List[Any]      )    ->							Union[str, Any]:


														"""simple docstring"""


														_a			  =       {}
														for key, single_urls in data_url.items():
																					for download_callback in self.download_callbacks:
																												if isinstance(lowerCAmelCase_							,    lowerCAmelCase_      ):
																																			for single_url in single_urls:
																																										download_callback(lowerCAmelCase_      )
																												else:
																																			_a			  =       single_urls
																																			download_callback(lowerCAmelCase_      )
            # we force the name of each key to be the last file / folder name of the url path
            # if the url has arguments, we need to encode them with urllib.parse.quote_plus
																					if isinstance(lowerCAmelCase_							,    lowerCAmelCase_      ):
																												_a			  =       [os.path.join(lowerCAmelCase_							,    urllib.parse.quote_plus(Path(lowerCAmelCase_      ).name      )      ) for x in single_urls]
																					else:
																												_a			  =       single_urls
																												_a			  =       os.path.join(lowerCAmelCase_							,    urllib.parse.quote_plus(Path(lowerCAmelCase_      ).name      )      )
																					_a			  =       value

														# make sure that values are unique
														if all(isinstance(lowerCAmelCase_							,    lowerCAmelCase_      ) for i in dummy_data_dict.values()      ) and len(set(dummy_data_dict.values()      )      ) < len(
														    dummy_data_dict.values()      ):
																					# append key to value to make its name unique
																					_a			  =       {key: value + key for key, value in dummy_data_dict.items()}

														return dummy_data_dict







							def      __lowerCAmelCase    (						self :						Any							,    lowerCAmelCase_ :						List[Any]							,    lowerCAmelCase_ :						str      )    ->							Optional[Any]:


														"""simple docstring"""


														_a			  =       []
														# trick: if there are many shards named like `data.txt-000001-of-00300`, only use the first one
														_a			  =       all(bool(re.findall('''[0-9]{3,}-of-[0-9]{3,}'''							,    lowerCAmelCase_      )      ) for url in data_url      )
														_a			  =       all(
														    url.startswith('''https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmed'''      ) for url in data_url      )
														if data_url and (is_tf_records or is_pubmed_records):
																					_a			  =       [data_url[0]] * len(lowerCAmelCase_      )
														for single_url in data_url:
																					for download_callback in self.download_callbacks:
																												download_callback(lowerCAmelCase_      )
																					# we force the name of each key to be the last file / folder name of the url path
																					# if the url has arguments, we need to encode them with urllib.parse.quote_plus
																					_a			  =       os.path.join(lowerCAmelCase_							,    urllib.parse.quote_plus(single_url.split('''/'''      )[-1]      )      )
																					dummy_data_list.append(lowerCAmelCase_      )
														return dummy_data_list







							def      __lowerCAmelCase    (						self :						Any							,    lowerCAmelCase_ :						List[str]							,    lowerCAmelCase_ :						Tuple      )    ->							Optional[Any]:


														"""simple docstring"""


														for download_callback in self.download_callbacks:
																					download_callback(lowerCAmelCase_      )
														# we force the name of each key to be the last file / folder name of the url path
														# if the url has arguments, we need to encode them with urllib.parse.quote_plus
														_a			  =       os.path.join(lowerCAmelCase_							,    urllib.parse.quote_plus(data_url.split('''/'''      )[-1]      )      )
														if os.path.exists(lowerCAmelCase_      ) or not self.load_existing_dummy_data:
																					return value
														else:
																					# Backward compatibility, maybe deprecate at one point.
																					# For many datasets with single url calls to dl_manager.download_and_extract,
																					# the dummy_data.zip file is actually the zipped downloaded file
																					# while now we expected the dummy_data.zip file to be a directory containing
																					# the downloaded file.
																					return path_to_dummy_data







							def      __lowerCAmelCase    (						self :						Optional[Any]      )    ->							int:


														"""simple docstring"""


														pass







							def      __lowerCAmelCase    (						self :						Optional[Any]      )    ->							Optional[Any]:


														"""simple docstring"""


														pass







							def      __lowerCAmelCase    (						self :						List[str]							,    lowerCAmelCase_ :						List[Any]      )    ->							Dict:


														"""simple docstring"""


														def _iter_archive_members(lowerCAmelCase_ :						Tuple      ):
																					# this preserves the order of the members inside the ZIP archive
																					_a			  =       Path(self.dummy_file      ).parent
																					_a			  =       path.relative_to(lowerCAmelCase_      )
																					with ZipFile(self.local_path_to_dummy_data      ) as zip_file:
																												_a			  =       zip_file.namelist()
																					for member in members:
																												if member.startswith(relative_path.as_posix()      ):
																																			yield dummy_parent_path.joinpath(lowerCAmelCase_      )

														_a			  =       Path(lowerCAmelCase_      )
														_a			  =       _iter_archive_members(lowerCAmelCase_      ) if self.use_local_dummy_data else path.rglob('''*'''      )
														for file_path in file_paths:
																					if file_path.is_file() and not file_path.name.startswith(('''.''', '''__''')      ):
																												yield file_path.relative_to(lowerCAmelCase_      ).as_posix(), file_path.open('''rb'''      )







							def      __lowerCAmelCase    (						self :						Tuple							,    lowerCAmelCase_ :						Dict      )    ->							Dict:


														"""simple docstring"""


														if not isinstance(lowerCAmelCase_							,    lowerCAmelCase_      ):
																					_a			  =       [paths]
														for path in paths:
																					if os.path.isfile(lowerCAmelCase_      ):
																												if os.path.basename(lowerCAmelCase_      ).startswith(('''.''', '''__''')      ):
																																			return
																												yield path
																					else:
																												for dirpath, dirnames, filenames in os.walk(lowerCAmelCase_      ):
																																			if os.path.basename(lowerCAmelCase_      ).startswith(('''.''', '''__''')      ):
																																										continue
																																			dirnames.sort()
																																			for filename in sorted(lowerCAmelCase_      ):
																																										if filename.startswith(('''.''', '''__''')      ):
																																																	continue
																																										yield os.path.join(lowerCAmelCase_							,    lowerCAmelCase_      )







'''simple docstring'''


import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
				from transformers import LlamaTokenizerFast
except ImportError as e:
				warnings.warn(e)
				warnings.warn(
				    'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
				)
				_snake_case   :   int     							=	None
_snake_case   :   Tuple     							=	{
    '7B': 11008,
    '13B': 13824,
    '30B': 17920,
    '65B': 22016,
    '70B': 28672,
}
_snake_case   :   int     							=	{
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}
def 	snake_case_					(UpperCamelCase :						Union[str, Any]				,		UpperCamelCase :						Optional[Any]=1				,		UpperCamelCase :						Optional[Any]=256   ):





							'''simple docstring'''


							return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3   )   ) + multiple_of - 1) // multiple_of)
def 	snake_case_					(UpperCamelCase :						Optional[int]   ):





							'''simple docstring'''


							with open(UpperCamelCase				,		'''r'''   ) as f:
														return json.load(UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						str				,		UpperCamelCase :						str   ):





							'''simple docstring'''


							with open(UpperCamelCase				,		'''w'''   ) as f:
														json.dump(UpperCamelCase				,		UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						Optional[Any]				,		UpperCamelCase :						List[str]				,		UpperCamelCase :						Union[str, Any]				,		UpperCamelCase :						str=True   ):





							'''simple docstring'''


							os.makedirs(UpperCamelCase				,		exist_ok=UpperCamelCase   )
							_a			  =       os.path.join(UpperCamelCase				,		'''tmp'''   )
							os.makedirs(UpperCamelCase				,		exist_ok=UpperCamelCase   )

							_a			  =       read_json(os.path.join(UpperCamelCase				,		'''params.json'''   )   )
							_a			  =       NUM_SHARDS[model_size]
							_a			  =       params['''n_layers''']
							_a			  =       params['''n_heads''']
							_a			  =       n_heads // num_shards
							_a			  =       params['''dim''']
							_a			  =       dim // n_heads
							_a			  =       10000.0
							_a			  =       1.0 / (base ** (torch.arange(0				,		UpperCamelCase				,		2   ).float() / dims_per_head))

							if "n_kv_heads" in params:
														_a			  =       params['''n_kv_heads''']  # for GQA / MQA
														_a			  =       n_heads_per_shard // num_key_value_heads
														_a			  =       dim // num_key_value_heads
							else:  # compatibility with other checkpoints
														_a			  =       n_heads
														_a			  =       n_heads_per_shard
														_a			  =       dim

							# permute for sliced rotary
							def permute(UpperCamelCase :						List[str]				,		UpperCamelCase :						str=n_heads				,		UpperCamelCase :						int=dim				,		UpperCamelCase :						Optional[Any]=dim   ):
														return w.view(UpperCamelCase				,		dima // n_heads // 2				,		2				,		UpperCamelCase   ).transpose(1				,		2   ).reshape(UpperCamelCase				,		UpperCamelCase   )

							print(f'Fetching all parameters from the checkpoint at {input_base_path}.'   )
							# Load weights
							if model_size == "7B":
														# Not sharded
														# (The sharded implementation would also work, but this is simpler.)
														_a			  =       torch.load(os.path.join(UpperCamelCase				,		'''consolidated.00.pth'''   )				,		map_location='''cpu'''   )
							else:
														# Sharded
														_a			  =       [
														    torch.load(os.path.join(UpperCamelCase				,		f'consolidated.{i:02d}.pth'   )				,		map_location='''cpu'''   )
														    for i in range(UpperCamelCase   )
														]
							_a			  =       0
							_a			  =       {'''weight_map''': {}}
							for layer_i in range(UpperCamelCase   ):
														_a			  =       f'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'
														if model_size == "7B":
																					# Unsharded
																					_a			  =       {
																					    f'model.layers.{layer_i}.self_attn.q_proj.weight': permute(
																					        loaded[f'layers.{layer_i}.attention.wq.weight']   ),
																					    f'model.layers.{layer_i}.self_attn.k_proj.weight': permute(
																					        loaded[f'layers.{layer_i}.attention.wk.weight']   ),
																					    f'model.layers.{layer_i}.self_attn.v_proj.weight': loaded[f'layers.{layer_i}.attention.wv.weight'],
																					    f'model.layers.{layer_i}.self_attn.o_proj.weight': loaded[f'layers.{layer_i}.attention.wo.weight'],
																					    f'model.layers.{layer_i}.mlp.gate_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w1.weight'],
																					    f'model.layers.{layer_i}.mlp.down_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w2.weight'],
																					    f'model.layers.{layer_i}.mlp.up_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w3.weight'],
																					    f'model.layers.{layer_i}.input_layernorm.weight': loaded[f'layers.{layer_i}.attention_norm.weight'],
																					    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[f'layers.{layer_i}.ffn_norm.weight'],
																					}
														else:
																					# Sharded
																					# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
																					# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
																					# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

																					_a			  =       {
																					    f'model.layers.{layer_i}.input_layernorm.weight': loaded[0][
																					        f'layers.{layer_i}.attention_norm.weight'
																					    ].clone(),
																					    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[0][
																					        f'layers.{layer_i}.ffn_norm.weight'
																					    ].clone(),
																					}
																					_a			  =       permute(
																					    torch.cat(
																					        [
																					            loaded[i][f'layers.{layer_i}.attention.wq.weight'].view(UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase   )
																					            for i in range(UpperCamelCase   )
																					        ]				,		dim=0				,		).reshape(UpperCamelCase				,		UpperCamelCase   )   )
																					_a			  =       permute(
																					    torch.cat(
																					        [
																					            loaded[i][f'layers.{layer_i}.attention.wk.weight'].view(
																					                UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase   )
																					            for i in range(UpperCamelCase   )
																					        ]				,		dim=0				,		).reshape(UpperCamelCase				,		UpperCamelCase   )				,		UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase				,		)
																					_a			  =       torch.cat(
																					    [
																					        loaded[i][f'layers.{layer_i}.attention.wv.weight'].view(
																					            UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase   )
																					        for i in range(UpperCamelCase   )
																					    ]				,		dim=0				,		).reshape(UpperCamelCase				,		UpperCamelCase   )

																					_a			  =       torch.cat(
																					    [loaded[i][f'layers.{layer_i}.attention.wo.weight'] for i in range(UpperCamelCase   )]				,		dim=1   )
																					_a			  =       torch.cat(
																					    [loaded[i][f'layers.{layer_i}.feed_forward.w1.weight'] for i in range(UpperCamelCase   )]				,		dim=0   )
																					_a			  =       torch.cat(
																					    [loaded[i][f'layers.{layer_i}.feed_forward.w2.weight'] for i in range(UpperCamelCase   )]				,		dim=1   )
																					_a			  =       torch.cat(
																					    [loaded[i][f'layers.{layer_i}.feed_forward.w3.weight'] for i in range(UpperCamelCase   )]				,		dim=0   )

														_a			  =       inv_freq
														for k, v in state_dict.items():
																					_a			  =       filename
																					param_count += v.numel()
														torch.save(UpperCamelCase				,		os.path.join(UpperCamelCase				,		UpperCamelCase   )   )

							_a			  =       f'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'
							if model_size == "7B":
														# Unsharded
														_a			  =       {
														    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
														    '''model.norm.weight''': loaded['''norm.weight'''],
														    '''lm_head.weight''': loaded['''output.weight'''],
														}
							else:
														_a			  =       {
														    '''model.norm.weight''': loaded[0]['''norm.weight'''],
														    '''model.embed_tokens.weight''': torch.cat(
														        [loaded[i]['''tok_embeddings.weight'''] for i in range(UpperCamelCase   )]				,		dim=1   ),
														    '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(UpperCamelCase   )]				,		dim=0   ),
														}

							for k, v in state_dict.items():
														_a			  =       filename
														param_count += v.numel()
							torch.save(UpperCamelCase				,		os.path.join(UpperCamelCase				,		UpperCamelCase   )   )

							# Write configs
							_a			  =       {'''total_size''': param_count * 2}
							write_json(UpperCamelCase				,		os.path.join(UpperCamelCase				,		'''pytorch_model.bin.index.json'''   )   )
							_a			  =       params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
							_a			  =       params['''multiple_of'''] if '''multiple_of''' in params else 256
							_a			  =       LlamaConfig(
							    hidden_size=UpperCamelCase				,		intermediate_size=compute_intermediate_size(UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase   )				,		num_attention_heads=params['''n_heads''']				,		num_hidden_layers=params['''n_layers''']				,		rms_norm_eps=params['''norm_eps''']				,		num_key_value_heads=UpperCamelCase				,		)
							config.save_pretrained(UpperCamelCase   )

							# Make space so we can load the model properly now.
							del state_dict
							del loaded
							gc.collect()

							print('''Loading the checkpoint in a Llama model.'''   )
							_a			  =       LlamaForCausalLM.from_pretrained(UpperCamelCase				,		torch_dtype=torch.floataa				,		low_cpu_mem_usage=UpperCamelCase   )
							# Avoid saving this as part of the config.
							del model.config._name_or_path

							print('''Saving in the Transformers format.'''   )
							model.save_pretrained(UpperCamelCase				,		safe_serialization=UpperCamelCase   )
							shutil.rmtree(UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						Optional[int]				,		UpperCamelCase :						Tuple   ):





							'''simple docstring'''


							_a			  =       LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
							print(f'Saving a {tokenizer_class.__name__} to {tokenizer_path}.'   )
							_a			  =       tokenizer_class(UpperCamelCase   )
							tokenizer.save_pretrained(UpperCamelCase   )
def 	snake_case_					():





							'''simple docstring'''


							_a			  =       argparse.ArgumentParser()
							parser.add_argument(
							    '''--input_dir'''				,		help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''				,		)
							parser.add_argument(
							    '''--model_size'''				,		choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']				,		)
							parser.add_argument(
							    '''--output_dir'''				,		help='''Location to write HF model and tokenizer'''				,		)
							parser.add_argument('''--safe_serialization'''				,		type=UpperCamelCase				,		help='''Whether or not to save using `safetensors`.'''   )
							_a			  =       parser.parse_args()
							if args.model_size != "tokenizer_only":
														write_model(
														    model_path=args.output_dir				,		input_base_path=os.path.join(args.input_dir				,		args.model_size   )				,		model_size=args.model_size				,		safe_serialization=args.safe_serialization				,		)
							_a			  =       os.path.join(args.input_dir				,		'''tokenizer.model'''   )
							write_tokenizer(args.output_dir				,		UpperCamelCase   )


if __name__ == "__main__":
				main()

