






'''simple docstring'''







import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
		from transformers import LlamaTokenizerFast
except ImportError as e:
		warnings.warn(e)
		warnings.warn(
		    'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
		)
		__lowerCAmelCase									=   None
__lowerCAmelCase									=   {
    '7B': 11_008,
    '13B': 13_824,
    '30B': 17_920,
    '65B': 22_016,
    '70B': 28_672,
}
__lowerCAmelCase									=   {
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}





def 		_UpperCAmelCase (				__A				: Dict    ,		__A				: int=1    ,		__A				: Union[str, Any]=2_56   ):
				return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3   )   ) + multiple_of - 1) // multiple_of)





def 		_UpperCAmelCase (				__A				: int   ):
				with open(lowerCAmelCase_    ,		'''r'''   ) as f:
								return json.load(lowerCAmelCase_   )





def 		_UpperCAmelCase (				__A				: Optional[int]    ,		__A				: Tuple   ):
				with open(lowerCAmelCase_    ,		'''w'''   ) as f:
								json.dump(lowerCAmelCase_    ,		lowerCAmelCase_   )





def 		_UpperCAmelCase (				__A				: Optional[int]    ,		__A				: int    ,		__A				: Union[str, Any]    ,		__A				: List[str]=True   ):
				os.makedirs(lowerCAmelCase_    ,		exist_ok=lowerCAmelCase_   )
				a_		:	str									=   os.path.join(lowerCAmelCase_    ,		'''tmp'''   )
				os.makedirs(lowerCAmelCase_    ,		exist_ok=lowerCAmelCase_   )

				a_		:	List[str]									=   read_json(os.path.join(lowerCAmelCase_    ,		'''params.json'''   )   )
				a_		:	int									=   NUM_SHARDS[model_size]
				a_		:	str									=   params['''n_layers''']
				a_		:	Optional[int]									=   params['''n_heads''']
				a_		:	List[Any]									=   n_heads // num_shards
				a_		:	str									=   params['''dim''']
				a_		:	Dict									=   dim // n_heads
				a_		:	Union[str, Any]									=   1_00_00.0
				a_		:	List[Any]									=   1.0 / (base ** (torch.arange(0    ,		lowerCAmelCase_    ,		2   ).float() / dims_per_head))

				if "n_kv_heads" in params:
								a_		:	Optional[Any]									=   params['''n_kv_heads''']  # for GQA / MQA
								a_		:	List[Any]									=   n_heads_per_shard // num_key_value_heads
								a_		:	Union[str, Any]									=   dim // num_key_value_heads
				else:  # compatibility with other checkpoints
								a_		:	str									=   n_heads
								a_		:	int									=   n_heads_per_shard
								a_		:	Union[str, Any]									=   dim

				# permute for sliced rotary
				def permute(__A				: str    ,		__A				: Optional[int]=n_heads    ,		__A				: Union[str, Any]=dim    ,		__A				: Tuple=dim   ):
								return w.view(lowerCAmelCase_    ,		dima // n_heads // 2    ,		2    ,		lowerCAmelCase_   ).transpose(1    ,		2   ).reshape(lowerCAmelCase_    ,		lowerCAmelCase_   )

				print(f'Fetching all parameters from the checkpoint at {input_base_path}.'   )
				# Load weights
				if model_size == "7B":
								# Not sharded
								# (The sharded implementation would also work, but this is simpler.)
								a_		:	List[str]									=   torch.load(os.path.join(lowerCAmelCase_    ,		'''consolidated.00.pth'''   )    ,		map_location='''cpu'''   )
				else:
								# Sharded
								a_		:	int									=   [
								    torch.load(os.path.join(lowerCAmelCase_    ,		f'consolidated.{i:02d}.pth'   )    ,		map_location='''cpu'''   )
								    for i in range(lowerCAmelCase_   )
								]
				a_		:	List[str]									=   0
				a_		:	Any									=   {'''weight_map''': {}}
				for layer_i in range(lowerCAmelCase_   ):
								a_		:	Dict									=   f'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'
								if model_size == "7B":
												# Unsharded
												a_		:	List[str]									=   {
												    f'model.layers.{layer_i}.self_attn.q_proj.weight': permute(
												        loaded[f'layers.{layer_i}.attention.wq.weight']   ),
												    f'model.layers.{layer_i}.self_attn.k_proj.weight': permute(
												        loaded[f'layers.{layer_i}.attention.wk.weight']   ),
												    f'model.layers.{layer_i}.self_attn.v_proj.weight': loaded[f'layers.{layer_i}.attention.wv.weight'],
												    f'model.layers.{layer_i}.self_attn.o_proj.weight': loaded[f'layers.{layer_i}.attention.wo.weight'],
												    f'model.layers.{layer_i}.mlp.gate_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w1.weight'],
												    f'model.layers.{layer_i}.mlp.down_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w2.weight'],
												    f'model.layers.{layer_i}.mlp.up_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w3.weight'],
												    f'model.layers.{layer_i}.input_layernorm.weight': loaded[f'layers.{layer_i}.attention_norm.weight'],
												    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[f'layers.{layer_i}.ffn_norm.weight'],
												}
								else:
												# Sharded
												# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
												# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
												# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

												a_		:	List[Any]									=   {
												    f'model.layers.{layer_i}.input_layernorm.weight': loaded[0][
												        f'layers.{layer_i}.attention_norm.weight'
												    ].clone(),
												    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[0][
												        f'layers.{layer_i}.ffn_norm.weight'
												    ].clone(),
												}
												a_		:	Tuple									=   permute(
												    torch.cat(
												        [
												            loaded[i][f'layers.{layer_i}.attention.wq.weight'].view(lowerCAmelCase_    ,		lowerCAmelCase_    ,		lowerCAmelCase_   )
												            for i in range(lowerCAmelCase_   )
												        ]    ,		dim=0    ,		).reshape(lowerCAmelCase_    ,		lowerCAmelCase_   )   )
												a_		:	Tuple									=   permute(
												    torch.cat(
												        [
												            loaded[i][f'layers.{layer_i}.attention.wk.weight'].view(
												                lowerCAmelCase_    ,		lowerCAmelCase_    ,		lowerCAmelCase_   )
												            for i in range(lowerCAmelCase_   )
												        ]    ,		dim=0    ,		).reshape(lowerCAmelCase_    ,		lowerCAmelCase_   )    ,		lowerCAmelCase_    ,		lowerCAmelCase_    ,		lowerCAmelCase_    ,		)
												a_		:	List[Any]									=   torch.cat(
												    [
												        loaded[i][f'layers.{layer_i}.attention.wv.weight'].view(
												            lowerCAmelCase_    ,		lowerCAmelCase_    ,		lowerCAmelCase_   )
												        for i in range(lowerCAmelCase_   )
												    ]    ,		dim=0    ,		).reshape(lowerCAmelCase_    ,		lowerCAmelCase_   )

												a_		:	Tuple									=   torch.cat(
												    [loaded[i][f'layers.{layer_i}.attention.wo.weight'] for i in range(lowerCAmelCase_   )]    ,		dim=1   )
												a_		:	str									=   torch.cat(
												    [loaded[i][f'layers.{layer_i}.feed_forward.w1.weight'] for i in range(lowerCAmelCase_   )]    ,		dim=0   )
												a_		:	Optional[Any]									=   torch.cat(
												    [loaded[i][f'layers.{layer_i}.feed_forward.w2.weight'] for i in range(lowerCAmelCase_   )]    ,		dim=1   )
												a_		:	Union[str, Any]									=   torch.cat(
												    [loaded[i][f'layers.{layer_i}.feed_forward.w3.weight'] for i in range(lowerCAmelCase_   )]    ,		dim=0   )

								a_		:	Optional[Any]									=   inv_freq
								for k, v in state_dict.items():
												a_		:	int									=   filename
												param_count += v.numel()
								torch.save(lowerCAmelCase_    ,		os.path.join(lowerCAmelCase_    ,		lowerCAmelCase_   )   )

				a_		:	Dict									=   f'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'
				if model_size == "7B":
								# Unsharded
								a_		:	Union[str, Any]									=   {
								    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
								    '''model.norm.weight''': loaded['''norm.weight'''],
								    '''lm_head.weight''': loaded['''output.weight'''],
								}
				else:
								a_		:	Optional[Any]									=   {
								    '''model.norm.weight''': loaded[0]['''norm.weight'''],
								    '''model.embed_tokens.weight''': torch.cat(
								        [loaded[i]['''tok_embeddings.weight'''] for i in range(lowerCAmelCase_   )]    ,		dim=1   ),
								    '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(lowerCAmelCase_   )]    ,		dim=0   ),
								}

				for k, v in state_dict.items():
								a_		:	Union[str, Any]									=   filename
								param_count += v.numel()
				torch.save(lowerCAmelCase_    ,		os.path.join(lowerCAmelCase_    ,		lowerCAmelCase_   )   )

				# Write configs
				a_		:	Optional[int]									=   {'''total_size''': param_count * 2}
				write_json(lowerCAmelCase_    ,		os.path.join(lowerCAmelCase_    ,		'''pytorch_model.bin.index.json'''   )   )
				a_		:	Dict									=   params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
				a_		:	str									=   params['''multiple_of'''] if '''multiple_of''' in params else 2_56
				a_		:	Any									=   LlamaConfig(
				    hidden_size=lowerCAmelCase_    ,		intermediate_size=compute_intermediate_size(lowerCAmelCase_    ,		lowerCAmelCase_    ,		lowerCAmelCase_   )    ,		num_attention_heads=params['''n_heads''']    ,		num_hidden_layers=params['''n_layers''']    ,		rms_norm_eps=params['''norm_eps''']    ,		num_key_value_heads=lowerCAmelCase_    ,		)
				config.save_pretrained(lowerCAmelCase_   )

				# Make space so we can load the model properly now.
				del state_dict
				del loaded
				gc.collect()

				print('''Loading the checkpoint in a Llama model.'''   )
				a_		:	str									=   LlamaForCausalLM.from_pretrained(lowerCAmelCase_    ,		torch_dtype=torch.floataa    ,		low_cpu_mem_usage=lowerCAmelCase_   )
				# Avoid saving this as part of the config.
				del model.config._name_or_path

				print('''Saving in the Transformers format.'''   )
				model.save_pretrained(lowerCAmelCase_    ,		safe_serialization=lowerCAmelCase_   )
				shutil.rmtree(lowerCAmelCase_   )





def 		_UpperCAmelCase (				__A				: int    ,		__A				: int   ):
				# Initialize the tokenizer based on the `spm` model
				a_		:	Optional[Any]									=   LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
				print(f'Saving a {tokenizer_class.__name__} to {tokenizer_path}.'   )
				a_		:	List[str]									=   tokenizer_class(lowerCAmelCase_   )
				tokenizer.save_pretrained(lowerCAmelCase_   )





def 		_UpperCAmelCase (				):
				a_		:	List[Any]									=   argparse.ArgumentParser()
				parser.add_argument(
				    '''--input_dir'''    ,		help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''    ,		)
				parser.add_argument(
				    '''--model_size'''    ,		choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']    ,		)
				parser.add_argument(
				    '''--output_dir'''    ,		help='''Location to write HF model and tokenizer'''    ,		)
				parser.add_argument('''--safe_serialization'''    ,		type=lowerCAmelCase_    ,		help='''Whether or not to save using `safetensors`.'''   )
				a_		:	List[str]									=   parser.parse_args()
				if args.model_size != "tokenizer_only":
								write_model(
								    model_path=args.output_dir    ,		input_base_path=os.path.join(args.input_dir    ,		args.model_size   )    ,		model_size=args.model_size    ,		safe_serialization=args.safe_serialization    ,		)
				a_		:	Tuple									=   os.path.join(args.input_dir    ,		'''tokenizer.model'''   )
				write_tokenizer(args.output_dir    ,		lowerCAmelCase_   )


if __name__ == "__main__":
		main()








import json
import multiprocessing as mp
import re
from collections import defaultdict
from functools import partial
from typing import Dict, List, Optional, Set, Tuple, Type

from datasets import Dataset
from datasketch import MinHash, MinHashLSH
from dpu_utils.utils.iterators import ThreadedIterator
from tqdm import tqdm


__a	 =      re.compile("""[^A-Za-z_0-9]""")
# parameters used in DuplicationIndex
__a	 =      10
__a	 =      256
def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->Optional[MinHash]:
   if len(lowerCAmelCase_ ) < MIN_NUM_TOKENS:
      return None
   UpperCAmelCase      					=  MinHash(num_perm=lowerCAmelCase_ )
   for token in set(lowerCAmelCase_ ):
      min_hash.update(token.encode() )
   return min_hash
def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->Set[str]:
   return {t for t in NON_ALPHA.split(lowerCAmelCase_ ) if len(t.strip() ) > 0}






class 	__lowercase					:





      def __init__(		self    :       List[str]       ,							*,
          __lowerCamelCase    :       float = 0.85       ,							)     -> Any:






         """simple docstring"""







         UpperCAmelCase      					=  duplication_jaccard_threshold
         UpperCAmelCase      					=  NUM_PERM
         UpperCAmelCase      					=  MinHashLSH(threshold=self._duplication_jaccard_threshold       ,							num_perm=self._num_perm							)

         UpperCAmelCase      					=  defaultdict(__lowerCamelCase							)





      def 							_lowercase					(		self    :       Tuple       ,							__lowerCamelCase    :       Tuple       ,							__lowerCamelCase    :       MinHash							)     -> None:






         """simple docstring"""







         UpperCAmelCase      					=  self._index.query(__lowerCamelCase							)
         if code_key in self._index.keys:
            print(F"""Duplicate key {code_key}"""							)
            return

         self._index.insert(__lowerCamelCase       ,							__lowerCamelCase							)
         if len(__lowerCamelCase							) > 0:
            for base_duplicate in close_duplicates:
               if base_duplicate in self._duplicate_clusters:
                  self._duplicate_clusters[base_duplicate].add(__lowerCamelCase							)
                  break
            else:
               self._duplicate_clusters[close_duplicates[0]].add(__lowerCamelCase							)





      def 							_lowercase					(		self    :       Union[str, Any]							)     -> List[List[Dict]]:






         """simple docstring"""







         UpperCAmelCase      					=  []
         for base, duplicates in self._duplicate_clusters.items():
            UpperCAmelCase      					=  [base] + list(__lowerCamelCase							)
            # reformat the cluster to be a list of dict
            UpperCAmelCase      					=  [{"""base_index""": el[0], """repo_name""": el[1], """path""": el[2]} for el in cluster]
            duplicate_clusters.append(__lowerCamelCase							)
         return duplicate_clusters





      def 							_lowercase					(		self    :       Tuple       ,							__lowerCamelCase    :       Optional[int]							)     -> None:






         """simple docstring"""







         UpperCAmelCase      					=  self.get_duplicate_clusters()
         with open(__lowerCamelCase       ,							"""w"""							) as f:
            json.dump(__lowerCamelCase       ,							__lowerCamelCase							)

def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->Tuple:
   UpperCAmelCase					,	UpperCAmelCase      					=  element
   UpperCAmelCase      					=  get_min_hash([t for t in NON_ALPHA.split(data["""content"""] ) if len(t.strip() ) > 0] )
   if min_hash is not None:
      return (index, data["repo_name"], data["path"]), min_hash
def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->int:
   with mp.Pool() as pool:
      for data in pool.imap_unordered(
          _compute_min_hash	,    ThreadedIterator(lowerCAmelCase_	,    max_queue_size=1_0_0_0_0 )	,    chunksize=1_0_0	,    ):
         if data is not None:
            yield data
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_ )    ->Any:
   UpperCAmelCase      					=  DuplicationIndex(duplication_jaccard_threshold=lowerCAmelCase_ )

   for filename, min_hash in tqdm(ThreadedIterator(minhash_iter(enumerate(lowerCAmelCase_ ) )	,    max_queue_size=1_0_0 ) ):
      di.add(lowerCAmelCase_	,    lowerCAmelCase_ )

   # Returns a List[Cluster] where Cluster is List[str] with the filenames.
   return di.get_duplicate_clusters()
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_ )    ->float:
   UpperCAmelCase      					=  get_tokens(lowerCAmelCase_ )
   UpperCAmelCase      					=  get_tokens(lowerCAmelCase_ )
   return len(tokensa & tokensa ) / len(tokensa | tokensa )


__a	 =      None
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_ )    ->Dict:
   UpperCAmelCase      					=  []
   for elementa in cluster:
      UpperCAmelCase      					=  _shared_dataset[elementa["""base_index"""]]["""content"""]
      for elementa in extremes:
         UpperCAmelCase      					=  _shared_dataset[elementa["""base_index"""]]["""content"""]
         if jaccard_similarity(lowerCAmelCase_	,    lowerCAmelCase_ ) >= jaccard_threshold:
            elementa["copies"] += 1
            break
      else:
         UpperCAmelCase      					=  1
         extremes.append(lowerCAmelCase_ )
   return extremes
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_	,    lowerCAmelCase_ )    ->Optional[int]:
   global _shared_dataset
   UpperCAmelCase      					=  dataset
   UpperCAmelCase      					=  []
   UpperCAmelCase      					=  partial(_find_cluster_extremes_shared	,    jaccard_threshold=lowerCAmelCase_ )
   with mp.Pool() as pool:
      for extremes in tqdm(
          pool.imap_unordered(
              lowerCAmelCase_	,    lowerCAmelCase_	,    )	,    total=len(lowerCAmelCase_ )	,    ):
         extremes_list.append(lowerCAmelCase_ )
   return extremes_list







def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_ = 0.85 )    ->Tuple[Type[Dataset], List[List[Dict]]]:
   UpperCAmelCase      					=  make_duplicate_clusters(lowerCAmelCase_	,    lowerCAmelCase_ )
   UpperCAmelCase      					=  {x["""base_index"""] for cluster in duplicate_clusters for x in cluster}
   UpperCAmelCase      					=  {}
   UpperCAmelCase      					=  find_extremes(lowerCAmelCase_	,    lowerCAmelCase_	,    lowerCAmelCase_ )
   for extremes in extremes_clusters:
      for element in extremes:
         UpperCAmelCase      					=  element
   UpperCAmelCase      					=  duplicate_indices - set(extreme_dict.keys() )
   UpperCAmelCase      					=  dataset.filter(lambda lowerCAmelCase_	,    lowerCAmelCase_ : idx not in remove_indices	,    with_indices=lowerCAmelCase_ )

   # update duplicate_clusters
   for cluster in duplicate_clusters:
      for element in cluster:
         UpperCAmelCase      					=  element["""base_index"""] in extreme_dict
         if element["is_extreme"]:
            UpperCAmelCase      					=  extreme_dict[element["""base_index"""]]["""copies"""]

   print(F"""Original dataset size: {len(lowerCAmelCase_ )}""" )
   print(F"""Number of duplicate clusters: {len(lowerCAmelCase_ )}""" )
   print(F"""Files in duplicate cluster: {len(lowerCAmelCase_ )}""" )
   print(F"""Unique files in duplicate cluster: {len(lowerCAmelCase_ )}""" )
   print(F"""Filtered dataset size: {len(lowerCAmelCase_ )}""" )

   return ds_filter, duplicate_clusters



