

import os
import time
from dataclasses import dataclass, field
from enum import Enum
from typing import Dict, List, Optional, Union

import torch
from filelock import FileLock
from torch.utils.data import Dataset

from ...models.auto.modeling_auto import MODEL_FOR_QUESTION_ANSWERING_MAPPING
from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging
from ..processors.squad import SquadFeatures, SquadVaProcessor, SquadVaProcessor, squad_convert_examples_to_features


A_			:					Union[str, Any]    			=     logging.get_logger(__name__)

A_			:					int    			=     list(MODEL_FOR_QUESTION_ANSWERING_MAPPING.keys())
A_			:					List[Any]    			=     tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)



@dataclass
class        _a		:


					'''simple docstring'''






					UpperCAmelCase__:  str    				=			field(
					    default=_SCREAMING_SNAKE_CASE						,  metadata={'''help''': '''Model type selected in the list: ''' + ''', '''.join(_SCREAMING_SNAKE_CASE							)}							)
					UpperCAmelCase__:  str    				=			field(
					    default=_SCREAMING_SNAKE_CASE						,  metadata={'''help''': '''The input data dir. Should contain the .json files for the SQuAD task.'''}							)
					UpperCAmelCase__:  int    				=			field(
					    default=1_28						,  metadata={
					        '''help''': (
					            '''The maximum total input sequence length after tokenization. Sequences longer '''
					            '''than this will be truncated, sequences shorter will be padded.'''
					        )
					    }						,  )
					UpperCAmelCase__:  int    				=			field(
					    default=1_28						,  metadata={'''help''': '''When splitting up a long document into chunks, how much stride to take between chunks.'''}						,  )
					UpperCAmelCase__:  int    				=			field(
					    default=64						,  metadata={
					        '''help''': (
					            '''The maximum number of tokens for the question. Questions longer than this will '''
					            '''be truncated to this length.'''
					        )
					    }						,  )
					UpperCAmelCase__:  int    				=			field(
					    default=30						,  metadata={
					        '''help''': (
					            '''The maximum length of an answer that can be generated. This is needed because the start '''
					            '''and end predictions are not conditioned on one another.'''
					        )
					    }						,  )
					UpperCAmelCase__:  bool    				=			field(
					    default=_SCREAMING_SNAKE_CASE						,  metadata={'''help''': '''Overwrite the cached training and evaluation sets'''}							)
					UpperCAmelCase__:  bool    				=			field(
					    default=_SCREAMING_SNAKE_CASE						,  metadata={'''help''': '''If true, the SQuAD examples contain some that do not have an answer.'''}							)
					UpperCAmelCase__:  float    				=			field(
					    default=0.0						,  metadata={'''help''': '''If null_score - best_non_null is greater than the threshold predict null.'''}							)
					UpperCAmelCase__:  int    				=			field(
					    default=20						,  metadata={'''help''': '''If null_score - best_non_null is greater than the threshold predict null.'''}							)
					UpperCAmelCase__:  int    				=			field(
					    default=0						,  metadata={
					        '''help''': (
					            '''language id of input for language-specific xlm models (see'''
					            ''' tokenization_xlm.PRETRAINED_INIT_CONFIGURATION)'''
					        )
					    }						,  )
					UpperCAmelCase__:  int    				=			field(default=1						,  metadata={'''help''': '''multiple threads for converting example to features'''}							)



class        _a		(_SCREAMING_SNAKE_CASE							):


					'''simple docstring'''






					UpperCAmelCase__:  str    				=			'''train'''
					UpperCAmelCase__:  Union[str, Any]    				=			'''dev'''



class        _a		(_SCREAMING_SNAKE_CASE							):


					'''simple docstring'''






					UpperCAmelCase__:  SquadDataTrainingArguments
					UpperCAmelCase__:  List[SquadFeatures]
					UpperCAmelCase__:  Split
					UpperCAmelCase__:  bool






					def __init__(					self       ,			A__       ,			A__       ,			A__ = None       ,			A__ = Split.train       ,			A__ = False       ,			A__ = None       ,			A__ = "pt"       ,			):
						A__				:       List[Any]	  =						args
						A__				:       List[str]	  =						is_language_sensitive
						A__				:       Optional[int]	  =						SquadVaProcessor() if args.version_2_with_negative else SquadVaProcessor()
						if isinstance(_lowercase       ,			_lowercase     ):
							try:
								A__				:       List[str]	  =						Split[mode]
							except KeyError:
								raise KeyError("""mode is not a valid split name"""     )
						A__				:       Tuple	  =						mode
						# Load data features from cache or dataset file
						A__				:       Union[str, Any]	  =						"""v2""" if args.version_2_with_negative else """v1"""
						A__				:       List[str]	  =						os.path.join(
						    cache_dir if cache_dir is not None else args.data_dir       ,			F"""cached_{mode.value}_{tokenizer.__class__.__name__}_{args.max_seq_length}_{version_tag}"""       ,			)

						# Make sure only the first process in distributed training processes the dataset,
						# and the others will use the cache.
						A__				:       Any	  =						cached_features_file + """.lock"""
						with FileLock(_lowercase     ):
							if os.path.exists(_lowercase     ) and not args.overwrite_cache:
								A__				:       Union[str, Any]	  =						time.time()
								A__				:       int	  =						torch.load(_lowercase     )

								# Legacy cache files have only features, while new cache files
								# will have dataset and examples also.
								A__				:       Optional[int]	  =						self.old_features["""features"""]
								A__				:       Optional[Any]	  =						self.old_features.get("""dataset"""       ,			_lowercase     )
								A__				:       List[Any]	  =						self.old_features.get("""examples"""       ,			_lowercase     )
								logger.info(
								    F"""Loading features from cached file {cached_features_file} [took %.3f s]"""       ,			time.time() - start     )

								if self.dataset is None or self.examples is None:
									logger.warning(
									    F"""Deleting cached file {cached_features_file} will allow dataset and examples to be cached in"""
									    """ future run"""     )
							else:
								if mode == Split.dev:
									A__				:       Optional[Any]	  =						self.processor.get_dev_examples(args.data_dir     )
								else:
									A__				:       int	  =						self.processor.get_train_examples(args.data_dir     )

								A__  ,  A__				:       int	  =						squad_convert_examples_to_features(
								    examples=self.examples       ,			tokenizer=_lowercase       ,			max_seq_length=args.max_seq_length       ,			doc_stride=args.doc_stride       ,			max_query_length=args.max_query_length       ,			is_training=mode == Split.train       ,			threads=args.threads       ,			return_dataset=_lowercase       ,			)

								A__				:       Union[str, Any]	  =						time.time()
								torch.save(
								    {"""features""": self.features, """dataset""": self.dataset, """examples""": self.examples}       ,			_lowercase       ,			)
								# ^ This seems to take a lot of time so I want to investigate why and how we can improve.
								logger.info(
								    F"""Saving features into cached file {cached_features_file} [took {time.time() - start:.3f} s]"""     )






					def __len__(					self     ):
						return len(self.features     )






					def __getitem__(					self       ,			A__     ):
						A__				:       List[Any]	  =						self.features[i]

						A__				:       str	  =						torch.tensor(feature.input_ids       ,			dtype=torch.long     )
						A__				:       Optional[int]	  =						torch.tensor(feature.attention_mask       ,			dtype=torch.long     )
						A__				:       Optional[Any]	  =						torch.tensor(feature.token_type_ids       ,			dtype=torch.long     )
						A__				:       Any	  =						torch.tensor(feature.cls_index       ,			dtype=torch.long     )
						A__				:       str	  =						torch.tensor(feature.p_mask       ,			dtype=torch.float     )
						A__				:       Optional[int]	  =						torch.tensor(feature.is_impossible       ,			dtype=torch.float     )

						A__				:       Tuple	  =						{
						    """input_ids""": input_ids,
						    """attention_mask""": attention_mask,
						    """token_type_ids""": token_type_ids,
						}

						if self.args.model_type in ["xlm", "roberta", "distilbert", "camembert"]:
							del inputs["token_type_ids"]

						if self.args.model_type in ["xlnet", "xlm"]:
							inputs.update({"""cls_index""": cls_index, """p_mask""": p_mask}     )
							if self.args.version_2_with_negative:
								inputs.update({"""is_impossible""": is_impossible}     )
							if self.is_language_sensitive:
								inputs.update({"""langs""": (torch.ones(input_ids.shape       ,			dtype=torch.intaa     ) * self.args.lang_id)}     )

						if self.mode == Split.train:
							A__				:       int	  =						torch.tensor(feature.start_position       ,			dtype=torch.long     )
							A__				:       Union[str, Any]	  =						torch.tensor(feature.end_position       ,			dtype=torch.long     )
							inputs.update({"""start_positions""": start_positions, """end_positions""": end_positions}     )

						return inputs





'''simple docstring'''



import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
						from transformers import LlamaTokenizerFast
except ImportError as e:
						warnings.warn(e)
						warnings.warn(
						    """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
						)
						_lowercase          =						None
_lowercase          =						{
    """7B""": 11008,
    """13B""": 13824,
    """30B""": 17920,
    """65B""": 22016,
    """70B""": 28672,
}
_lowercase          =						{
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}


def A (__lowerCamelCase :int					,      __lowerCamelCase :Optional[Any]=1					,      __lowerCamelCase :List[Any]=256						):
						return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3						)						) + multiple_of - 1) // multiple_of)


def A (__lowerCamelCase :Any						):
						with open(__lowerCamelCase					,      """r"""						) as f:
												return json.load(__lowerCamelCase						)


def A (__lowerCamelCase :List[Any]					,      __lowerCamelCase :int						):
						with open(__lowerCamelCase					,      """w"""						) as f:
												json.dump(__lowerCamelCase					,      __lowerCamelCase						)


def A (__lowerCamelCase :Optional[Any]					,      __lowerCamelCase :Tuple					,      __lowerCamelCase :Optional[Any]					,      __lowerCamelCase :Tuple=True						):
						os.makedirs(__lowerCamelCase					,      exist_ok=__lowerCamelCase						)
						_lowerCAmelCase    		=					os.path.join(__lowerCamelCase					,      """tmp"""						)
						os.makedirs(__lowerCamelCase					,      exist_ok=__lowerCamelCase						)

						_lowerCAmelCase    		=					read_json(os.path.join(__lowerCamelCase					,      """params.json"""						)						)
						_lowerCAmelCase    		=					NUM_SHARDS[model_size]
						_lowerCAmelCase    		=					params["""n_layers"""]
						_lowerCAmelCase    		=					params["""n_heads"""]
						_lowerCAmelCase    		=					n_heads // num_shards
						_lowerCAmelCase    		=					params["""dim"""]
						_lowerCAmelCase    		=					dim // n_heads
						_lowerCAmelCase    		=					10_000.0
						_lowerCAmelCase    		=					1.0 / (base ** (torch.arange(0					,      __lowerCamelCase					,      2						).float() / dims_per_head))

						if "n_kv_heads" in params:
												_lowerCAmelCase    		=					params["""n_kv_heads"""]  # for GQA / MQA
												_lowerCAmelCase    		=					n_heads_per_shard // num_key_value_heads
												_lowerCAmelCase    		=					dim // num_key_value_heads
						else:  # compatibility with other checkpoints
												_lowerCAmelCase    		=					n_heads
												_lowerCAmelCase    		=					n_heads_per_shard
												_lowerCAmelCase    		=					dim

						# permute for sliced rotary
						def permute(__lowerCamelCase :Optional[int]					,      __lowerCamelCase :str=n_heads					,      __lowerCamelCase :str=dim					,      __lowerCamelCase :List[Any]=dim						):
												return w.view(__lowerCamelCase					,      dima // n_heads // 2					,      2					,      __lowerCamelCase						).transpose(1					,      2						).reshape(__lowerCamelCase					,      __lowerCamelCase						)

						print(f'Fetching all parameters from the checkpoint at {input_base_path}.'						)
						# Load weights
						if model_size == "7B":
												# Not sharded
												# (The sharded implementation would also work, but this is simpler.)
												_lowerCAmelCase    		=					torch.load(os.path.join(__lowerCamelCase					,      """consolidated.00.pth"""						)					,      map_location="""cpu"""						)
						else:
												# Sharded
												_lowerCAmelCase    		=					[
												    torch.load(os.path.join(__lowerCamelCase					,      f'consolidated.{i:02d}.pth'						)					,      map_location="""cpu"""						)
												    for i in range(__lowerCamelCase						)
												]
						_lowerCAmelCase    		=					0
						_lowerCAmelCase    		=					{"""weight_map""": {}}
						for layer_i in range(__lowerCamelCase						):
												_lowerCAmelCase    		=					f'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'
												if model_size == "7B":
																		# Unsharded
																		_lowerCAmelCase    		=					{
																		    f'model.layers.{layer_i}.self_attn.q_proj.weight': permute(
																		        loaded[f'layers.{layer_i}.attention.wq.weight']						),
																		    f'model.layers.{layer_i}.self_attn.k_proj.weight': permute(
																		        loaded[f'layers.{layer_i}.attention.wk.weight']						),
																		    f'model.layers.{layer_i}.self_attn.v_proj.weight': loaded[f'layers.{layer_i}.attention.wv.weight'],
																		    f'model.layers.{layer_i}.self_attn.o_proj.weight': loaded[f'layers.{layer_i}.attention.wo.weight'],
																		    f'model.layers.{layer_i}.mlp.gate_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w1.weight'],
																		    f'model.layers.{layer_i}.mlp.down_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w2.weight'],
																		    f'model.layers.{layer_i}.mlp.up_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w3.weight'],
																		    f'model.layers.{layer_i}.input_layernorm.weight': loaded[f'layers.{layer_i}.attention_norm.weight'],
																		    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[f'layers.{layer_i}.ffn_norm.weight'],
																		}
												else:
																		# Sharded
																		# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
																		# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
																		# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

																		_lowerCAmelCase    		=					{
																		    f'model.layers.{layer_i}.input_layernorm.weight': loaded[0][
																		        f'layers.{layer_i}.attention_norm.weight'
																		    ].clone(),
																		    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[0][
																		        f'layers.{layer_i}.ffn_norm.weight'
																		    ].clone(),
																		}
																		_lowerCAmelCase    		=					permute(
																		    torch.cat(
																		        [
																		            loaded[i][f'layers.{layer_i}.attention.wq.weight'].view(__lowerCamelCase					,      __lowerCamelCase					,      __lowerCamelCase						)
																		            for i in range(__lowerCamelCase						)
																		        ]					,      dim=0					,      ).reshape(__lowerCamelCase					,      __lowerCamelCase						)						)
																		_lowerCAmelCase    		=					permute(
																		    torch.cat(
																		        [
																		            loaded[i][f'layers.{layer_i}.attention.wk.weight'].view(
																		                __lowerCamelCase					,      __lowerCamelCase					,      __lowerCamelCase						)
																		            for i in range(__lowerCamelCase						)
																		        ]					,      dim=0					,      ).reshape(__lowerCamelCase					,      __lowerCamelCase						)					,      __lowerCamelCase					,      __lowerCamelCase					,      __lowerCamelCase					,      )
																		_lowerCAmelCase    		=					torch.cat(
																		    [
																		        loaded[i][f'layers.{layer_i}.attention.wv.weight'].view(
																		            __lowerCamelCase					,      __lowerCamelCase					,      __lowerCamelCase						)
																		        for i in range(__lowerCamelCase						)
																		    ]					,      dim=0					,      ).reshape(__lowerCamelCase					,      __lowerCamelCase						)

																		_lowerCAmelCase    		=					torch.cat(
																		    [loaded[i][f'layers.{layer_i}.attention.wo.weight'] for i in range(__lowerCamelCase						)]					,      dim=1						)
																		_lowerCAmelCase    		=					torch.cat(
																		    [loaded[i][f'layers.{layer_i}.feed_forward.w1.weight'] for i in range(__lowerCamelCase						)]					,      dim=0						)
																		_lowerCAmelCase    		=					torch.cat(
																		    [loaded[i][f'layers.{layer_i}.feed_forward.w2.weight'] for i in range(__lowerCamelCase						)]					,      dim=1						)
																		_lowerCAmelCase    		=					torch.cat(
																		    [loaded[i][f'layers.{layer_i}.feed_forward.w3.weight'] for i in range(__lowerCamelCase						)]					,      dim=0						)

												_lowerCAmelCase    		=					inv_freq
												for k, v in state_dict.items():
																		_lowerCAmelCase    		=					filename
																		param_count += v.numel()
												torch.save(__lowerCamelCase					,      os.path.join(__lowerCamelCase					,      __lowerCamelCase						)						)

						_lowerCAmelCase    		=					f'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'
						if model_size == "7B":
												# Unsharded
												_lowerCAmelCase    		=					{
												    """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
												    """model.norm.weight""": loaded["""norm.weight"""],
												    """lm_head.weight""": loaded["""output.weight"""],
												}
						else:
												_lowerCAmelCase    		=					{
												    """model.norm.weight""": loaded[0]["""norm.weight"""],
												    """model.embed_tokens.weight""": torch.cat(
												        [loaded[i]["""tok_embeddings.weight"""] for i in range(__lowerCamelCase						)]					,      dim=1						),
												    """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(__lowerCamelCase						)]					,      dim=0						),
												}

						for k, v in state_dict.items():
												_lowerCAmelCase    		=					filename
												param_count += v.numel()
						torch.save(__lowerCamelCase					,      os.path.join(__lowerCamelCase					,      __lowerCamelCase						)						)

						# Write configs
						_lowerCAmelCase    		=					{"""total_size""": param_count * 2}
						write_json(__lowerCamelCase					,      os.path.join(__lowerCamelCase					,      """pytorch_model.bin.index.json"""						)						)
						_lowerCAmelCase    		=					params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
						_lowerCAmelCase    		=					params["""multiple_of"""] if """multiple_of""" in params else 256
						_lowerCAmelCase    		=					LlamaConfig(
						    hidden_size=__lowerCamelCase					,      intermediate_size=compute_intermediate_size(__lowerCamelCase					,      __lowerCamelCase					,      __lowerCamelCase						)					,      num_attention_heads=params["""n_heads"""]					,      num_hidden_layers=params["""n_layers"""]					,      rms_norm_eps=params["""norm_eps"""]					,      num_key_value_heads=__lowerCamelCase					,      )
						config.save_pretrained(__lowerCamelCase						)

						# Make space so we can load the model properly now.
						del state_dict
						del loaded
						gc.collect()

						print("""Loading the checkpoint in a Llama model."""						)
						_lowerCAmelCase    		=					LlamaForCausalLM.from_pretrained(__lowerCamelCase					,      torch_dtype=torch.floataa					,      low_cpu_mem_usage=__lowerCamelCase						)
						# Avoid saving this as part of the config.
						del model.config._name_or_path

						print("""Saving in the Transformers format."""						)
						model.save_pretrained(__lowerCamelCase					,      safe_serialization=__lowerCamelCase						)
						shutil.rmtree(__lowerCamelCase						)


def A (__lowerCamelCase :Union[str, Any]					,      __lowerCamelCase :Union[str, Any]						):
						# Initialize the tokenizer based on the `spm` model
						_lowerCAmelCase    		=					LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
						print(f'Saving a {tokenizer_class.__name__} to {tokenizer_path}.'						)
						_lowerCAmelCase    		=					tokenizer_class(__lowerCamelCase						)
						tokenizer.save_pretrained(__lowerCamelCase						)


def A ():
						_lowerCAmelCase    		=					argparse.ArgumentParser()
						parser.add_argument(
						    """--input_dir"""					,      help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""					,      )
						parser.add_argument(
						    """--model_size"""					,      choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]					,      )
						parser.add_argument(
						    """--output_dir"""					,      help="""Location to write HF model and tokenizer"""					,      )
						parser.add_argument("""--safe_serialization"""					,      type=__lowerCamelCase					,      help="""Whether or not to save using `safetensors`."""						)
						_lowerCAmelCase    		=					parser.parse_args()
						if args.model_size != "tokenizer_only":
												write_model(
												    model_path=args.output_dir					,      input_base_path=os.path.join(args.input_dir					,      args.model_size						)					,      model_size=args.model_size					,      safe_serialization=args.safe_serialization					,      )
						_lowerCAmelCase    		=					os.path.join(args.input_dir					,      """tokenizer.model"""						)
						write_tokenizer(args.output_dir					,      __lowerCamelCase						)


if __name__ == "__main__":
						main()
