

# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig


def 					_UpperCAmelCase				(						a__  ,	a__  ,	a__=0):

			'''simple docstring'''



			if name is None:
						a_						:						Dict 							= None
			else:
						a_						:						Optional[Any] 							= """.""" * max(0  ,	spaces - 2) + """# {:""" + str(5_0 - spaces) + """s}"""
						a_						:						Tuple 							= fmt.format(a__)

			# Print and recurse (if needed).
			if isinstance(a__  ,	a__):
						if msg is not None:
									print(a__)
						for k in val.keys():
									recursive_print(a__  ,	val[k]  ,	spaces + 2)
			elif isinstance(a__  ,	torch.Tensor):
						print(a__  ,	""":"""  ,	val.size())
			else:
						print(a__  ,	""":"""  ,	a__)


def 					_UpperCAmelCase				(						a__  ,	a__  ,	a__  ,	a__  ,	a__):

			'''simple docstring'''



			a_						:						List[Any] 							= param.size()
			if checkpoint_version == 1.0:
						# version 1.0 stores [num_heads * hidden_size * num_splits, :]
						a_						:						Any 							= (num_heads, hidden_size, num_splits) + input_shape[1:]
						a_						:						List[str] 							= param.view(*a__)
						a_						:						Optional[Any] 							= param.transpose(0  ,	2)
						a_						:						Tuple 							= param.transpose(1  ,	2).contiguous()
			elif checkpoint_version >= 2.0:
						# other versions store [num_heads * num_splits * hidden_size, :]
						a_						:						Optional[Any] 							= (num_heads, num_splits, hidden_size) + input_shape[1:]
						a_						:						Optional[int] 							= param.view(*a__)
						a_						:						Any 							= param.transpose(0  ,	1).contiguous()
			a_						:						str 							= param.view(*a__)
			return param


def 					_UpperCAmelCase				(						a__  ,	a__  ,	a__):

			'''simple docstring'''



			a_						:						List[Any] 							= {}

			# old versions did not store training args
			a_						:						List[str] 							= input_state_dict.get("""args"""  ,	a__)
			if ds_args is not None:
						# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
						# from pprint import pprint
						# pprint(vars(ds_args))

						a_						:						Optional[Any] 							= ds_args.padded_vocab_size
						a_						:						Any 							= ds_args.max_position_embeddings
						a_						:						List[str] 							= ds_args.hidden_size
						a_						:						Tuple 							= ds_args.num_layers
						a_						:						Optional[Any] 							= ds_args.num_attention_heads
						a_						:						Optional[Any] 							= ds_args.ffn_hidden_size
						# pprint(config)

			# The number of heads.
			a_						:						Union[str, Any] 							= config.n_head
			# The hidden_size per head.
			a_						:						Dict 							= config.n_embd // config.n_head
			# Megatron-LM checkpoint version
			if "checkpoint_version" in input_state_dict.keys():
						a_						:						Union[str, Any] 							= input_state_dict["""checkpoint_version"""]
			else:
						a_						:						List[Any] 							= 0.0

			# The model.
			a_						:						Any 							= input_state_dict["""model"""]
			# The language model.
			a_						:						Any 							= model["""language_model"""]
			# The embeddings.
			a_						:						Optional[Any] 							= lm["""embedding"""]

			# The word embeddings.
			a_						:						List[Any] 							= embeddings["""word_embeddings"""]["""weight"""]
			# Truncate the embedding table to vocab_size rows.
			a_						:						Optional[Any] 							= word_embeddings[: config.vocab_size, :]
			a_						:						Union[str, Any] 							= word_embeddings

			# The position embeddings.
			a_						:						str 							= embeddings["""position_embeddings"""]["""weight"""]
			# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
			a_						:						Union[str, Any] 							= pos_embeddings.size(0)
			if n_positions != config.n_positions:
						raise ValueError(
						    f'''pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match''')
			# Store the position embeddings.
			a_						:						Optional[Any] 							= pos_embeddings

			# The transformer.
			a_						:						Dict 							= lm["""transformer"""] if """transformer""" in lm.keys() else lm["""encoder"""]

			# The regex to extract layer names.
			a_						:						Tuple 							= re.compile(r"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)""")

			# The simple map of names for "automated" rules.
			a_						:						List[str] 							= {
			    """attention.dense""": """.attn.c_proj.""",
			    """self_attention.dense""": """.attn.c_proj.""",
			    """mlp.dense_h_to_4h""": """.mlp.c_fc.""",
			    """mlp.dense_4h_to_h""": """.mlp.c_proj.""",
			}

			# Extract the layers.
			for key, val in transformer.items():
						# Match the name.
						a_						:						List[str] 							= layer_re.match(a__)

						# Stop if that's not a layer
						if m is None:
									break

						# The index of the layer.
						a_						:						Union[str, Any] 							= int(m.group(1))
						# The name of the operation.
						a_						:						Optional[Any] 							= m.group(2)
						# Is it a weight or a bias?
						a_						:						Union[str, Any] 							= m.group(3)

						# The name of the layer.
						a_						:						Dict 							= f'''transformer.h.{layer_idx}'''

						# For layernorm(s), simply store the layer norm.
						if op_name.endswith("""layernorm"""):
									a_						:						str 							= """ln_1""" if op_name.startswith("""input""") else """ln_2"""
									a_						:						int 							= val

						# Transpose the QKV matrix.
						elif (
						    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
						) and weight_or_bias == "weight":
									# Insert a tensor of 1x1xDxD bias.
									a_						:						Tuple 							= torch.tril(torch.ones((n_positions, n_positions)  ,	dtype=torch.floataa)).view(
									    1  ,	1  ,	a__  ,	a__)
									a_						:						Union[str, Any] 							= causal_mask

									# Insert a "dummy" tensor for masked_bias.
									a_						:						int 							= torch.tensor(-1e4  ,	dtype=torch.floataa)
									a_						:						List[str] 							= masked_bias

									a_						:						Optional[Any] 							= fix_query_key_value_ordering(a__  ,	a__  ,	3  ,	a__  ,	a__)
									# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
									a_						:						Tuple 							= out_val.transpose(0  ,	1).contiguous()
									# Store.
									a_						:						int 							= out_val

						# Transpose the bias.
						elif (
						    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
						) and weight_or_bias == "bias":
									a_						:						Tuple 							= fix_query_key_value_ordering(a__  ,	a__  ,	3  ,	a__  ,	a__)
									# Store. No change of shape.
									a_						:						Any 							= out_val

						# Transpose the weights.
						elif weight_or_bias == "weight":
									a_						:						Union[str, Any] 							= megatron_to_transformers[op_name]
									a_						:						Tuple 							= val.transpose(0  ,	1)

						# Copy the bias.
						elif weight_or_bias == "bias":
									a_						:						List[str] 							= megatron_to_transformers[op_name]
									a_						:						str 							= val

    # DEBUG.
			assert config.n_layer == layer_idx + 1

			# The final layernorm.
			a_						:						Any 							= transformer["""final_layernorm.weight"""]
			a_						:						List[str] 							= transformer["""final_layernorm.bias"""]

			# For LM head, transformers' wants the matrix to weight embeddings.
			a_						:						str 							= word_embeddings

			# It should be done!
			return output_state_dict


def 					_UpperCAmelCase				(						):

			'''simple docstring'''



			a_						:						str 							= argparse.ArgumentParser()
			parser.add_argument("""--print-checkpoint-structure"""  ,	action="""store_true""")
			parser.add_argument(
			    """path_to_checkpoint"""  ,	type=a__  ,	help="""Path to the checkpoint file (.zip archive or direct .pt file)"""  ,	)
			parser.add_argument(
			    """--config_file"""  ,	default=""""""  ,	type=a__  ,	help="""An optional config json file describing the pre-trained model."""  ,	)
			a_						:						Any 							= parser.parse_args()

			# Extract the basename.
			a_						:						Optional[Any] 							= os.path.dirname(args.path_to_checkpoint)

			# Load the model.
			# the .zip is very optional, let's keep it for backward compatibility
			print(f'''Extracting PyTorch state dictionary from {args.path_to_checkpoint}''')
			if args.path_to_checkpoint.endswith(""".zip"""):
						with zipfile.ZipFile(args.path_to_checkpoint  ,	"""r""") as checkpoint:
									with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt""") as pytorch_dict:
												a_						:						Optional[Any] 							= torch.load(a__  ,	map_location="""cpu""")
			else:
						a_						:						List[str] 							= torch.load(args.path_to_checkpoint  ,	map_location="""cpu""")

			a_						:						Any 							= input_state_dict.get("""args"""  ,	a__)

			# Read the config, or default to the model released by NVIDIA.
			if args.config_file == "":
						if ds_args is not None:
									if ds_args.bias_gelu_fusion:
												a_						:						Dict 							= """gelu_fast"""
									elif ds_args.openai_gelu:
												a_						:						Any 							= """gelu_new"""
									else:
												a_						:						Optional[Any] 							= """gelu"""
						else:
									# in the very early days this used to be "gelu_new"
									a_						:						Dict 							= """gelu_new"""

						# Spell out all parameters in case the defaults change.
						a_						:						List[str] 							= GPTaConfig(
						    vocab_size=5_0_2_5_7  ,	n_positions=1_0_2_4  ,	n_embd=1_0_2_4  ,	n_layer=2_4  ,	n_head=1_6  ,	n_inner=4_0_9_6  ,	activation_function=a__  ,	resid_pdrop=0.1  ,	embd_pdrop=0.1  ,	attn_pdrop=0.1  ,	layer_norm_epsilon=1e-5  ,	initializer_range=0.02  ,	summary_type="""cls_index"""  ,	summary_use_proj=a__  ,	summary_activation=a__  ,	summary_proj_to_labels=a__  ,	summary_first_dropout=0.1  ,	scale_attn_weights=a__  ,	use_cache=a__  ,	bos_token_id=5_0_2_5_6  ,	eos_token_id=5_0_2_5_6  ,	)
			else:
						a_						:						Tuple 							= GPTaConfig.from_json_file(args.config_file)

			a_						:						Union[str, Any] 							= ["""GPT2LMHeadModel"""]

			# Convert.
			print("""Converting""")
			a_						:						int 							= convert_megatron_checkpoint(a__  ,	a__  ,	a__)

			# Print the structure of converted state dict.
			if args.print_checkpoint_structure:
						recursive_print(a__  ,	a__)

			# Add tokenizer class info to config
			# see https://github.com/huggingface/transformers/issues/13906)
			if ds_args is not None:
						a_						:						Optional[Any] 							= ds_args.tokenizer_type
						if tokenizer_type == "GPT2BPETokenizer":
									a_						:						Any 							= """gpt2"""
						elif tokenizer_type == "PretrainedFromHF":
									a_						:						Optional[Any] 							= ds_args.tokenizer_name_or_path
						else:
									raise ValueError(f'''Unrecognized tokenizer_type {tokenizer_type}''')
			else:
						a_						:						str 							= """gpt2"""

			a_						:						Any 							= AutoTokenizer.from_pretrained(a__)
			a_						:						Optional[int] 							= type(a__).__name__
			a_						:						List[Any] 							= tokenizer_class

			# Store the config to file.
			print("""Saving config""")
			config.save_pretrained(a__)

			# Save tokenizer based on args
			print(f'''Adding {tokenizer_class} tokenizer files''')
			tokenizer.save_pretrained(a__)

			# Store the state_dict to file.
			a_						:						int 							= os.path.join(a__  ,	"""pytorch_model.bin""")
			print(f'''Saving checkpoint to "{output_checkpoint_file}"''')
			torch.save(a__  ,	a__)


####################################################################################################

if __name__ == "__main__":
	main()

####################################################################################################





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


__snake_case				:      int   		=			{
    """configuration_llama""": ["""LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP""", """LlamaConfig"""],
}

try:
	if not is_sentencepiece_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__snake_case				:      int   		=			["""LlamaTokenizer"""]

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__snake_case				:      int   		=			["""LlamaTokenizerFast"""]

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__snake_case				:      Union[str, Any]   		=			[
	    """LlamaForCausalLM""",
	    """LlamaModel""",
	    """LlamaPreTrainedModel""",
	    """LlamaForSequenceClassification""",
	]


if TYPE_CHECKING:
	from .configuration_llama import LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP, LlamaConfig

	try:
		if not is_sentencepiece_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_llama import LlamaTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_llama_fast import LlamaTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_llama import LlamaForCausalLM, LlamaForSequenceClassification, LlamaModel, LlamaPreTrainedModel


else:
	import sys

	__snake_case				:      Dict   		=			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


