
# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig



def   __lowerCAmelCase    (				UpperCAmelCase__					:   Union[str, Any]      ,					UpperCAmelCase__					:   Any      ,					UpperCAmelCase__					:   Any=0							)	->     Any:
							# Format the message.
							if name is None:
														lowerCamelCase_					=       None
							else:
														lowerCamelCase_					=       """.""" * max(0      ,					spaces - 2							) + """# {:""" + str(5_0 - spaces							) + """s}"""
														lowerCamelCase_					=       fmt.format(UpperCAmelCase__							)

							# Print and recurse (if needed).
							if isinstance(UpperCAmelCase__      ,					UpperCAmelCase__							):
														if msg is not None:
																					print(UpperCAmelCase__							)
														for k in val.keys():
																					recursive_print(UpperCAmelCase__      ,					val[k]      ,					spaces + 2							)
							elif isinstance(UpperCAmelCase__      ,					torch.Tensor							):
														print(UpperCAmelCase__      ,					""":"""      ,					val.size()							)
							else:
														print(UpperCAmelCase__      ,					""":"""      ,					UpperCAmelCase__							)



def   __lowerCAmelCase    (				UpperCAmelCase__					:   Optional[int]      ,					UpperCAmelCase__					:   int      ,					UpperCAmelCase__					:   str      ,					UpperCAmelCase__					:   Tuple      ,					UpperCAmelCase__					:   Any							)	->     Dict:
							# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
							# for compatibility with later versions of NVIDIA Megatron-LM.
							# The inverse operation is performed inside Megatron-LM to read checkpoints:
							# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
							# If param is the weight tensor of the self-attention block, the returned tensor
							# will have to be transposed one more time to be read by HuggingFace GPT2.
							lowerCamelCase_					=       param.size()
							if checkpoint_version == 1.0:
														# version 1.0 stores [num_heads * hidden_size * num_splits, :]
														lowerCamelCase_					=       (num_heads, hidden_size, num_splits) + input_shape[1:]
														lowerCamelCase_					=       param.view(*UpperCAmelCase__							)
														lowerCamelCase_					=       param.transpose(0      ,					2							)
														lowerCamelCase_					=       param.transpose(1      ,					2							).contiguous()
							elif checkpoint_version >= 2.0:
														# other versions store [num_heads * num_splits * hidden_size, :]
														lowerCamelCase_					=       (num_heads, num_splits, hidden_size) + input_shape[1:]
														lowerCamelCase_					=       param.view(*UpperCAmelCase__							)
														lowerCamelCase_					=       param.transpose(0      ,					1							).contiguous()
							lowerCamelCase_					=       param.view(*UpperCAmelCase__							)
							return param



def   __lowerCAmelCase    (				UpperCAmelCase__					:   Union[str, Any]      ,					UpperCAmelCase__					:   str      ,					UpperCAmelCase__					:   List[Any]							)	->     Any:
							# The converted output model.
							lowerCamelCase_					=       {}

							# old versions did not store training args
							lowerCamelCase_					=       input_state_dict.get("""args"""      ,					UpperCAmelCase__							)
							if ds_args is not None:
														# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
														# from pprint import pprint
														# pprint(vars(ds_args))

														lowerCamelCase_					=       ds_args.padded_vocab_size
														lowerCamelCase_					=       ds_args.max_position_embeddings
														lowerCamelCase_					=       ds_args.hidden_size
														lowerCamelCase_					=       ds_args.num_layers
														lowerCamelCase_					=       ds_args.num_attention_heads
														lowerCamelCase_					=       ds_args.ffn_hidden_size
														# pprint(config)

							# The number of heads.
							lowerCamelCase_					=       config.n_head
							# The hidden_size per head.
							lowerCamelCase_					=       config.n_embd // config.n_head
							# Megatron-LM checkpoint version
							if "checkpoint_version" in input_state_dict.keys():
														lowerCamelCase_					=       input_state_dict["""checkpoint_version"""]
							else:
														lowerCamelCase_					=       0.0

							# The model.
							lowerCamelCase_					=       input_state_dict["""model"""]
							# The language model.
							lowerCamelCase_					=       model["""language_model"""]
							# The embeddings.
							lowerCamelCase_					=       lm["""embedding"""]

							# The word embeddings.
							lowerCamelCase_					=       embeddings["""word_embeddings"""]["""weight"""]
							# Truncate the embedding table to vocab_size rows.
							lowerCamelCase_					=       word_embeddings[: config.vocab_size, :]
							lowerCamelCase_					=       word_embeddings

							# The position embeddings.
							lowerCamelCase_					=       embeddings["""position_embeddings"""]["""weight"""]
							# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
							lowerCamelCase_					=       pos_embeddings.size(0							)
							if n_positions != config.n_positions:
														raise ValueError(
														    F'''pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'''							)
							# Store the position embeddings.
							lowerCamelCase_					=       pos_embeddings

							# The transformer.
							lowerCamelCase_					=       lm["""transformer"""] if """transformer""" in lm.keys() else lm["""encoder"""]

							# The regex to extract layer names.
							lowerCamelCase_					=       re.compile(R"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"""							)

							# The simple map of names for "automated" rules.
							lowerCamelCase_					=       {
							    """attention.dense""": """.attn.c_proj.""",
							    """self_attention.dense""": """.attn.c_proj.""",
							    """mlp.dense_h_to_4h""": """.mlp.c_fc.""",
							    """mlp.dense_4h_to_h""": """.mlp.c_proj.""",
							}

							# Extract the layers.
							for key, val in transformer.items():
														# Match the name.
														lowerCamelCase_					=       layer_re.match(UpperCAmelCase__							)

														# Stop if that's not a layer
														if m is None:
																					break

														# The index of the layer.
														lowerCamelCase_					=       int(m.group(1							)							)
														# The name of the operation.
														lowerCamelCase_					=       m.group(2							)
														# Is it a weight or a bias?
														lowerCamelCase_					=       m.group(3							)

														# The name of the layer.
														lowerCamelCase_					=       F'''transformer.h.{layer_idx}'''

														# For layernorm(s), simply store the layer norm.
														if op_name.endswith("""layernorm"""							):
																					lowerCamelCase_					=       """ln_1""" if op_name.startswith("""input"""							) else """ln_2"""
																					lowerCamelCase_					=       val

														# Transpose the QKV matrix.
														elif (
														    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
														) and weight_or_bias == "weight":
																					# Insert a tensor of 1x1xDxD bias.
																					lowerCamelCase_					=       torch.tril(torch.ones((n_positions, n_positions)      ,					dtype=torch.floataa							)							).view(
																					    1      ,					1      ,					UpperCAmelCase__      ,					UpperCAmelCase__							)
																					lowerCamelCase_					=       causal_mask

																					# Insert a "dummy" tensor for masked_bias.
																					lowerCamelCase_					=       torch.tensor(-1e4      ,					dtype=torch.floataa							)
																					lowerCamelCase_					=       masked_bias

																					lowerCamelCase_					=       fix_query_key_value_ordering(UpperCAmelCase__      ,					UpperCAmelCase__      ,					3      ,					UpperCAmelCase__      ,					UpperCAmelCase__							)
																					# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
																					lowerCamelCase_					=       out_val.transpose(0      ,					1							).contiguous()
																					# Store.
																					lowerCamelCase_					=       out_val

														# Transpose the bias.
														elif (
														    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
														) and weight_or_bias == "bias":
																					lowerCamelCase_					=       fix_query_key_value_ordering(UpperCAmelCase__      ,					UpperCAmelCase__      ,					3      ,					UpperCAmelCase__      ,					UpperCAmelCase__							)
																					# Store. No change of shape.
																					lowerCamelCase_					=       out_val

														# Transpose the weights.
														elif weight_or_bias == "weight":
																					lowerCamelCase_					=       megatron_to_transformers[op_name]
																					lowerCamelCase_					=       val.transpose(0      ,					1							)

														# Copy the bias.
														elif weight_or_bias == "bias":
																					lowerCamelCase_					=       megatron_to_transformers[op_name]
																					lowerCamelCase_					=       val

    # DEBUG.
							assert config.n_layer == layer_idx + 1

							# The final layernorm.
							lowerCamelCase_					=       transformer["""final_layernorm.weight"""]
							lowerCamelCase_					=       transformer["""final_layernorm.bias"""]

							# For LM head, transformers' wants the matrix to weight embeddings.
							lowerCamelCase_					=       word_embeddings

							# It should be done!
							return output_state_dict



def   __lowerCAmelCase    (				)	->     Optional[int]:
							# Create the argument parser.
							lowerCamelCase_					=       argparse.ArgumentParser()
							parser.add_argument("""--print-checkpoint-structure"""      ,					action="""store_true"""							)
							parser.add_argument(
							    """path_to_checkpoint"""      ,					type=UpperCAmelCase__      ,					help="""Path to the checkpoint file (.zip archive or direct .pt file)"""      ,					)
							parser.add_argument(
							    """--config_file"""      ,					default=""""""      ,					type=UpperCAmelCase__      ,					help="""An optional config json file describing the pre-trained model."""      ,					)
							lowerCamelCase_					=       parser.parse_args()

							# Extract the basename.
							lowerCamelCase_					=       os.path.dirname(args.path_to_checkpoint							)

							# Load the model.
							# the .zip is very optional, let's keep it for backward compatibility
							print(F'''Extracting PyTorch state dictionary from {args.path_to_checkpoint}'''							)
							if args.path_to_checkpoint.endswith(""".zip"""							):
														with zipfile.ZipFile(args.path_to_checkpoint      ,					"""r"""							) as checkpoint:
																					with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt"""							) as pytorch_dict:
																												lowerCamelCase_					=       torch.load(UpperCAmelCase__      ,					map_location="""cpu"""							)
							else:
														lowerCamelCase_					=       torch.load(args.path_to_checkpoint      ,					map_location="""cpu"""							)

							lowerCamelCase_					=       input_state_dict.get("""args"""      ,					UpperCAmelCase__							)

							# Read the config, or default to the model released by NVIDIA.
							if args.config_file == "":
														if ds_args is not None:
																					if ds_args.bias_gelu_fusion:
																												lowerCamelCase_					=       """gelu_fast"""
																					elif ds_args.openai_gelu:
																												lowerCamelCase_					=       """gelu_new"""
																					else:
																												lowerCamelCase_					=       """gelu"""
														else:
																					# in the very early days this used to be "gelu_new"
																					lowerCamelCase_					=       """gelu_new"""

														# Spell out all parameters in case the defaults change.
														lowerCamelCase_					=       GPTaConfig(
														    vocab_size=5_0_2_5_7      ,					n_positions=1_0_2_4      ,					n_embd=1_0_2_4      ,					n_layer=2_4      ,					n_head=1_6      ,					n_inner=4_0_9_6      ,					activation_function=UpperCAmelCase__      ,					resid_pdrop=0.1      ,					embd_pdrop=0.1      ,					attn_pdrop=0.1      ,					layer_norm_epsilon=1e-5      ,					initializer_range=0.0_2      ,					summary_type="""cls_index"""      ,					summary_use_proj=UpperCAmelCase__      ,					summary_activation=UpperCAmelCase__      ,					summary_proj_to_labels=UpperCAmelCase__      ,					summary_first_dropout=0.1      ,					scale_attn_weights=UpperCAmelCase__      ,					use_cache=UpperCAmelCase__      ,					bos_token_id=5_0_2_5_6      ,					eos_token_id=5_0_2_5_6      ,					)
							else:
														lowerCamelCase_					=       GPTaConfig.from_json_file(args.config_file							)

							lowerCamelCase_					=       ["""GPT2LMHeadModel"""]

							# Convert.
							print("""Converting"""							)
							lowerCamelCase_					=       convert_megatron_checkpoint(UpperCAmelCase__      ,					UpperCAmelCase__      ,					UpperCAmelCase__							)

							# Print the structure of converted state dict.
							if args.print_checkpoint_structure:
														recursive_print(UpperCAmelCase__      ,					UpperCAmelCase__							)

							# Add tokenizer class info to config
							# see https://github.com/huggingface/transformers/issues/13906)
							if ds_args is not None:
														lowerCamelCase_					=       ds_args.tokenizer_type
														if tokenizer_type == "GPT2BPETokenizer":
																					lowerCamelCase_					=       """gpt2"""
														elif tokenizer_type == "PretrainedFromHF":
																					lowerCamelCase_					=       ds_args.tokenizer_name_or_path
														else:
																					raise ValueError(F'''Unrecognized tokenizer_type {tokenizer_type}'''							)
							else:
														lowerCamelCase_					=       """gpt2"""

							lowerCamelCase_					=       AutoTokenizer.from_pretrained(UpperCAmelCase__							)
							lowerCamelCase_					=       type(UpperCAmelCase__							).__name__
							lowerCamelCase_					=       tokenizer_class

							# Store the config to file.
							print("""Saving config"""							)
							config.save_pretrained(UpperCAmelCase__							)

							# Save tokenizer based on args
							print(F'''Adding {tokenizer_class} tokenizer files'''							)
							tokenizer.save_pretrained(UpperCAmelCase__							)

							# Store the state_dict to file.
							lowerCamelCase_					=       os.path.join(UpperCAmelCase__      ,					"""pytorch_model.bin"""							)
							print(F'''Saving checkpoint to "{output_checkpoint_file}"'''							)
							torch.save(UpperCAmelCase__      ,					UpperCAmelCase__							)


####################################################################################################

if __name__ == "__main__":
		main()

####################################################################################################







def   __lowerCAmelCase    (				UpperCAmelCase__					:   int							)	->     int:

							if not isinstance(UpperCAmelCase__      ,					UpperCAmelCase__							):
														raise ValueError("""multiplicative_persistence() only accepts integral values"""							)
							if num < 0:
														raise ValueError("""multiplicative_persistence() does not accept negative values"""							)

							lowerCamelCase_					=       0
							lowerCamelCase_					=       str(UpperCAmelCase__							)

							while len(UpperCAmelCase__							) != 1:
														lowerCamelCase_					=       [int(UpperCAmelCase__							) for i in num_string]

														lowerCamelCase_					=       1
														for i in range(0      ,					len(UpperCAmelCase__							)							):
																					total *= numbers[i]

														lowerCamelCase_					=       str(UpperCAmelCase__							)

														steps += 1
							return steps



def   __lowerCAmelCase    (				UpperCAmelCase__					:   int							)	->     int:

							if not isinstance(UpperCAmelCase__      ,					UpperCAmelCase__							):
														raise ValueError("""additive_persistence() only accepts integral values"""							)
							if num < 0:
														raise ValueError("""additive_persistence() does not accept negative values"""							)

							lowerCamelCase_					=       0
							lowerCamelCase_					=       str(UpperCAmelCase__							)

							while len(UpperCAmelCase__							) != 1:
														lowerCamelCase_					=       [int(UpperCAmelCase__							) for i in num_string]

														lowerCamelCase_					=       0
														for i in range(0      ,					len(UpperCAmelCase__							)							):
																					total += numbers[i]

														lowerCamelCase_					=       str(UpperCAmelCase__							)

														steps += 1
							return steps


if __name__ == "__main__":
		import doctest

		doctest.testmod()





