



"""simple docstring"""

# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig





def    __magic_name__     ( _lowerCamelCase					:			Union[str, Any]	,							_lowerCamelCase					:			Any	,							_lowerCamelCase					:			Union[str, Any]=0       ):
							# Format the message.
							if name is None:
														__a		:		Dict						=							None
							else:
														__a		:		List[Any]						=							"." * max(0	,							spaces - 2       ) + "# {:" + str(5_0 - spaces       ) + "s}"
														__a		:		Optional[Any]						=							fmt.format(__UpperCamelCase       )

							# Print and recurse (if needed).
							if isinstance(__UpperCamelCase	,							__UpperCamelCase       ):
														if msg is not None:
																					print(__UpperCamelCase       )
														for k in val.keys():
																					recursive_print(__UpperCamelCase	,							val[k]	,							spaces + 2       )
							elif isinstance(__UpperCamelCase	,							torch.Tensor       ):
														print(__UpperCamelCase	,							""":"""	,							val.size()       )
							else:
														print(__UpperCamelCase	,							""":"""	,							__UpperCamelCase       )





def    __magic_name__     ( _lowerCamelCase					:			List[Any]	,							_lowerCamelCase					:			List[Any]	,							_lowerCamelCase					:			Optional[Any]	,							_lowerCamelCase					:			Any	,							_lowerCamelCase					:			Optional[Any]       ):
							# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
							# for compatibility with later versions of NVIDIA Megatron-LM.
							# The inverse operation is performed inside Megatron-LM to read checkpoints:
							# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
							# If param is the weight tensor of the self-attention block, the returned tensor
							# will have to be transposed one more time to be read by HuggingFace GPT2.
							__a		:		Optional[int]						=							param.size()
							if checkpoint_version == 1.0:
														# version 1.0 stores [num_heads * hidden_size * num_splits, :]
														__a		:		Optional[int]						=							(num_heads, hidden_size, num_splits) + input_shape[1:]
														__a		:		List[Any]						=							param.view(*__UpperCamelCase       )
														__a		:		Tuple						=							param.transpose(0	,							2       )
														__a		:		List[str]						=							param.transpose(1	,							2       ).contiguous()
							elif checkpoint_version >= 2.0:
														# other versions store [num_heads * num_splits * hidden_size, :]
														__a		:		Dict						=							(num_heads, num_splits, hidden_size) + input_shape[1:]
														__a		:		Optional[int]						=							param.view(*__UpperCamelCase       )
														__a		:		Any						=							param.transpose(0	,							1       ).contiguous()
							__a		:		Dict						=							param.view(*__UpperCamelCase       )
							return param





def    __magic_name__     ( _lowerCamelCase					:			Any	,							_lowerCamelCase					:			List[Any]	,							_lowerCamelCase					:			Any       ):
							# The converted output model.
							__a		:		Dict						=							{}

							# old versions did not store training args
							__a		:		Union[str, Any]						=							input_state_dict.get("""args"""	,							__UpperCamelCase       )
							if ds_args is not None:
														# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
														# from pprint import pprint
														# pprint(vars(ds_args))

														__a		:		Dict						=							ds_args.padded_vocab_size
														__a		:		Optional[int]						=							ds_args.max_position_embeddings
														__a		:		Any						=							ds_args.hidden_size
														__a		:		int						=							ds_args.num_layers
														__a		:		int						=							ds_args.num_attention_heads
														__a		:		Any						=							ds_args.ffn_hidden_size
														# pprint(config)

							# The number of heads.
							__a		:		Tuple						=							config.n_head
							# The hidden_size per head.
							__a		:		Dict						=							config.n_embd // config.n_head
							# Megatron-LM checkpoint version
							if "checkpoint_version" in input_state_dict.keys():
														__a		:		Union[str, Any]						=							input_state_dict["checkpoint_version"]
							else:
														__a		:		Union[str, Any]						=							0.0

							# The model.
							__a		:		List[Any]						=							input_state_dict["model"]
							# The language model.
							__a		:		str						=							model["language_model"]
							# The embeddings.
							__a		:		List[Any]						=							lm["embedding"]

							# The word embeddings.
							__a		:		int						=							embeddings["word_embeddings"]["weight"]
							# Truncate the embedding table to vocab_size rows.
							__a		:		List[str]						=							word_embeddings[: config.vocab_size, :]
							__a		:		int						=							word_embeddings

							# The position embeddings.
							__a		:		Union[str, Any]						=							embeddings["position_embeddings"]["weight"]
							# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
							__a		:		int						=							pos_embeddings.size(0       )
							if n_positions != config.n_positions:
														raise ValueError(
														    F'''pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'''       )
							# Store the position embeddings.
							__a		:		Optional[Any]						=							pos_embeddings

							# The transformer.
							__a		:		Dict						=							lm["transformer"] if "transformer" in lm.keys() else lm["encoder"]

							# The regex to extract layer names.
							__a		:		Any						=							re.compile(r"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"""       )

							# The simple map of names for "automated" rules.
							__a		:		Any						=							{
							    "attention.dense": ".attn.c_proj.",
							    "self_attention.dense": ".attn.c_proj.",
							    "mlp.dense_h_to_4h": ".mlp.c_fc.",
							    "mlp.dense_4h_to_h": ".mlp.c_proj.",
							}

							# Extract the layers.
							for key, val in transformer.items():
														# Match the name.
														__a		:		int						=							layer_re.match(__UpperCamelCase       )

														# Stop if that's not a layer
														if m is None:
																					break

														# The index of the layer.
														__a		:		Any						=							int(m.group(1       )       )
														# The name of the operation.
														__a		:		Optional[Any]						=							m.group(2       )
														# Is it a weight or a bias?
														__a		:		str						=							m.group(3       )

														# The name of the layer.
														__a		:		Optional[Any]						=							F'''transformer.h.{layer_idx}'''

														# For layernorm(s), simply store the layer norm.
														if op_name.endswith("""layernorm"""       ):
																					__a		:		List[str]						=							"ln_1" if op_name.startswith("""input"""       ) else "ln_2"
																					__a		:		Any						=							val

														# Transpose the QKV matrix.
														elif (
														    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
														) and weight_or_bias == "weight":
																					# Insert a tensor of 1x1xDxD bias.
																					__a		:		List[str]						=							torch.tril(torch.ones((n_positions, n_positions)	,							dtype=torch.floataa       )       ).view(
																					    1	,							1	,							__UpperCamelCase	,							__UpperCamelCase       )
																					__a		:		Optional[Any]						=							causal_mask

																					# Insert a "dummy" tensor for masked_bias.
																					__a		:		List[Any]						=							torch.tensor(-1E4	,							dtype=torch.floataa       )
																					__a		:		Dict						=							masked_bias

																					__a		:		Union[str, Any]						=							fix_query_key_value_ordering(__UpperCamelCase	,							__UpperCamelCase	,							3	,							__UpperCamelCase	,							__UpperCamelCase       )
																					# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
																					__a		:		Optional[int]						=							out_val.transpose(0	,							1       ).contiguous()
																					# Store.
																					__a		:		List[str]						=							out_val

														# Transpose the bias.
														elif (
														    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
														) and weight_or_bias == "bias":
																					__a		:		Dict						=							fix_query_key_value_ordering(__UpperCamelCase	,							__UpperCamelCase	,							3	,							__UpperCamelCase	,							__UpperCamelCase       )
																					# Store. No change of shape.
																					__a		:		List[str]						=							out_val

														# Transpose the weights.
														elif weight_or_bias == "weight":
																					__a		:		Dict						=							megatron_to_transformers[op_name]
																					__a		:		Dict						=							val.transpose(0	,							1       )

														# Copy the bias.
														elif weight_or_bias == "bias":
																					__a		:		str						=							megatron_to_transformers[op_name]
																					__a		:		Union[str, Any]						=							val

    # DEBUG.
							assert config.n_layer == layer_idx + 1

							# The final layernorm.
							__a		:		str						=							transformer["final_layernorm.weight"]
							__a		:		int						=							transformer["final_layernorm.bias"]

							# For LM head, transformers' wants the matrix to weight embeddings.
							__a		:		Any						=							word_embeddings

							# It should be done!
							return output_state_dict





def    __magic_name__     ( ):
							# Create the argument parser.
							__a		:		List[str]						=							argparse.ArgumentParser()
							parser.add_argument("""--print-checkpoint-structure"""	,							action="""store_true"""       )
							parser.add_argument(
							    """path_to_checkpoint"""	,							type=__UpperCamelCase	,							help="""Path to the checkpoint file (.zip archive or direct .pt file)"""	,							)
							parser.add_argument(
							    """--config_file"""	,							default=""""""	,							type=__UpperCamelCase	,							help="""An optional config json file describing the pre-trained model."""	,							)
							__a		:		Any						=							parser.parse_args()

							# Extract the basename.
							__a		:		List[str]						=							os.path.dirname(args.path_to_checkpoint       )

							# Load the model.
							# the .zip is very optional, let's keep it for backward compatibility
							print(F'''Extracting PyTorch state dictionary from {args.path_to_checkpoint}'''       )
							if args.path_to_checkpoint.endswith(""".zip"""       ):
														with zipfile.ZipFile(args.path_to_checkpoint	,							"""r"""       ) as checkpoint:
																					with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt"""       ) as pytorch_dict:
																												__a		:		Tuple						=							torch.load(__UpperCamelCase	,							map_location="""cpu"""       )
							else:
														__a		:		Tuple						=							torch.load(args.path_to_checkpoint	,							map_location="""cpu"""       )

							__a		:		int						=							input_state_dict.get("""args"""	,							__UpperCamelCase       )

							# Read the config, or default to the model released by NVIDIA.
							if args.config_file == "":
														if ds_args is not None:
																					if ds_args.bias_gelu_fusion:
																												__a		:		int						=							"gelu_fast"
																					elif ds_args.openai_gelu:
																												__a		:		int						=							"gelu_new"
																					else:
																												__a		:		Tuple						=							"gelu"
														else:
																					# in the very early days this used to be "gelu_new"
																					__a		:		str						=							"gelu_new"

														# Spell out all parameters in case the defaults change.
														__a		:		Dict						=							GPTaConfig(
														    vocab_size=5_0_2_5_7	,							n_positions=1_0_2_4	,							n_embd=1_0_2_4	,							n_layer=2_4	,							n_head=1_6	,							n_inner=4_0_9_6	,							activation_function=__UpperCamelCase	,							resid_pdrop=0.1	,							embd_pdrop=0.1	,							attn_pdrop=0.1	,							layer_norm_epsilon=1E-5	,							initializer_range=0.02	,							summary_type="""cls_index"""	,							summary_use_proj=__UpperCamelCase	,							summary_activation=__UpperCamelCase	,							summary_proj_to_labels=__UpperCamelCase	,							summary_first_dropout=0.1	,							scale_attn_weights=__UpperCamelCase	,							use_cache=__UpperCamelCase	,							bos_token_id=5_0_2_5_6	,							eos_token_id=5_0_2_5_6	,							)
							else:
														__a		:		Optional[Any]						=							GPTaConfig.from_json_file(args.config_file       )

							__a		:		str						=							["GPT2LMHeadModel"]

							# Convert.
							print("""Converting"""       )
							__a		:		Optional[int]						=							convert_megatron_checkpoint(__UpperCamelCase	,							__UpperCamelCase	,							__UpperCamelCase       )

							# Print the structure of converted state dict.
							if args.print_checkpoint_structure:
														recursive_print(__UpperCamelCase	,							__UpperCamelCase       )

							# Add tokenizer class info to config
							# see https://github.com/huggingface/transformers/issues/13906)
							if ds_args is not None:
														__a		:		Tuple						=							ds_args.tokenizer_type
														if tokenizer_type == "GPT2BPETokenizer":
																					__a		:		Dict						=							"gpt2"
														elif tokenizer_type == "PretrainedFromHF":
																					__a		:		Tuple						=							ds_args.tokenizer_name_or_path
														else:
																					raise ValueError(F'''Unrecognized tokenizer_type {tokenizer_type}'''       )
							else:
														__a		:		str						=							"gpt2"

							__a		:		str						=							AutoTokenizer.from_pretrained(__UpperCamelCase       )
							__a		:		Union[str, Any]						=							type(__UpperCamelCase       ).__name__
							__a		:		Tuple						=							tokenizer_class

							# Store the config to file.
							print("""Saving config"""       )
							config.save_pretrained(__UpperCamelCase       )

							# Save tokenizer based on args
							print(F'''Adding {tokenizer_class} tokenizer files'''       )
							tokenizer.save_pretrained(__UpperCamelCase       )

							# Store the state_dict to file.
							__a		:		Tuple						=							os.path.join(__UpperCamelCase	,							"""pytorch_model.bin"""       )
							print(F'''Saving checkpoint to "{output_checkpoint_file}"'''       )
							torch.save(__UpperCamelCase	,							__UpperCamelCase       )


####################################################################################################

if __name__ == "__main__":
	main()

####################################################################################################






from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowerCamelCase		       =      logging.get_logger(__name__)


lowerCamelCase		       =      {
    """Salesforce/codegen-350M-nl""": """https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json""",
    """Salesforce/codegen-350M-multi""": """https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json""",
    """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json""",
    """Salesforce/codegen-2B-nl""": """https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json""",
    """Salesforce/codegen-2B-multi""": """https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json""",
    """Salesforce/codegen-2B-mono""": """https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json""",
    """Salesforce/codegen-6B-nl""": """https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json""",
    """Salesforce/codegen-6B-multi""": """https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json""",
    """Salesforce/codegen-6B-mono""": """https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json""",
    """Salesforce/codegen-16B-nl""": """https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json""",
    """Salesforce/codegen-16B-multi""": """https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json""",
    """Salesforce/codegen-16B-mono""": """https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json""",
}


class 				_a						(       SCREAMING_SNAKE_CASE			):





							'''simple docstring'''

							A				:str =       "codegen"
							A				:Tuple =       {
							    "max_position_embeddings": "n_positions",
							    "hidden_size": "n_embd",
							    "num_attention_heads": "n_head",
							    "num_hidden_layers": "n_layer",
							}



							def __init__(							self ,					__UpperCAmelCase=5_0400 ,					__UpperCAmelCase=2048 ,					__UpperCAmelCase=2048 ,					__UpperCAmelCase=4096 ,					__UpperCAmelCase=28 ,					__UpperCAmelCase=16 ,					__UpperCAmelCase=64 ,					__UpperCAmelCase=None ,					__UpperCAmelCase="gelu_new" ,					__UpperCAmelCase=0.0 ,					__UpperCAmelCase=0.0 ,					__UpperCAmelCase=0.0 ,					__UpperCAmelCase=1E-5 ,					__UpperCAmelCase=0.0_2 ,					__UpperCAmelCase=True ,					__UpperCAmelCase=5_0256 ,					__UpperCAmelCase=5_0256 ,					__UpperCAmelCase=False ,					**__UpperCAmelCase ,					):







											"""simple docstring"""



											a__	:							Dict      						= vocab_size
											a__	:							int      						= n_ctx
											a__	:							str      						= n_positions
											a__	:							Union[str, Any]      						= n_embd
											a__	:							Union[str, Any]      						= n_layer
											a__	:							Optional[int]      						= n_head
											a__	:							Dict      						= n_inner
											a__	:							Optional[Any]      						= rotary_dim
											a__	:							str      						= activation_function
											a__	:							List[Any]      						= resid_pdrop
											a__	:							str      						= embd_pdrop
											a__	:							List[str]      						= attn_pdrop
											a__	:							str      						= layer_norm_epsilon
											a__	:							Any      						= initializer_range
											a__	:							List[str]      						= use_cache

											a__	:							Tuple      						= bos_token_id
											a__	:							List[Any]      						= eos_token_id

											super().__init__(
											    bos_token_id=__UpperCAmelCase ,					eos_token_id=__UpperCAmelCase ,					tie_word_embeddings=__UpperCAmelCase ,					**__UpperCAmelCase				)





class 				_a						(       SCREAMING_SNAKE_CASE			):





							'''simple docstring'''

							def __init__(							self ,					__UpperCAmelCase ,					__UpperCAmelCase = "default" ,					__UpperCAmelCase = None ,					__UpperCAmelCase = False ,					):







											"""simple docstring"""



											super().__init__(__UpperCAmelCase ,					task=__UpperCAmelCase ,					patching_specs=__UpperCAmelCase ,					use_past=__UpperCAmelCase				)
											if not getattr(self._config ,					"pad_token_id" ,					__UpperCAmelCase				):
															# TODO: how to do that better?
															a__	:							Union[str, Any]      						= 0



							@property
							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Optional[Any]      						= OrderedDict({"input_ids": {0: "batch", 1: "sequence"}}				)
											if self.use_past:
															self.fill_with_past_key_values_(__UpperCAmelCase ,					direction="inputs"				)
															a__	:							Any      						= {0: "batch", 1: "past_sequence + sequence"}
											else:
															a__	:							List[Any]      						= {0: "batch", 1: "sequence"}

											return common_inputs



							@property
							def 		_A	(							self				):







											"""simple docstring"""



											return self._config.n_layer



							@property
							def 		_A	(							self				):







											"""simple docstring"""



											return self._config.n_head



							def 		_A	(							self ,					__UpperCAmelCase ,					__UpperCAmelCase = -1 ,					__UpperCAmelCase = -1 ,					__UpperCAmelCase = False ,					__UpperCAmelCase = None ,					):







											"""simple docstring"""



											a__	:							Dict      						= super(__UpperCAmelCase ,					self				).generate_dummy_inputs(
											    __UpperCAmelCase ,					batch_size=__UpperCAmelCase ,					seq_length=__UpperCAmelCase ,					is_pair=__UpperCAmelCase ,					framework=__UpperCAmelCase				)

											# We need to order the input in the way they appears in the forward()
											a__	:							Union[str, Any]      						= OrderedDict({"input_ids": common_inputs["input_ids"]}				)

											# Need to add the past_keys
											if self.use_past:
															if not is_torch_available():
																			raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."				)
															else:
																			import torch

																			a__						,   a__	:							Union[str, Any]      						= common_inputs["input_ids"].shape
																			# Not using the same length for past_key_values
																			a__	:							Any      						= seqlen + 2
																			a__	:							str      						= (
																			    batch,
																			    self.num_attention_heads,
																			    past_key_values_length,
																			    self._config.hidden_size // self.num_attention_heads,
																			)
																			a__	:							str      						= [
																			    (torch.zeros(__UpperCAmelCase				), torch.zeros(__UpperCAmelCase				)) for _ in range(self.num_layers				)
																			]

											a__	:							Union[str, Any]      						= common_inputs["attention_mask"]
											if self.use_past:
															a__	:							str      						= ordered_inputs["attention_mask"].dtype
															a__	:							Union[str, Any]      						= torch.cat(
															    [ordered_inputs["attention_mask"], torch.ones(__UpperCAmelCase ,					__UpperCAmelCase ,					dtype=__UpperCAmelCase				)] ,					dim=1				)

											return ordered_inputs




							@property
							def 		_A	(							self				):







											"""simple docstring"""



											return 13


