





# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig


def 							SCREAMING_SNAKE_CASE__						(    __a   , __a   , __a=0   ):
	# Format the message.
	if name is None:
		snake_case_		:			Tuple											=	None
	else:
		snake_case_		:			Any											=	'.' * max(0   , spaces - 2   ) + '# {:' + str(50 - spaces   ) + 's}'
		snake_case_		:			Tuple											=	fmt.format(__A   )

	# Print and recurse (if needed).
	if isinstance(__A   , __A   ):
		if msg is not None:
			print(__A   )
		for k in val.keys():
			recursive_print(__A   , val[k]   , spaces + 2   )
	elif isinstance(__A   , torch.Tensor   ):
		print(__A   , ':'   , val.size()   )
	else:
		print(__A   , ':'   , __A   )


def 							SCREAMING_SNAKE_CASE__						(    __a   , __a   , __a   , __a   , __a   ):
	# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
	# for compatibility with later versions of NVIDIA Megatron-LM.
	# The inverse operation is performed inside Megatron-LM to read checkpoints:
	# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
	# If param is the weight tensor of the self-attention block, the returned tensor
	# will have to be transposed one more time to be read by HuggingFace GPT2.
	snake_case_		:			Dict											=	param.size()
	if checkpoint_version == 1.0:
		# version 1.0 stores [num_heads * hidden_size * num_splits, :]
		snake_case_		:			Tuple											=	(num_heads, hidden_size, num_splits) + input_shape[1:]
		snake_case_		:			List[Any]											=	param.view(*__A   )
		snake_case_		:			List[Any]											=	param.transpose(0   , 2   )
		snake_case_		:			Optional[int]											=	param.transpose(1   , 2   ).contiguous()
	elif checkpoint_version >= 2.0:
		# other versions store [num_heads * num_splits * hidden_size, :]
		snake_case_		:			Dict											=	(num_heads, num_splits, hidden_size) + input_shape[1:]
		snake_case_		:			Optional[Any]											=	param.view(*__A   )
		snake_case_		:			Any											=	param.transpose(0   , 1   ).contiguous()
	snake_case_		:			Dict											=	param.view(*__A   )
	return param


def 							SCREAMING_SNAKE_CASE__						(    __a   , __a   , __a   ):
	# The converted output model.
	snake_case_		:			str											=	{}

	# old versions did not store training args
	snake_case_		:			List[Any]											=	input_state_dict.get('args'   , __A   )
	if ds_args is not None:
		# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
		# from pprint import pprint
		# pprint(vars(ds_args))

		snake_case_		:			Optional[int]											=	ds_args.padded_vocab_size
		snake_case_		:			Dict											=	ds_args.max_position_embeddings
		snake_case_		:			str											=	ds_args.hidden_size
		snake_case_		:			int											=	ds_args.num_layers
		snake_case_		:			Optional[int]											=	ds_args.num_attention_heads
		snake_case_		:			Union[str, Any]											=	ds_args.ffn_hidden_size
		# pprint(config)

	# The number of heads.
	snake_case_		:			int											=	config.n_head
	# The hidden_size per head.
	snake_case_		:			Optional[Any]											=	config.n_embd // config.n_head
	# Megatron-LM checkpoint version
	if "checkpoint_version" in input_state_dict.keys():
		snake_case_		:			List[str]											=	input_state_dict['checkpoint_version']
	else:
		snake_case_		:			List[str]											=	0.0

	# The model.
	snake_case_		:			Dict											=	input_state_dict['model']
	# The language model.
	snake_case_		:			Any											=	model['language_model']
	# The embeddings.
	snake_case_		:			Any											=	lm['embedding']

	# The word embeddings.
	snake_case_		:			List[Any]											=	embeddings['word_embeddings']['weight']
	# Truncate the embedding table to vocab_size rows.
	snake_case_		:			Tuple											=	word_embeddings[: config.vocab_size, :]
	snake_case_		:			Optional[Any]											=	word_embeddings

	# The position embeddings.
	snake_case_		:			int											=	embeddings['position_embeddings']['weight']
	# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
	snake_case_		:			List[str]											=	pos_embeddings.size(0   )
	if n_positions != config.n_positions:
		raise ValueError(
		    f"""pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match"""   )
	# Store the position embeddings.
	snake_case_		:			str											=	pos_embeddings

	# The transformer.
	snake_case_		:			Dict											=	lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

	# The regex to extract layer names.
	snake_case_		:			List[str]											=	re.compile(r'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'   )

	# The simple map of names for "automated" rules.
	snake_case_		:			Dict											=	{
	    'attention.dense': '.attn.c_proj.',
	    'self_attention.dense': '.attn.c_proj.',
	    'mlp.dense_h_to_4h': '.mlp.c_fc.',
	    'mlp.dense_4h_to_h': '.mlp.c_proj.',
	}

	# Extract the layers.
	for key, val in transformer.items():
		# Match the name.
		snake_case_		:			Optional[int]											=	layer_re.match(__A   )

		# Stop if that's not a layer
		if m is None:
			break

		# The index of the layer.
		snake_case_		:			Any											=	int(m.group(1   )   )
		# The name of the operation.
		snake_case_		:			List[Any]											=	m.group(2   )
		# Is it a weight or a bias?
		snake_case_		:			Tuple											=	m.group(3   )

		# The name of the layer.
		snake_case_		:			Optional[Any]											=	f"""transformer.h.{layer_idx}"""

		# For layernorm(s), simply store the layer norm.
		if op_name.endswith('layernorm'   ):
			snake_case_		:			str											=	'ln_1' if op_name.startswith('input'   ) else 'ln_2'
			snake_case_		:			Union[str, Any]											=	val

		# Transpose the QKV matrix.
		elif (
		    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
		) and weight_or_bias == "weight":
			# Insert a tensor of 1x1xDxD bias.
			snake_case_		:			Tuple											=	torch.tril(torch.ones((n_positions, n_positions)   , dtype=torch.floataa   )   ).view(
			    1   , 1   , __A   , __A   )
			snake_case_		:			List[str]											=	causal_mask

			# Insert a "dummy" tensor for masked_bias.
			snake_case_		:			Any											=	torch.tensor(-1E4   , dtype=torch.floataa   )
			snake_case_		:			Tuple											=	masked_bias

			snake_case_		:			Tuple											=	fix_query_key_value_ordering(__A   , __A   , 3   , __A   , __A   )
			# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
			snake_case_		:			List[Any]											=	out_val.transpose(0   , 1   ).contiguous()
			# Store.
			snake_case_		:			Optional[int]											=	out_val

		# Transpose the bias.
		elif (
		    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
		) and weight_or_bias == "bias":
			snake_case_		:			str											=	fix_query_key_value_ordering(__A   , __A   , 3   , __A   , __A   )
			# Store. No change of shape.
			snake_case_		:			Optional[int]											=	out_val

		# Transpose the weights.
		elif weight_or_bias == "weight":
			snake_case_		:			Union[str, Any]											=	megatron_to_transformers[op_name]
			snake_case_		:			str											=	val.transpose(0   , 1   )

		# Copy the bias.
		elif weight_or_bias == "bias":
			snake_case_		:			Optional[Any]											=	megatron_to_transformers[op_name]
			snake_case_		:			Dict											=	val

		# DEBUG.
	assert config.n_layer == layer_idx + 1

	# The final layernorm.
	snake_case_		:			List[str]											=	transformer['final_layernorm.weight']
	snake_case_		:			Optional[int]											=	transformer['final_layernorm.bias']

	# For LM head, transformers' wants the matrix to weight embeddings.
	snake_case_		:			int											=	word_embeddings

	# It should be done!
	return output_state_dict


def 							SCREAMING_SNAKE_CASE__						(    ):
	# Create the argument parser.
	snake_case_		:			List[Any]											=	argparse.ArgumentParser()
	parser.add_argument('--print-checkpoint-structure'   , action='store_true'   )
	parser.add_argument(
	    'path_to_checkpoint'   , type=__A   , help='Path to the checkpoint file (.zip archive or direct .pt file)'   , )
	parser.add_argument(
	    '--config_file'   , default=''   , type=__A   , help='An optional config json file describing the pre-trained model.'   , )
	snake_case_		:			Any											=	parser.parse_args()

	# Extract the basename.
	snake_case_		:			Optional[int]											=	os.path.dirname(args.path_to_checkpoint   )

	# Load the model.
	# the .zip is very optional, let's keep it for backward compatibility
	print(f"""Extracting PyTorch state dictionary from {args.path_to_checkpoint}"""   )
	if args.path_to_checkpoint.endswith('.zip'   ):
		with zipfile.ZipFile(args.path_to_checkpoint   , 'r'   ) as checkpoint:
			with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'   ) as pytorch_dict:
				snake_case_		:			Dict											=	torch.load(__A   , map_location='cpu'   )
	else:
		snake_case_		:			Tuple											=	torch.load(args.path_to_checkpoint   , map_location='cpu'   )

	snake_case_		:			int											=	input_state_dict.get('args'   , __A   )

	# Read the config, or default to the model released by NVIDIA.
	if args.config_file == "":
		if ds_args is not None:
			if ds_args.bias_gelu_fusion:
				snake_case_		:			Tuple											=	'gelu_fast'
			elif ds_args.openai_gelu:
				snake_case_		:			Tuple											=	'gelu_new'
			else:
				snake_case_		:			Any											=	'gelu'
		else:
			# in the very early days this used to be "gelu_new"
			snake_case_		:			Optional[int]											=	'gelu_new'

		# Spell out all parameters in case the defaults change.
		snake_case_		:			Optional[int]											=	GPTaConfig(
		    vocab_size=5_02_57   , n_positions=10_24   , n_embd=10_24   , n_layer=24   , n_head=16   , n_inner=40_96   , activation_function=__A   , resid_pdrop=0.1   , embd_pdrop=0.1   , attn_pdrop=0.1   , layer_norm_epsilon=1E-5   , initializer_range=0.02   , summary_type='cls_index'   , summary_use_proj=__A   , summary_activation=__A   , summary_proj_to_labels=__A   , summary_first_dropout=0.1   , scale_attn_weights=__A   , use_cache=__A   , bos_token_id=5_02_56   , eos_token_id=5_02_56   , )
	else:
		snake_case_		:			Tuple											=	GPTaConfig.from_json_file(args.config_file   )

	snake_case_		:			int											=	['GPT2LMHeadModel']

	# Convert.
	print('Converting'   )
	snake_case_		:			List[str]											=	convert_megatron_checkpoint(__A   , __A   , __A   )

	# Print the structure of converted state dict.
	if args.print_checkpoint_structure:
		recursive_print(__A   , __A   )

	# Add tokenizer class info to config
	# see https://github.com/huggingface/transformers/issues/13906)
	if ds_args is not None:
		snake_case_		:			Tuple											=	ds_args.tokenizer_type
		if tokenizer_type == "GPT2BPETokenizer":
			snake_case_		:			Any											=	'gpt2'
		elif tokenizer_type == "PretrainedFromHF":
			snake_case_		:			List[str]											=	ds_args.tokenizer_name_or_path
		else:
			raise ValueError(f"""Unrecognized tokenizer_type {tokenizer_type}"""   )
	else:
		snake_case_		:			Optional[Any]											=	'gpt2'

	snake_case_		:			Union[str, Any]											=	AutoTokenizer.from_pretrained(__A   )
	snake_case_		:			Any											=	type(__A   ).__name__
	snake_case_		:			Tuple											=	tokenizer_class

	# Store the config to file.
	print('Saving config'   )
	config.save_pretrained(__A   )

	# Save tokenizer based on args
	print(f"""Adding {tokenizer_class} tokenizer files"""   )
	tokenizer.save_pretrained(__A   )

	# Store the state_dict to file.
	snake_case_		:			Optional[Any]											=	os.path.join(__A   , 'pytorch_model.bin'   )
	print(f"""Saving checkpoint to \"{output_checkpoint_file}\""""   )
	torch.save(__A   , __A   )


####################################################################################################

if __name__ == "__main__":
						main()

####################################################################################################








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


_SCREAMING_SNAKE_CASE      	=		{"""configuration_swin""": ["""SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP""", """SwinConfig""", """SwinOnnxConfig"""]}


try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_SCREAMING_SNAKE_CASE      	=		[
						    """SWIN_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """SwinForImageClassification""",
						    """SwinForMaskedImageModeling""",
						    """SwinModel""",
						    """SwinPreTrainedModel""",
						    """SwinBackbone""",
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_SCREAMING_SNAKE_CASE      	=		[
						    """TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """TFSwinForImageClassification""",
						    """TFSwinForMaskedImageModeling""",
						    """TFSwinModel""",
						    """TFSwinPreTrainedModel""",
						]

if TYPE_CHECKING:
						from .configuration_swin import SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP, SwinConfig, SwinOnnxConfig

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_swin import (
												    SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
												    SwinBackbone,
												    SwinForImageClassification,
												    SwinForMaskedImageModeling,
												    SwinModel,
												    SwinPreTrainedModel,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_swin import (
												    TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFSwinForImageClassification,
												    TFSwinForMaskedImageModeling,
												    TFSwinModel,
												    TFSwinPreTrainedModel,
												)

else:
						import sys

						_SCREAMING_SNAKE_CASE      	=		_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

