


"""simple docstring"""


import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig






def    a__       ( __lowercase   ,	__lowercase   ,	__lowercase=0      )		->		int:
				# Format the message.
				if name is None:
								_A         =	None
				else:
								_A         =	"." * max(0   ,	spaces - 2      ) + "# {:" + str(50 - spaces      ) + "s}"
								_A         =	fmt.format(__lowercase      )

				# Print and recurse (if needed).
				if isinstance(__lowercase   ,	__lowercase      ):
								if msg is not None:
												print(__lowercase      )
								for k in val.keys():
												recursive_print(__lowercase   ,	val[k]   ,	spaces + 2      )
				elif isinstance(__lowercase   ,	torch.Tensor      ):
								print(__lowercase   ,	":"   ,	val.size()      )
				else:
								print(__lowercase   ,	":"   ,	__lowercase      )






def    a__       ( __lowercase   ,	__lowercase   ,	__lowercase   ,	__lowercase      )		->		List[Any]:
				# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
				# for compatibility with later versions of NVIDIA Megatron-LM.
				# The inverse operation is performed inside Megatron-LM to read checkpoints:
				# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
				# If param is the weight tensor of the self-attention block, the returned tensor
				# will have to be transposed one more time to be read by HuggingFace GPT2.
				_A         =	param.size()
				# other versions store [num_heads * num_splits * hidden_size, :]
				_A         =	(num_heads, num_splits, hidden_size) + input_shape[1:]
				_A         =	param.view(*__lowercase      )
				_A         =	param.transpose(0   ,	1      ).contiguous()
				_A         =	param.view(*__lowercase      )
				return param






def    a__       ( __lowercase   ,	__lowercase      )		->		List[str]:
				_A         =	config.n_positions
				_A         =	config.n_layer
				_A         =	config.vocab_size
				_A         =	config.n_head
				_A         =	config.n_embd // config.n_head

				_A         =	sd_megatron["model.language_model.embedding.word_embeddings.weight"][:vocab_size, :]
				_A         =	{
				    "transformer.wte.weight": word_embeddings,
				    "transformer.wpe.weight": sd_megatron["model.language_model.embedding.position_embeddings.weight"],
				    "transformer.ln_f.weight": sd_megatron["model.language_model.encoder.final_layernorm.weight"],
				    "transformer.ln_f.bias": sd_megatron["model.language_model.encoder.final_layernorm.bias"],
				}

				_A         =	"model.language_model.encoder.layers."
				for i in range(__lowercase      ):
								_A         =	torch.tril(torch.ones((n_positions, n_positions)   ,	dtype=torch.bool      )      )
								_A         =	causal_mask.view(1   ,	1   ,	__lowercase   ,	__lowercase      )
								_A         =	causal_mask
								_A         =	torch.tensor(-1E4   ,	dtype=torch.bfloataa      )

								_A         =	sd_megatron[f"""{pf}{i}.input_layernorm.weight"""]
								_A         =	sd_megatron[f"""{pf}{i}.input_layernorm.bias"""]

								_A         =	sd_megatron[f"""{pf}{i}.self_attention.query_key_value.weight"""]
								_A         =	fix_query_key_value_ordering(__lowercase   ,	3   ,	__lowercase   ,	__lowercase      )
								_A         =	vala.transpose(0   ,	1      ).contiguous()

								_A         =	sd_megatron[f"""{pf}{i}.self_attention.query_key_value.bias"""]
								_A         =	fix_query_key_value_ordering(__lowercase   ,	3   ,	__lowercase   ,	__lowercase      )
								_A         =	vala

								_A         =	sd_megatron[f"""{pf}{i}.self_attention.dense.weight"""].transpose(
								    0   ,	1      )
								_A         =	sd_megatron[f"""{pf}{i}.self_attention.dense.bias"""]
								_A         =	sd_megatron[f"""{pf}{i}.post_attention_layernorm.weight"""]
								_A         =	sd_megatron[f"""{pf}{i}.post_attention_layernorm.bias"""]
								_A         =	sd_megatron[f"""{pf}{i}.mlp.dense_h_to_4h.weight"""].transpose(0   ,	1      )
								_A         =	sd_megatron[f"""{pf}{i}.mlp.dense_h_to_4h.bias"""]
								_A         =	sd_megatron[f"""{pf}{i}.mlp.dense_4h_to_h.weight"""].transpose(
								    0   ,	1      )
								_A         =	sd_megatron[f"""{pf}{i}.mlp.dense_4h_to_h.bias"""]

				# For LM head, transformers' wants the matrix to weight embeddings.
				_A         =	word_embeddings

				return sd_hf






def    a__       ( __lowercase   ,	__lowercase      )		->		int:
				_A         =	6_4000
				_A         =	config_megatron["encoder_seq_length"]
				_A         =	config_megatron["hidden_size"]
				_A         =	config_megatron["num_layers"]
				_A         =	config_megatron["num_attention_heads"]
				_A         =	config_megatron["ffn_hidden_size"]
				_A         =	"gelu"
				_A         =	0.1
				_A         =	0.1
				_A         =	0.1
				_A         =	config_megatron["layernorm_epsilon"]  # 1e-5
				_A         =	config_megatron["init_method_std"]  # 0.02
				_A         =	config_megatron["apply_query_key_layer_scaling"]  # True
				_A         =	True
				_A         =	True

				# This identifies the 6.7B (7B) model which uses a different tokenizer
				if config_megatron["hidden_size"] == 4096:
								_A         =	1  # <|endoftext|>
								_A         =	1  # <|endoftext|>
								_A         =	0  # <unk>
				else:
								_A         =	2  # <s>
								_A         =	3  # <|endoftext|>
								_A         =	0  # <pad>

				return config_hf






def    a__       ( __lowercase      )		->		List[str]:
				print(__lowercase      )

				_A         =	args.checkpoint_path
				_A         =	args.save_path
				if isfile(__lowercase      ):
								raise FileNotFoundError(f"""ERROR! could not find file {checkpoint_path}"""      )

				# Load the model.
				_A         =	torch.load(__lowercase   ,	map_location="cpu"      )

				# Load the config.
				_A         =	checkpoint["hyper_parameters"]["cfg"]
				_A         =	GPTaConfig()
				_A         =	copy_config(config_hf=__lowercase   ,	config_megatron=__lowercase      )
				_A         =	["GPT2LMHeadModel"]

				_A         =	checkpoint["state_dict"]

				# Convert.
				print("Converting"      )
				_A         =	convert_megatron_checkpoint(__lowercase   ,	__lowercase      )

				# Print the structure of converted state dict.
				if args.print_checkpoint_structure:
								recursive_print(__lowercase   ,	__lowercase      )

				_A         =	"GPTSw3Tokenizer"

				# Store the config to file.
				print("Saving config"      )
				config_hf.save_pretrained(__lowercase      )

				# Store the state_dict to file.
				_A         =	os.path.join(__lowercase   ,	"pytorch_model.bin"      )
				print(f"""Saving checkpoint to \"{output_checkpoint_file}\""""      )
				torch.save(__lowercase   ,	__lowercase      )


if __name__ == "__main__":
		a_											=			argparse.ArgumentParser()
		parser.add_argument(
		    "--checkpoint_path",
		    type=str,
		    required=True,
		    help="e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000",
		)
		parser.add_argument("--save_path", type=str, required=True, help="e.g. /home/user/gpt-sw3/hf")
		parser.add_argument("--print-checkpoint-structure", action="store_true")
		a_											=			parser.parse_args()
		main(_args)



"""simple docstring"""


import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


a_											=			get_logger(__name__)


if TYPE_CHECKING:
		from .builder import DatasetBuilder






def    a__       ( __lowercase   ,	__lowercase = None      )		->		Optional[Any]:
				_A         =	importlib.import_module(__lowercase      )

				# TODO(QL): always update the module to add subsequent new authentication
				if hasattr(__lowercase   ,	"_patched_for_streaming"      ) and module._patched_for_streaming:
								return

				def wrap_auth(__lowercase      ):
								@wraps(__lowercase      )
								def wrapper(*__lowercase   ,	**__lowercase      ):
												return function(*__lowercase   ,	use_auth_token=__lowercase   ,	**__lowercase      )

								_A         =	"wrap_auth"
								return wrapper

				# open files in a streaming fashion
				patch_submodule(__lowercase   ,	"open"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"os.listdir"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"os.walk"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"glob.glob"   ,	wrap_auth(__lowercase      )      ).start()
				# allow to navigate in remote zip files
				patch_submodule(__lowercase   ,	"os.path.join"   ,	__lowercase      ).start()
				patch_submodule(__lowercase   ,	"os.path.dirname"   ,	__lowercase      ).start()
				patch_submodule(__lowercase   ,	"os.path.basename"   ,	__lowercase      ).start()
				patch_submodule(__lowercase   ,	"os.path.relpath"   ,	__lowercase      ).start()
				patch_submodule(__lowercase   ,	"os.path.split"   ,	__lowercase      ).start()
				patch_submodule(__lowercase   ,	"os.path.splitext"   ,	__lowercase      ).start()
				# allow checks on paths
				patch_submodule(__lowercase   ,	"os.path.exists"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"os.path.isdir"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"os.path.isfile"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"os.path.getsize"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"pathlib.Path"   ,	__lowercase      ).start()
				# file readers
				patch_submodule(__lowercase   ,	"gzip.open"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"numpy.load"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"pandas.read_csv"   ,	wrap_auth(__lowercase      )   ,	attrs=["__version__"]      ).start()
				patch_submodule(__lowercase   ,	"pandas.read_excel"   ,	wrap_auth(__lowercase      )   ,	attrs=["__version__"]      ).start()
				patch_submodule(__lowercase   ,	"scipy.io.loadmat"   ,	wrap_auth(__lowercase      )   ,	attrs=["__version__"]      ).start()
				patch_submodule(__lowercase   ,	"xml.etree.ElementTree.parse"   ,	wrap_auth(__lowercase      )      ).start()
				patch_submodule(__lowercase   ,	"xml.dom.minidom.parse"   ,	wrap_auth(__lowercase      )      ).start()
				_A         =	True






def    a__       ( __lowercase      )		->		str:
				extend_module_for_streaming(builder.__module__   ,	use_auth_token=builder.use_auth_token      )
				# if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
				if not builder.__module__.startswith("datasets."      ):  # check that it's not a packaged builder like csv
								for imports in get_imports(inspect.getfile(builder.__class__      )      ):
												if imports[0] == "internal":
																_A         =	imports[1]
																_A         =	".".join(builder.__module__.split("."      )[:-1] + [internal_import_name]      )
																extend_module_for_streaming(__lowercase   ,	use_auth_token=builder.use_auth_token      )

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
				from .builder import DatasetBuilder

				_A         =	[
				    cls.__module__
				    for cls in type(__lowercase      ).__mro__[1:]  # make sure it's not the same module we've already patched
				    if issubclass(cls   ,	__lowercase      ) and cls.__module__ != DatasetBuilder.__module__
				]  # check it's not a standard builder from datasets.builder
				for module in parent_builder_modules:
								extend_module_for_streaming(__lowercase   ,	use_auth_token=builder.use_auth_token      )