





import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig





def 						lowerCamelCase__     (     UpperCamelCase__			:       str       ,    UpperCamelCase__			:       str       ,    UpperCamelCase__			:       str=0						)			-> Tuple:



					'''simple docstring'''




					if name is None:
										_snake_case									=				None
					else:
										_snake_case									=				'.' * max(0       ,    spaces - 2						) + '# {:' + str(50 - spaces						) + 's}'
										_snake_case									=				fmt.format(UpperCamelCase__						)

					# Print and recurse (if needed).
					if isinstance(UpperCamelCase__       ,    UpperCamelCase__						):
										if msg is not None:
															print(UpperCamelCase__						)
										for k in val.keys():
															recursive_print(UpperCamelCase__       ,    val[k]       ,    spaces + 2						)
					elif isinstance(UpperCamelCase__       ,    torch.Tensor						):
										print(UpperCamelCase__       ,    ':'       ,    val.size()						)
					else:
										print(UpperCamelCase__       ,    ':'       ,    UpperCamelCase__						)





def 						lowerCamelCase__     (     UpperCamelCase__			:       Optional[Any]       ,    UpperCamelCase__			:       Union[str, Any]       ,    UpperCamelCase__			:       List[str]       ,    UpperCamelCase__			:       str       ,    UpperCamelCase__			:       Optional[int]						)			-> Optional[Any]:



					'''simple docstring'''




					_snake_case									=				param.size()
					if checkpoint_version == 1.0:
										# version 1.0 stores [num_heads * hidden_size * num_splits, :]
										_snake_case									=				(num_heads, hidden_size, num_splits) + input_shape[1:]
										_snake_case									=				param.view(*UpperCamelCase__						)
										_snake_case									=				param.transpose(0       ,    2						)
										_snake_case									=				param.transpose(1       ,    2						).contiguous()
					elif checkpoint_version >= 2.0:
										# other versions store [num_heads * num_splits * hidden_size, :]
										_snake_case									=				(num_heads, num_splits, hidden_size) + input_shape[1:]
										_snake_case									=				param.view(*UpperCamelCase__						)
										_snake_case									=				param.transpose(0       ,    1						).contiguous()
					_snake_case									=				param.view(*UpperCamelCase__						)
					return param





def 						lowerCamelCase__     (     UpperCamelCase__			:       List[str]       ,    UpperCamelCase__			:       List[str]       ,    UpperCamelCase__			:       List[Any]						)			-> int:



					'''simple docstring'''




					_snake_case									=				{}

					# old versions did not store training args
					_snake_case									=				input_state_dict.get('args'       ,    UpperCamelCase__						)
					if ds_args is not None:
										# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
										# from pprint import pprint
										# pprint(vars(ds_args))

										_snake_case									=				ds_args.padded_vocab_size
										_snake_case									=				ds_args.max_position_embeddings
										_snake_case									=				ds_args.hidden_size
										_snake_case									=				ds_args.num_layers
										_snake_case									=				ds_args.num_attention_heads
										_snake_case									=				ds_args.ffn_hidden_size
										# pprint(config)

					# The number of heads.
					_snake_case									=				config.n_head
					# The hidden_size per head.
					_snake_case									=				config.n_embd // config.n_head
					# Megatron-LM checkpoint version
					if "checkpoint_version" in input_state_dict.keys():
										_snake_case									=				input_state_dict['checkpoint_version']
					else:
										_snake_case									=				0.0

					# The model.
					_snake_case									=				input_state_dict['model']
					# The language model.
					_snake_case									=				model['language_model']
					# The embeddings.
					_snake_case									=				lm['embedding']

					# The word embeddings.
					_snake_case									=				embeddings['word_embeddings']['weight']
					# Truncate the embedding table to vocab_size rows.
					_snake_case									=				word_embeddings[: config.vocab_size, :]
					_snake_case									=				word_embeddings

					# The position embeddings.
					_snake_case									=				embeddings['position_embeddings']['weight']
					# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
					_snake_case									=				pos_embeddings.size(0						)
					if n_positions != config.n_positions:
										raise ValueError(
										    F'''pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'''						)
					# Store the position embeddings.
					_snake_case									=				pos_embeddings

					# The transformer.
					_snake_case									=				lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

					# The regex to extract layer names.
					_snake_case									=				re.compile(R'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'						)

					# The simple map of names for "automated" rules.
					_snake_case									=				{
					    'attention.dense': '.attn.c_proj.',
					    'self_attention.dense': '.attn.c_proj.',
					    'mlp.dense_h_to_4h': '.mlp.c_fc.',
					    'mlp.dense_4h_to_h': '.mlp.c_proj.',
					}

					# Extract the layers.
					for key, val in transformer.items():
										# Match the name.
										_snake_case									=				layer_re.match(UpperCamelCase__						)

										# Stop if that's not a layer
										if m is None:
															break

										# The index of the layer.
										_snake_case									=				int(m.group(1						)						)
										# The name of the operation.
										_snake_case									=				m.group(2						)
										# Is it a weight or a bias?
										_snake_case									=				m.group(3						)

										# The name of the layer.
										_snake_case									=				F'''transformer.h.{layer_idx}'''

										# For layernorm(s), simply store the layer norm.
										if op_name.endswith('layernorm'						):
															_snake_case									=				'ln_1' if op_name.startswith('input'						) else 'ln_2'
															_snake_case									=				val

										# Transpose the QKV matrix.
										elif (
										    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
										) and weight_or_bias == "weight":
															# Insert a tensor of 1x1xDxD bias.
															_snake_case									=				torch.tril(torch.ones((n_positions, n_positions)       ,    dtype=torch.floataa						)						).view(
															    1       ,    1       ,    UpperCamelCase__       ,    UpperCamelCase__						)
															_snake_case									=				causal_mask

															# Insert a "dummy" tensor for masked_bias.
															_snake_case									=				torch.tensor(-1e4       ,    dtype=torch.floataa						)
															_snake_case									=				masked_bias

															_snake_case									=				fix_query_key_value_ordering(UpperCamelCase__       ,    UpperCamelCase__       ,    3       ,    UpperCamelCase__       ,    UpperCamelCase__						)
															# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
															_snake_case									=				out_val.transpose(0       ,    1						).contiguous()
															# Store.
															_snake_case									=				out_val

										# Transpose the bias.
										elif (
										    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
										) and weight_or_bias == "bias":
															_snake_case									=				fix_query_key_value_ordering(UpperCamelCase__       ,    UpperCamelCase__       ,    3       ,    UpperCamelCase__       ,    UpperCamelCase__						)
															# Store. No change of shape.
															_snake_case									=				out_val

										# Transpose the weights.
										elif weight_or_bias == "weight":
															_snake_case									=				megatron_to_transformers[op_name]
															_snake_case									=				val.transpose(0       ,    1						)

										# Copy the bias.
										elif weight_or_bias == "bias":
															_snake_case									=				megatron_to_transformers[op_name]
															_snake_case									=				val

    # DEBUG.
					assert config.n_layer == layer_idx + 1

					# The final layernorm.
					_snake_case									=				transformer['final_layernorm.weight']
					_snake_case									=				transformer['final_layernorm.bias']

					# For LM head, transformers' wants the matrix to weight embeddings.
					_snake_case									=				word_embeddings

					# It should be done!
					return output_state_dict





def 						lowerCamelCase__     (     )			-> List[str]:



					'''simple docstring'''




					_snake_case									=				argparse.ArgumentParser()
					parser.add_argument('--print-checkpoint-structure'       ,    action='store_true'						)
					parser.add_argument(
					    'path_to_checkpoint'       ,    type=UpperCamelCase__       ,    help='Path to the checkpoint file (.zip archive or direct .pt file)'       ,    )
					parser.add_argument(
					    '--config_file'       ,    default=''       ,    type=UpperCamelCase__       ,    help='An optional config json file describing the pre-trained model.'       ,    )
					_snake_case									=				parser.parse_args()

					# Extract the basename.
					_snake_case									=				os.path.dirname(args.path_to_checkpoint						)

					# Load the model.
					# the .zip is very optional, let's keep it for backward compatibility
					print(F'''Extracting PyTorch state dictionary from {args.path_to_checkpoint}'''						)
					if args.path_to_checkpoint.endswith('.zip'						):
										with zipfile.ZipFile(args.path_to_checkpoint       ,    'r'						) as checkpoint:
															with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'						) as pytorch_dict:
																				_snake_case									=				torch.load(UpperCamelCase__       ,    map_location='cpu'						)
					else:
										_snake_case									=				torch.load(args.path_to_checkpoint       ,    map_location='cpu'						)

					_snake_case									=				input_state_dict.get('args'       ,    UpperCamelCase__						)

					# Read the config, or default to the model released by NVIDIA.
					if args.config_file == "":
										if ds_args is not None:
															if ds_args.bias_gelu_fusion:
																				_snake_case									=				'gelu_fast'
															elif ds_args.openai_gelu:
																				_snake_case									=				'gelu_new'
															else:
																				_snake_case									=				'gelu'
										else:
															# in the very early days this used to be "gelu_new"
															_snake_case									=				'gelu_new'

										# Spell out all parameters in case the defaults change.
										_snake_case									=				GPTaConfig(
										    vocab_size=50_257       ,    n_positions=1_024       ,    n_embd=1_024       ,    n_layer=24       ,    n_head=16       ,    n_inner=4_096       ,    activation_function=UpperCamelCase__       ,    resid_pdrop=0.1       ,    embd_pdrop=0.1       ,    attn_pdrop=0.1       ,    layer_norm_epsilon=1e-5       ,    initializer_range=0.02       ,    summary_type='cls_index'       ,    summary_use_proj=UpperCamelCase__       ,    summary_activation=UpperCamelCase__       ,    summary_proj_to_labels=UpperCamelCase__       ,    summary_first_dropout=0.1       ,    scale_attn_weights=UpperCamelCase__       ,    use_cache=UpperCamelCase__       ,    bos_token_id=50_256       ,    eos_token_id=50_256       ,    )
					else:
										_snake_case									=				GPTaConfig.from_json_file(args.config_file						)

					_snake_case									=				['GPT2LMHeadModel']

					# Convert.
					print('Converting'						)
					_snake_case									=				convert_megatron_checkpoint(UpperCamelCase__       ,    UpperCamelCase__       ,    UpperCamelCase__						)

					# Print the structure of converted state dict.
					if args.print_checkpoint_structure:
										recursive_print(UpperCamelCase__       ,    UpperCamelCase__						)

					# Add tokenizer class info to config
					# see https://github.com/huggingface/transformers/issues/13906)
					if ds_args is not None:
										_snake_case									=				ds_args.tokenizer_type
										if tokenizer_type == "GPT2BPETokenizer":
															_snake_case									=				'gpt2'
										elif tokenizer_type == "PretrainedFromHF":
															_snake_case									=				ds_args.tokenizer_name_or_path
										else:
															raise ValueError(F'''Unrecognized tokenizer_type {tokenizer_type}'''						)
					else:
										_snake_case									=				'gpt2'

					_snake_case									=				AutoTokenizer.from_pretrained(UpperCamelCase__						)
					_snake_case									=				type(UpperCamelCase__						).__name__
					_snake_case									=				tokenizer_class

					# Store the config to file.
					print('Saving config'						)
					config.save_pretrained(UpperCamelCase__						)

					# Save tokenizer based on args
					print(F'''Adding {tokenizer_class} tokenizer files'''						)
					tokenizer.save_pretrained(UpperCamelCase__						)

					# Store the state_dict to file.
					_snake_case									=				os.path.join(UpperCamelCase__       ,    'pytorch_model.bin'						)
					print(F'''Saving checkpoint to \"{output_checkpoint_file}\"'''						)
					torch.save(UpperCamelCase__       ,    UpperCamelCase__						)


####################################################################################################

if __name__ == "__main__":
					main()

####################################################################################################










from typing import TYPE_CHECKING

from ...utils import _LazyModule


UpperCAmelCase_      							=     {"""tokenization_byt5""": ["""ByT5Tokenizer"""]}


if TYPE_CHECKING:
					from .tokenization_byta import ByTaTokenizer
else:
					import sys

					UpperCAmelCase_      							=     _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



