






import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig
def    snake_case_		(  lowerCAmelCase_							:    int							,							lowerCAmelCase_							:    str							,							lowerCAmelCase_							:    int=0   ):
							# Format the message.
							if name is None:
														__lowercase   :				Union[str, Any]						      =  None
							else:
														__lowercase   :				Any						      =  "." * max(0							,							spaces - 2   ) + "# {:" + str(50 - spaces   ) + "s}"
														__lowercase   :				Dict						      =  fmt.format(__snake_case   )

							# Print and recurse (if needed).
							if isinstance(__snake_case							,							__snake_case   ):
														if msg is not None:
																					print(__snake_case   )
														for k in val.keys():
																					recursive_print(__snake_case							,							val[k]							,							spaces + 2   )
							elif isinstance(__snake_case							,							torch.Tensor   ):
														print(__snake_case							,							""":"""							,							val.size()   )
							else:
														print(__snake_case							,							""":"""							,							__snake_case   )
def    snake_case_		(  lowerCAmelCase_							:    Union[str, Any]							,							lowerCAmelCase_							:    str							,							lowerCAmelCase_							:    List[str]							,							lowerCAmelCase_							:    List[str]							,							lowerCAmelCase_							:    str   ):
							# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
							# for compatibility with later versions of NVIDIA Megatron-LM.
							# The inverse operation is performed inside Megatron-LM to read checkpoints:
							# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
							# If param is the weight tensor of the self-attention block, the returned tensor
							# will have to be transposed one more time to be read by HuggingFace GPT2.
							__lowercase   :				Optional[int]						      =  param.size()
							if checkpoint_version == 1.0:
														# version 1.0 stores [num_heads * hidden_size * num_splits, :]
														__lowercase   :				Optional[int]						      =  (num_heads, hidden_size, num_splits) + input_shape[1:]
														__lowercase   :				List[str]						      =  param.view(*__snake_case   )
														__lowercase   :				int						      =  param.transpose(0							,							2   )
														__lowercase   :				List[Any]						      =  param.transpose(1							,							2   ).contiguous()
							elif checkpoint_version >= 2.0:
														# other versions store [num_heads * num_splits * hidden_size, :]
														__lowercase   :				Union[str, Any]						      =  (num_heads, num_splits, hidden_size) + input_shape[1:]
														__lowercase   :				str						      =  param.view(*__snake_case   )
														__lowercase   :				Optional[int]						      =  param.transpose(0							,							1   ).contiguous()
							__lowercase   :				Union[str, Any]						      =  param.view(*__snake_case   )
							return param
def    snake_case_		(  lowerCAmelCase_							:    Tuple							,							lowerCAmelCase_							:    Union[str, Any]							,							lowerCAmelCase_							:    Optional[int]   ):
							# The converted output model.
							__lowercase   :				Tuple						      =  {}

							# old versions did not store training args
							__lowercase   :				Dict						      =  input_state_dict.get("""args"""							,							__snake_case   )
							if ds_args is not None:
														# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
														# from pprint import pprint
														# pprint(vars(ds_args))

														__lowercase   :				str						      =  ds_args.padded_vocab_size
														__lowercase   :				List[Any]						      =  ds_args.max_position_embeddings
														__lowercase   :				int						      =  ds_args.hidden_size
														__lowercase   :				Union[str, Any]						      =  ds_args.num_layers
														__lowercase   :				int						      =  ds_args.num_attention_heads
														__lowercase   :				Union[str, Any]						      =  ds_args.ffn_hidden_size
														# pprint(config)

							# The number of heads.
							__lowercase   :				Tuple						      =  config.n_head
							# The hidden_size per head.
							__lowercase   :				List[str]						      =  config.n_embd // config.n_head
							# Megatron-LM checkpoint version
							if "checkpoint_version" in input_state_dict.keys():
														__lowercase   :				Any						      =  input_state_dict["checkpoint_version"]
							else:
														__lowercase   :				Tuple						      =  0.0

							# The model.
							__lowercase   :				Optional[Any]						      =  input_state_dict["model"]
							# The language model.
							__lowercase   :				str						      =  model["language_model"]
							# The embeddings.
							__lowercase   :				Union[str, Any]						      =  lm["embedding"]

							# The word embeddings.
							__lowercase   :				Union[str, Any]						      =  embeddings["word_embeddings"]["weight"]
							# Truncate the embedding table to vocab_size rows.
							__lowercase   :				Union[str, Any]						      =  word_embeddings[: config.vocab_size, :]
							__lowercase   :				Tuple						      =  word_embeddings

							# The position embeddings.
							__lowercase   :				Dict						      =  embeddings["position_embeddings"]["weight"]
							# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
							__lowercase   :				Tuple						      =  pos_embeddings.size(0   )
							if n_positions != config.n_positions:
														raise ValueError(
														    F"pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match"   )
							# Store the position embeddings.
							__lowercase   :				Optional[Any]						      =  pos_embeddings

							# The transformer.
							__lowercase   :				Optional[int]						      =  lm["transformer"] if "transformer" in lm.keys() else lm["encoder"]

							# The regex to extract layer names.
							__lowercase   :				List[str]						      =  re.compile(r"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"""   )

							# The simple map of names for "automated" rules.
							__lowercase   :				Any						      =  {
							    "attention.dense": ".attn.c_proj.",
							    "self_attention.dense": ".attn.c_proj.",
							    "mlp.dense_h_to_4h": ".mlp.c_fc.",
							    "mlp.dense_4h_to_h": ".mlp.c_proj.",
							}

							# Extract the layers.
							for key, val in transformer.items():
														# Match the name.
														__lowercase   :				str						      =  layer_re.match(__snake_case   )

														# Stop if that's not a layer
														if m is None:
																					break

														# The index of the layer.
														__lowercase   :				Union[str, Any]						      =  int(m.group(1   )   )
														# The name of the operation.
														__lowercase   :				int						      =  m.group(2   )
														# Is it a weight or a bias?
														__lowercase   :				str						      =  m.group(3   )

														# The name of the layer.
														__lowercase   :				Union[str, Any]						      =  F"transformer.h.{layer_idx}"

														# For layernorm(s), simply store the layer norm.
														if op_name.endswith("""layernorm"""   ):
																					__lowercase   :				int						      =  "ln_1" if op_name.startswith("""input"""   ) else "ln_2"
																					__lowercase   :				Any						      =  val

														# Transpose the QKV matrix.
														elif (
														    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
														) and weight_or_bias == "weight":
																					# Insert a tensor of 1x1xDxD bias.
																					__lowercase   :				Optional[Any]						      =  torch.tril(torch.ones((n_positions, n_positions)							,							dtype=torch.floataa   )   ).view(
																					    1							,							1							,							__snake_case							,							__snake_case   )
																					__lowercase   :				Dict						      =  causal_mask

																					# Insert a "dummy" tensor for masked_bias.
																					__lowercase   :				Any						      =  torch.tensor(-1e4							,							dtype=torch.floataa   )
																					__lowercase   :				List[Any]						      =  masked_bias

																					__lowercase   :				List[str]						      =  fix_query_key_value_ordering(__snake_case							,							__snake_case							,							3							,							__snake_case							,							__snake_case   )
																					# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
																					__lowercase   :				Union[str, Any]						      =  out_val.transpose(0							,							1   ).contiguous()
																					# Store.
																					__lowercase   :				str						      =  out_val

														# Transpose the bias.
														elif (
														    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
														) and weight_or_bias == "bias":
																					__lowercase   :				Tuple						      =  fix_query_key_value_ordering(__snake_case							,							__snake_case							,							3							,							__snake_case							,							__snake_case   )
																					# Store. No change of shape.
																					__lowercase   :				Union[str, Any]						      =  out_val

														# Transpose the weights.
														elif weight_or_bias == "weight":
																					__lowercase   :				Any						      =  megatron_to_transformers[op_name]
																					__lowercase   :				int						      =  val.transpose(0							,							1   )

														# Copy the bias.
														elif weight_or_bias == "bias":
																					__lowercase   :				str						      =  megatron_to_transformers[op_name]
																					__lowercase   :				Optional[Any]						      =  val

    # DEBUG.
							assert config.n_layer == layer_idx + 1

							# The final layernorm.
							__lowercase   :				Dict						      =  transformer["final_layernorm.weight"]
							__lowercase   :				Union[str, Any]						      =  transformer["final_layernorm.bias"]

							# For LM head, transformers' wants the matrix to weight embeddings.
							__lowercase   :				List[Any]						      =  word_embeddings

							# It should be done!
							return output_state_dict
def    snake_case_		(  ):
							# Create the argument parser.
							__lowercase   :				Dict						      =  argparse.ArgumentParser()
							parser.add_argument("""--print-checkpoint-structure"""							,							action="""store_true"""   )
							parser.add_argument(
							    """path_to_checkpoint"""							,							type=__snake_case							,							help="""Path to the checkpoint file (.zip archive or direct .pt file)"""							,							)
							parser.add_argument(
							    """--config_file"""							,							default=""""""							,							type=__snake_case							,							help="""An optional config json file describing the pre-trained model."""							,							)
							__lowercase   :				List[str]						      =  parser.parse_args()

							# Extract the basename.
							__lowercase   :				str						      =  os.path.dirname(args.path_to_checkpoint   )

							# Load the model.
							# the .zip is very optional, let's keep it for backward compatibility
							print(F"Extracting PyTorch state dictionary from {args.path_to_checkpoint}"   )
							if args.path_to_checkpoint.endswith(""".zip"""   ):
														with zipfile.ZipFile(args.path_to_checkpoint							,							"""r"""   ) as checkpoint:
																					with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt"""   ) as pytorch_dict:
																												__lowercase   :				List[str]						      =  torch.load(__snake_case							,							map_location="""cpu"""   )
							else:
														__lowercase   :				Optional[int]						      =  torch.load(args.path_to_checkpoint							,							map_location="""cpu"""   )

							__lowercase   :				Optional[int]						      =  input_state_dict.get("""args"""							,							__snake_case   )

							# Read the config, or default to the model released by NVIDIA.
							if args.config_file == "":
														if ds_args is not None:
																					if ds_args.bias_gelu_fusion:
																												__lowercase   :				Tuple						      =  "gelu_fast"
																					elif ds_args.openai_gelu:
																												__lowercase   :				List[Any]						      =  "gelu_new"
																					else:
																												__lowercase   :				List[str]						      =  "gelu"
														else:
																					# in the very early days this used to be "gelu_new"
																					__lowercase   :				Any						      =  "gelu_new"

														# Spell out all parameters in case the defaults change.
														__lowercase   :				List[str]						      =  GPTaConfig(
														    vocab_size=50257							,							n_positions=1024							,							n_embd=1024							,							n_layer=24							,							n_head=16							,							n_inner=4096							,							activation_function=__snake_case							,							resid_pdrop=0.1							,							embd_pdrop=0.1							,							attn_pdrop=0.1							,							layer_norm_epsilon=1e-5							,							initializer_range=0.02							,							summary_type="""cls_index"""							,							summary_use_proj=__snake_case							,							summary_activation=__snake_case							,							summary_proj_to_labels=__snake_case							,							summary_first_dropout=0.1							,							scale_attn_weights=__snake_case							,							use_cache=__snake_case							,							bos_token_id=50256							,							eos_token_id=50256							,							)
							else:
														__lowercase   :				Any						      =  GPTaConfig.from_json_file(args.config_file   )

							__lowercase   :				List[str]						      =  ["GPT2LMHeadModel"]

							# Convert.
							print("""Converting"""   )
							__lowercase   :				List[str]						      =  convert_megatron_checkpoint(__snake_case							,							__snake_case							,							__snake_case   )

							# Print the structure of converted state dict.
							if args.print_checkpoint_structure:
														recursive_print(__snake_case							,							__snake_case   )

							# Add tokenizer class info to config
							# see https://github.com/huggingface/transformers/issues/13906)
							if ds_args is not None:
														__lowercase   :				str						      =  ds_args.tokenizer_type
														if tokenizer_type == "GPT2BPETokenizer":
																					__lowercase   :				Union[str, Any]						      =  "gpt2"
														elif tokenizer_type == "PretrainedFromHF":
																					__lowercase   :				Optional[int]						      =  ds_args.tokenizer_name_or_path
														else:
																					raise ValueError(F"Unrecognized tokenizer_type {tokenizer_type}"   )
							else:
														__lowercase   :				str						      =  "gpt2"

							__lowercase   :				int						      =  AutoTokenizer.from_pretrained(__snake_case   )
							__lowercase   :				Optional[Any]						      =  type(__snake_case   ).__name__
							__lowercase   :				Union[str, Any]						      =  tokenizer_class

							# Store the config to file.
							print("""Saving config"""   )
							config.save_pretrained(__snake_case   )

							# Save tokenizer based on args
							print(F"Adding {tokenizer_class} tokenizer files"   )
							tokenizer.save_pretrained(__snake_case   )

							# Store the state_dict to file.
							__lowercase   :				Optional[Any]						      =  os.path.join(__snake_case							,							"""pytorch_model.bin"""   )
							print(F"Saving checkpoint to \"{output_checkpoint_file}\""   )
							torch.save(__snake_case							,							__snake_case   )


####################################################################################################

if __name__ == "__main__":
	main()

####################################################################################################

"""simple docstring"""




def     __magic_name__      ( __snake_case							:			int	,					__snake_case							:			int    )					->							Any:
       if b == 0:
              return 1
       if (b % 2) == 0:
              return actual_power(__snake_case	,					int(b / 2    )    ) * actual_power(__snake_case	,					int(b / 2    )    )
       else:
              return a * actual_power(__snake_case	,					int(b / 2    )    ) * actual_power(__snake_case	,					int(b / 2    )    )


def     __magic_name__      ( __snake_case							:			int	,					__snake_case							:			int    )					->							float:
       if b < 0:
              return 1 / actual_power(__snake_case	,					__snake_case    )
       return actual_power(__snake_case	,					__snake_case    )


if __name__ == "__main__":
   print(power(-2, -3))


