


# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig





def 						_lowerCAmelCase							(     _lowerCAmelCase   ,						_lowerCAmelCase   ,						_lowerCAmelCase=0			)    ->						List[Any]:


						'''simple docstring'''






						if name is None:
												__snake_case				=		None
						else:
												__snake_case				=		"." * max(0   ,						spaces - 2			) + "# {:" + str(50 - spaces			) + "s}"
												__snake_case				=		fmt.format(_lowerCAmelCase			)

						# Print and recurse (if needed).
						if isinstance(_lowerCAmelCase   ,						_lowerCAmelCase			):
												if msg is not None:
																		print(_lowerCAmelCase			)
												for k in val.keys():
																		recursive_print(_lowerCAmelCase   ,						val[k]   ,						spaces + 2			)
						elif isinstance(_lowerCAmelCase   ,						torch.Tensor			):
												print(_lowerCAmelCase   ,						":"   ,						val.size()			)
						else:
												print(_lowerCAmelCase   ,						":"   ,						_lowerCAmelCase			)





def 						_lowerCAmelCase							(     _lowerCAmelCase   ,						_lowerCAmelCase   ,						_lowerCAmelCase   ,						_lowerCAmelCase   ,						_lowerCAmelCase			)    ->						Optional[int]:


						'''simple docstring'''






						__snake_case				=		param.size()
						if checkpoint_version == 1.0:
												# version 1.0 stores [num_heads * hidden_size * num_splits, :]
												__snake_case				=		(num_heads, hidden_size, num_splits) + input_shape[1:]
												__snake_case				=		param.view(*_lowerCAmelCase			)
												__snake_case				=		param.transpose(0   ,						2			)
												__snake_case				=		param.transpose(1   ,						2			).contiguous()
						elif checkpoint_version >= 2.0:
												# other versions store [num_heads * num_splits * hidden_size, :]
												__snake_case				=		(num_heads, num_splits, hidden_size) + input_shape[1:]
												__snake_case				=		param.view(*_lowerCAmelCase			)
												__snake_case				=		param.transpose(0   ,						1			).contiguous()
						__snake_case				=		param.view(*_lowerCAmelCase			)
						return param





def 						_lowerCAmelCase							(     _lowerCAmelCase   ,						_lowerCAmelCase   ,						_lowerCAmelCase			)    ->						Union[str, Any]:


						'''simple docstring'''






						__snake_case				=		{}

						# old versions did not store training args
						__snake_case				=		input_state_dict.get("args"   ,						_lowerCAmelCase			)
						if ds_args is not None:
												# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
												# from pprint import pprint
												# pprint(vars(ds_args))

												__snake_case				=		ds_args.padded_vocab_size
												__snake_case				=		ds_args.max_position_embeddings
												__snake_case				=		ds_args.hidden_size
												__snake_case				=		ds_args.num_layers
												__snake_case				=		ds_args.num_attention_heads
												__snake_case				=		ds_args.ffn_hidden_size
												# pprint(config)

						# The number of heads.
						__snake_case				=		config.n_head
						# The hidden_size per head.
						__snake_case				=		config.n_embd // config.n_head
						# Megatron-LM checkpoint version
						if "checkpoint_version" in input_state_dict.keys():
												__snake_case				=		input_state_dict["checkpoint_version"]
						else:
												__snake_case				=		0.0

						# The model.
						__snake_case				=		input_state_dict["model"]
						# The language model.
						__snake_case				=		model["language_model"]
						# The embeddings.
						__snake_case				=		lm["embedding"]

						# The word embeddings.
						__snake_case				=		embeddings["word_embeddings"]["weight"]
						# Truncate the embedding table to vocab_size rows.
						__snake_case				=		word_embeddings[: config.vocab_size, :]
						__snake_case				=		word_embeddings

						# The position embeddings.
						__snake_case				=		embeddings["position_embeddings"]["weight"]
						# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
						__snake_case				=		pos_embeddings.size(0			)
						if n_positions != config.n_positions:
												raise ValueError(
												    F'''pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'''			)
						# Store the position embeddings.
						__snake_case				=		pos_embeddings

						# The transformer.
						__snake_case				=		lm["transformer"] if "transformer" in lm.keys() else lm["encoder"]

						# The regex to extract layer names.
						__snake_case				=		re.compile(r"layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"			)

						# The simple map of names for "automated" rules.
						__snake_case				=		{
						    "attention.dense": ".attn.c_proj.",
						    "self_attention.dense": ".attn.c_proj.",
						    "mlp.dense_h_to_4h": ".mlp.c_fc.",
						    "mlp.dense_4h_to_h": ".mlp.c_proj.",
						}

						# Extract the layers.
						for key, val in transformer.items():
												# Match the name.
												__snake_case				=		layer_re.match(_lowerCAmelCase			)

												# Stop if that's not a layer
												if m is None:
																		break

												# The index of the layer.
												__snake_case				=		int(m.group(1			)			)
												# The name of the operation.
												__snake_case				=		m.group(2			)
												# Is it a weight or a bias?
												__snake_case				=		m.group(3			)

												# The name of the layer.
												__snake_case				=		F'''transformer.h.{layer_idx}'''

												# For layernorm(s), simply store the layer norm.
												if op_name.endswith("layernorm"			):
																		__snake_case				=		"ln_1" if op_name.startswith("input"			) else "ln_2"
																		__snake_case				=		val

												# Transpose the QKV matrix.
												elif (
												    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
												) and weight_or_bias == "weight":
																		# Insert a tensor of 1x1xDxD bias.
																		__snake_case				=		torch.tril(torch.ones((n_positions, n_positions)   ,						dtype=torch.floataa			)			).view(
																		    1   ,						1   ,						_lowerCAmelCase   ,						_lowerCAmelCase			)
																		__snake_case				=		causal_mask

																		# Insert a "dummy" tensor for masked_bias.
																		__snake_case				=		torch.tensor(-1E4   ,						dtype=torch.floataa			)
																		__snake_case				=		masked_bias

																		__snake_case				=		fix_query_key_value_ordering(_lowerCAmelCase   ,						_lowerCAmelCase   ,						3   ,						_lowerCAmelCase   ,						_lowerCAmelCase			)
																		# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
																		__snake_case				=		out_val.transpose(0   ,						1			).contiguous()
																		# Store.
																		__snake_case				=		out_val

												# Transpose the bias.
												elif (
												    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
												) and weight_or_bias == "bias":
																		__snake_case				=		fix_query_key_value_ordering(_lowerCAmelCase   ,						_lowerCAmelCase   ,						3   ,						_lowerCAmelCase   ,						_lowerCAmelCase			)
																		# Store. No change of shape.
																		__snake_case				=		out_val

												# Transpose the weights.
												elif weight_or_bias == "weight":
																		__snake_case				=		megatron_to_transformers[op_name]
																		__snake_case				=		val.transpose(0   ,						1			)

												# Copy the bias.
												elif weight_or_bias == "bias":
																		__snake_case				=		megatron_to_transformers[op_name]
																		__snake_case				=		val

    # DEBUG.
						assert config.n_layer == layer_idx + 1

						# The final layernorm.
						__snake_case				=		transformer["final_layernorm.weight"]
						__snake_case				=		transformer["final_layernorm.bias"]

						# For LM head, transformers' wants the matrix to weight embeddings.
						__snake_case				=		word_embeddings

						# It should be done!
						return output_state_dict





def 						_lowerCAmelCase							(     )    ->						Any:


						'''simple docstring'''






						__snake_case				=		argparse.ArgumentParser()
						parser.add_argument("--print-checkpoint-structure"   ,						action="store_true"			)
						parser.add_argument(
						    "path_to_checkpoint"   ,						type=_lowerCAmelCase   ,						help="Path to the checkpoint file (.zip archive or direct .pt file)"   ,						)
						parser.add_argument(
						    "--config_file"   ,						default=""   ,						type=_lowerCAmelCase   ,						help="An optional config json file describing the pre-trained model."   ,						)
						__snake_case				=		parser.parse_args()

						# Extract the basename.
						__snake_case				=		os.path.dirname(args.path_to_checkpoint			)

						# Load the model.
						# the .zip is very optional, let's keep it for backward compatibility
						print(F'''Extracting PyTorch state dictionary from {args.path_to_checkpoint}'''			)
						if args.path_to_checkpoint.endswith(".zip"			):
												with zipfile.ZipFile(args.path_to_checkpoint   ,						"r"			) as checkpoint:
																		with checkpoint.open("release/mp_rank_00/model_optim_rng.pt"			) as pytorch_dict:
																								__snake_case				=		torch.load(_lowerCAmelCase   ,						map_location="cpu"			)
						else:
												__snake_case				=		torch.load(args.path_to_checkpoint   ,						map_location="cpu"			)

						__snake_case				=		input_state_dict.get("args"   ,						_lowerCAmelCase			)

						# Read the config, or default to the model released by NVIDIA.
						if args.config_file == "":
												if ds_args is not None:
																		if ds_args.bias_gelu_fusion:
																								__snake_case				=		"gelu_fast"
																		elif ds_args.openai_gelu:
																								__snake_case				=		"gelu_new"
																		else:
																								__snake_case				=		"gelu"
												else:
																		# in the very early days this used to be "gelu_new"
																		__snake_case				=		"gelu_new"

												# Spell out all parameters in case the defaults change.
												__snake_case				=		GPTaConfig(
												    vocab_size=5_0257   ,						n_positions=1024   ,						n_embd=1024   ,						n_layer=24   ,						n_head=16   ,						n_inner=4096   ,						activation_function=_lowerCAmelCase   ,						resid_pdrop=0.1   ,						embd_pdrop=0.1   ,						attn_pdrop=0.1   ,						layer_norm_epsilon=1E-5   ,						initializer_range=0.02   ,						summary_type="cls_index"   ,						summary_use_proj=_lowerCAmelCase   ,						summary_activation=_lowerCAmelCase   ,						summary_proj_to_labels=_lowerCAmelCase   ,						summary_first_dropout=0.1   ,						scale_attn_weights=_lowerCAmelCase   ,						use_cache=_lowerCAmelCase   ,						bos_token_id=5_0256   ,						eos_token_id=5_0256   ,						)
						else:
												__snake_case				=		GPTaConfig.from_json_file(args.config_file			)

						__snake_case				=		["GPT2LMHeadModel"]

						# Convert.
						print("Converting"			)
						__snake_case				=		convert_megatron_checkpoint(_lowerCAmelCase   ,						_lowerCAmelCase   ,						_lowerCAmelCase			)

						# Print the structure of converted state dict.
						if args.print_checkpoint_structure:
												recursive_print(_lowerCAmelCase   ,						_lowerCAmelCase			)

						# Add tokenizer class info to config
						# see https://github.com/huggingface/transformers/issues/13906)
						if ds_args is not None:
												__snake_case				=		ds_args.tokenizer_type
												if tokenizer_type == "GPT2BPETokenizer":
																		__snake_case				=		"gpt2"
												elif tokenizer_type == "PretrainedFromHF":
																		__snake_case				=		ds_args.tokenizer_name_or_path
												else:
																		raise ValueError(F'''Unrecognized tokenizer_type {tokenizer_type}'''			)
						else:
												__snake_case				=		"gpt2"

						__snake_case				=		AutoTokenizer.from_pretrained(_lowerCAmelCase			)
						__snake_case				=		type(_lowerCAmelCase			).__name__
						__snake_case				=		tokenizer_class

						# Store the config to file.
						print("Saving config"			)
						config.save_pretrained(_lowerCAmelCase			)

						# Save tokenizer based on args
						print(F'''Adding {tokenizer_class} tokenizer files'''			)
						tokenizer.save_pretrained(_lowerCAmelCase			)

						# Store the state_dict to file.
						__snake_case				=		os.path.join(_lowerCAmelCase   ,						"pytorch_model.bin"			)
						print(F'''Saving checkpoint to "{output_checkpoint_file}"'''			)
						torch.save(_lowerCAmelCase   ,						_lowerCAmelCase			)


####################################################################################################

if __name__ == "__main__":
	main()

####################################################################################################









from __future__ import annotations






class      UpperCamelCase:


					def __init__(    self						:		Optional[int]  ,     SCREAMING_SNAKE_CASE						:		str  ,     SCREAMING_SNAKE_CASE						:		str    )      ->						str:




											'''simple docstring'''

											__snake_case		,       __snake_case				=		text, pattern
											__snake_case		,       __snake_case				=		len(SCREAMING_SNAKE_CASE    ), len(SCREAMING_SNAKE_CASE    )


					def 				SCREAMING_SNAKE_CASE_	(    self						:		Optional[Any]  ,     SCREAMING_SNAKE_CASE						:		str    )      ->						int:




											'''simple docstring'''

											for i in range(self.patLen - 1  ,     -1  ,     -1    ):
																	if char == self.pattern[i]:
																							return i
											return -1


					def 				SCREAMING_SNAKE_CASE_	(    self						:		int  ,     SCREAMING_SNAKE_CASE						:		int    )      ->						int:




											'''simple docstring'''

											for i in range(self.patLen - 1  ,     -1  ,     -1    ):
																	if self.pattern[i] != self.text[current_pos + i]:
																							return current_pos + i
											return -1






					def 				SCREAMING_SNAKE_CASE_	(    self						:		List[Any]    )      ->						list[int]:




											'''simple docstring'''

											__snake_case				=		[]
											for i in range(self.textLen - self.patLen + 1    ):
																	__snake_case				=		self.mismatch_in_text(SCREAMING_SNAKE_CASE    )
																	if mismatch_index == -1:
																							positions.append(SCREAMING_SNAKE_CASE    )
																	else:
																							__snake_case				=		self.match_in_pattern(self.text[mismatch_index]    )
																							__snake_case				=		(
																							    mismatch_index - match_index
																							)  # shifting index lgtm [py/multiple-definition]
											return positions




A				:       str      =  'ABAABA'
A				:       Optional[Any]      =  'AB'
A				:       Any      =  BoyerMooreSearch(text, pattern)
A				:       Tuple      =  bms.bad_character_heuristic()

if len(positions) == 0:
	print('No match found')
else:
	print('Pattern found in following positions: ')
	print(positions)





