






"""simple docstring"""


# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig


def        UpperCAmelCase      (					a__     ,      a__     ,      a__=0 ):





						'''simple docstring'''






						if name is None:
												lowerCAmelCase   :List[Any]						=			None
						else:
												lowerCAmelCase   :Dict						=			'.' * max(0     ,      spaces - 2 ) + '# {:' + str(50 - spaces ) + 's}'
												lowerCAmelCase   :Any						=			fmt.format(a__ )

						# Print and recurse (if needed).
						if isinstance(a__     ,      a__ ):
												if msg is not None:
																		print(a__ )
												for k in val.keys():
																		recursive_print(a__     ,      val[k]     ,      spaces + 2 )
						elif isinstance(a__     ,      torch.Tensor ):
												print(a__     ,      ':'     ,      val.size() )
						else:
												print(a__     ,      ':'     ,      a__ )


def        UpperCAmelCase      (					a__     ,      a__     ,      a__     ,      a__     ,      a__ ):





						'''simple docstring'''






						lowerCAmelCase   :Union[str, Any]						=			param.size()
						if checkpoint_version == 1.0:
												# version 1.0 stores [num_heads * hidden_size * num_splits, :]
												lowerCAmelCase   :Union[str, Any]						=			(num_heads, hidden_size, num_splits) + input_shape[1:]
												lowerCAmelCase   :List[str]						=			param.view(*a__ )
												lowerCAmelCase   :Tuple						=			param.transpose(0     ,      2 )
												lowerCAmelCase   :int						=			param.transpose(1     ,      2 ).contiguous()
						elif checkpoint_version >= 2.0:
												# other versions store [num_heads * num_splits * hidden_size, :]
												lowerCAmelCase   :str						=			(num_heads, num_splits, hidden_size) + input_shape[1:]
												lowerCAmelCase   :Any						=			param.view(*a__ )
												lowerCAmelCase   :Union[str, Any]						=			param.transpose(0     ,      1 ).contiguous()
						lowerCAmelCase   :Optional[int]						=			param.view(*a__ )
						return param


def        UpperCAmelCase      (					a__     ,      a__     ,      a__ ):





						'''simple docstring'''






						lowerCAmelCase   :Any						=			{}

						# old versions did not store training args
						lowerCAmelCase   :Optional[int]						=			input_state_dict.get('args'     ,      a__ )
						if ds_args is not None:
												# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
												# from pprint import pprint
												# pprint(vars(ds_args))

												lowerCAmelCase   :Union[str, Any]						=			ds_args.padded_vocab_size
												lowerCAmelCase   :str						=			ds_args.max_position_embeddings
												lowerCAmelCase   :Optional[Any]						=			ds_args.hidden_size
												lowerCAmelCase   :Optional[Any]						=			ds_args.num_layers
												lowerCAmelCase   :Optional[Any]						=			ds_args.num_attention_heads
												lowerCAmelCase   :Tuple						=			ds_args.ffn_hidden_size
												# pprint(config)

						# The number of heads.
						lowerCAmelCase   :Optional[int]						=			config.n_head
						# The hidden_size per head.
						lowerCAmelCase   :List[Any]						=			config.n_embd // config.n_head
						# Megatron-LM checkpoint version
						if "checkpoint_version" in input_state_dict.keys():
												lowerCAmelCase   :Union[str, Any]						=			input_state_dict['checkpoint_version']
						else:
												lowerCAmelCase   :Optional[int]						=			0.0

						# The model.
						lowerCAmelCase   :List[str]						=			input_state_dict['model']
						# The language model.
						lowerCAmelCase   :Any						=			model['language_model']
						# The embeddings.
						lowerCAmelCase   :List[str]						=			lm['embedding']

						# The word embeddings.
						lowerCAmelCase   :Dict						=			embeddings['word_embeddings']['weight']
						# Truncate the embedding table to vocab_size rows.
						lowerCAmelCase   :int						=			word_embeddings[: config.vocab_size, :]
						lowerCAmelCase   :Union[str, Any]						=			word_embeddings

						# The position embeddings.
						lowerCAmelCase   :List[Any]						=			embeddings['position_embeddings']['weight']
						# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
						lowerCAmelCase   :int						=			pos_embeddings.size(0 )
						if n_positions != config.n_positions:
												raise ValueError(
												    F"""pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match""" )
						# Store the position embeddings.
						lowerCAmelCase   :Dict						=			pos_embeddings

						# The transformer.
						lowerCAmelCase   :Union[str, Any]						=			lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

						# The regex to extract layer names.
						lowerCAmelCase   :Any						=			re.compile(R'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)' )

						# The simple map of names for "automated" rules.
						lowerCAmelCase   :int						=			{
						    'attention.dense': '.attn.c_proj.',
						    'self_attention.dense': '.attn.c_proj.',
						    'mlp.dense_h_to_4h': '.mlp.c_fc.',
						    'mlp.dense_4h_to_h': '.mlp.c_proj.',
						}

						# Extract the layers.
						for key, val in transformer.items():
												# Match the name.
												lowerCAmelCase   :Union[str, Any]						=			layer_re.match(a__ )

												# Stop if that's not a layer
												if m is None:
																		break

												# The index of the layer.
												lowerCAmelCase   :str						=			int(m.group(1 ) )
												# The name of the operation.
												lowerCAmelCase   :Optional[int]						=			m.group(2 )
												# Is it a weight or a bias?
												lowerCAmelCase   :Union[str, Any]						=			m.group(3 )

												# The name of the layer.
												lowerCAmelCase   :Optional[int]						=			F"""transformer.h.{layer_idx}"""

												# For layernorm(s), simply store the layer norm.
												if op_name.endswith('layernorm' ):
																		lowerCAmelCase   :List[str]						=			'ln_1' if op_name.startswith('input' ) else 'ln_2'
																		lowerCAmelCase   :Any						=			val

												# Transpose the QKV matrix.
												elif (
												    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
												) and weight_or_bias == "weight":
																		# Insert a tensor of 1x1xDxD bias.
																		lowerCAmelCase   :Optional[int]						=			torch.tril(torch.ones((n_positions, n_positions)     ,      dtype=torch.floataa ) ).view(
																		    1     ,      1     ,      a__     ,      a__ )
																		lowerCAmelCase   :Dict						=			causal_mask

																		# Insert a "dummy" tensor for masked_bias.
																		lowerCAmelCase   :Tuple						=			torch.tensor(-1e4     ,      dtype=torch.floataa )
																		lowerCAmelCase   :Optional[Any]						=			masked_bias

																		lowerCAmelCase   :Optional[int]						=			fix_query_key_value_ordering(a__     ,      a__     ,      3     ,      a__     ,      a__ )
																		# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
																		lowerCAmelCase   :Optional[int]						=			out_val.transpose(0     ,      1 ).contiguous()
																		# Store.
																		lowerCAmelCase   :List[str]						=			out_val

												# Transpose the bias.
												elif (
												    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
												) and weight_or_bias == "bias":
																		lowerCAmelCase   :int						=			fix_query_key_value_ordering(a__     ,      a__     ,      3     ,      a__     ,      a__ )
																		# Store. No change of shape.
																		lowerCAmelCase   :Any						=			out_val

												# Transpose the weights.
												elif weight_or_bias == "weight":
																		lowerCAmelCase   :Union[str, Any]						=			megatron_to_transformers[op_name]
																		lowerCAmelCase   :Dict						=			val.transpose(0     ,      1 )

												# Copy the bias.
												elif weight_or_bias == "bias":
																		lowerCAmelCase   :Optional[int]						=			megatron_to_transformers[op_name]
																		lowerCAmelCase   :Optional[int]						=			val

    # DEBUG.
						assert config.n_layer == layer_idx + 1

						# The final layernorm.
						lowerCAmelCase   :List[str]						=			transformer['final_layernorm.weight']
						lowerCAmelCase   :Optional[Any]						=			transformer['final_layernorm.bias']

						# For LM head, transformers' wants the matrix to weight embeddings.
						lowerCAmelCase   :Optional[int]						=			word_embeddings

						# It should be done!
						return output_state_dict


def        UpperCAmelCase      (					):





						'''simple docstring'''






						lowerCAmelCase   :List[Any]						=			argparse.ArgumentParser()
						parser.add_argument('--print-checkpoint-structure'     ,      action='store_true' )
						parser.add_argument(
						    'path_to_checkpoint'     ,      type=a__     ,      help='Path to the checkpoint file (.zip archive or direct .pt file)'     ,      )
						parser.add_argument(
						    '--config_file'     ,      default=''     ,      type=a__     ,      help='An optional config json file describing the pre-trained model.'     ,      )
						lowerCAmelCase   :List[str]						=			parser.parse_args()

						# Extract the basename.
						lowerCAmelCase   :List[Any]						=			os.path.dirname(args.path_to_checkpoint )

						# Load the model.
						# the .zip is very optional, let's keep it for backward compatibility
						print(F"""Extracting PyTorch state dictionary from {args.path_to_checkpoint}""" )
						if args.path_to_checkpoint.endswith('.zip' ):
												with zipfile.ZipFile(args.path_to_checkpoint     ,      'r' ) as checkpoint:
																		with checkpoint.open('release/mp_rank_00/model_optim_rng.pt' ) as pytorch_dict:
																								lowerCAmelCase   :List[Any]						=			torch.load(a__     ,      map_location='cpu' )
						else:
												lowerCAmelCase   :str						=			torch.load(args.path_to_checkpoint     ,      map_location='cpu' )

						lowerCAmelCase   :Tuple						=			input_state_dict.get('args'     ,      a__ )

						# Read the config, or default to the model released by NVIDIA.
						if args.config_file == "":
												if ds_args is not None:
																		if ds_args.bias_gelu_fusion:
																								lowerCAmelCase   :List[Any]						=			'gelu_fast'
																		elif ds_args.openai_gelu:
																								lowerCAmelCase   :Dict						=			'gelu_new'
																		else:
																								lowerCAmelCase   :Optional[int]						=			'gelu'
												else:
																		# in the very early days this used to be "gelu_new"
																		lowerCAmelCase   :str						=			'gelu_new'

												# Spell out all parameters in case the defaults change.
												lowerCAmelCase   :Optional[int]						=			GPTaConfig(
												    vocab_size=5_02_57     ,      n_positions=10_24     ,      n_embd=10_24     ,      n_layer=24     ,      n_head=16     ,      n_inner=40_96     ,      activation_function=a__     ,      resid_pdrop=0.1     ,      embd_pdrop=0.1     ,      attn_pdrop=0.1     ,      layer_norm_epsilon=1e-5     ,      initializer_range=0.02     ,      summary_type='cls_index'     ,      summary_use_proj=a__     ,      summary_activation=a__     ,      summary_proj_to_labels=a__     ,      summary_first_dropout=0.1     ,      scale_attn_weights=a__     ,      use_cache=a__     ,      bos_token_id=5_02_56     ,      eos_token_id=5_02_56     ,      )
						else:
												lowerCAmelCase   :int						=			GPTaConfig.from_json_file(args.config_file )

						lowerCAmelCase   :int						=			['GPT2LMHeadModel']

						# Convert.
						print('Converting' )
						lowerCAmelCase   :str						=			convert_megatron_checkpoint(a__     ,      a__     ,      a__ )

						# Print the structure of converted state dict.
						if args.print_checkpoint_structure:
												recursive_print(a__     ,      a__ )

						# Add tokenizer class info to config
						# see https://github.com/huggingface/transformers/issues/13906)
						if ds_args is not None:
												lowerCAmelCase   :Optional[Any]						=			ds_args.tokenizer_type
												if tokenizer_type == "GPT2BPETokenizer":
																		lowerCAmelCase   :str						=			'gpt2'
												elif tokenizer_type == "PretrainedFromHF":
																		lowerCAmelCase   :Optional[int]						=			ds_args.tokenizer_name_or_path
												else:
																		raise ValueError(F"""Unrecognized tokenizer_type {tokenizer_type}""" )
						else:
												lowerCAmelCase   :Any						=			'gpt2'

						lowerCAmelCase   :Any						=			AutoTokenizer.from_pretrained(a__ )
						lowerCAmelCase   :Union[str, Any]						=			type(a__ ).__name__
						lowerCAmelCase   :str						=			tokenizer_class

						# Store the config to file.
						print('Saving config' )
						config.save_pretrained(a__ )

						# Save tokenizer based on args
						print(F"""Adding {tokenizer_class} tokenizer files""" )
						tokenizer.save_pretrained(a__ )

						# Store the state_dict to file.
						lowerCAmelCase   :List[Any]						=			os.path.join(a__     ,      'pytorch_model.bin' )
						print(F"""Saving checkpoint to \"{output_checkpoint_file}\"""" )
						torch.save(a__     ,      a__ )


####################################################################################################

if __name__ == "__main__":
		main()

####################################################################################################







"""simple docstring"""


import argparse

import torch

from transformers import MobileBertConfig, MobileBertForPreTraining, load_tf_weights_in_mobilebert
from transformers.utils import logging


logging.set_verbosity_info()


def        UpperCAmelCase      (					a__     ,      a__     ,      a__ ):





						'''simple docstring'''






						lowerCAmelCase   :Tuple						=			MobileBertConfig.from_json_file(a__ )
						print(F"""Building PyTorch model from configuration: {config}""" )
						lowerCAmelCase   :Tuple						=			MobileBertForPreTraining(a__ )
						# Load weights from tf checkpoint
						lowerCAmelCase   :Any						=			load_tf_weights_in_mobilebert(a__     ,      a__     ,      a__ )
						# Save pytorch-model
						print(F"""Save PyTorch model to {pytorch_dump_path}""" )
						torch.save(model.state_dict()     ,      a__ )


if __name__ == "__main__":
		__SCREAMING_SNAKE_CASE   =   argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
		)
		parser.add_argument(
		    '--mobilebert_config_file',
		    default=None,
		    type=str,
		    required=True,
		    help=(
		        'The config json file corresponding to the pre-trained MobileBERT model. \n'
		        'This specifies the model architecture.'
		    ),
		)
		parser.add_argument(
		    '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
		)
		__SCREAMING_SNAKE_CASE   =   parser.parse_args()
		convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.mobilebert_config_file, args.pytorch_dump_path)