
'''simple docstring'''







# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig
def  snake_case_		(    __SCREAMING_SNAKE_CASE :			str				,				__SCREAMING_SNAKE_CASE :			Union[str, Any]				,				__SCREAMING_SNAKE_CASE :			Optional[Any]=0							):



						"""simple docstring"""





						if name is None:
												lowercase_     :					int													=	None
						else:
												lowercase_     :					Dict													=	'''.''' * max(0				,				spaces - 2							) + '''# {:''' + str(50 - spaces							) + '''s}'''
												lowercase_     :					Any													=	fmt.format(__SCREAMING_SNAKE_CASE							)

						# Print and recurse (if needed).
						if isinstance(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE							):
												if msg is not None:
																		print(__SCREAMING_SNAKE_CASE							)
												for k in val.keys():
																		recursive_print(__SCREAMING_SNAKE_CASE				,				val[k]				,				spaces + 2							)
						elif isinstance(__SCREAMING_SNAKE_CASE				,				torch.Tensor							):
												print(__SCREAMING_SNAKE_CASE				,				''':'''				,				val.size()							)
						else:
												print(__SCREAMING_SNAKE_CASE				,				''':'''				,				__SCREAMING_SNAKE_CASE							)
def  snake_case_		(    __SCREAMING_SNAKE_CASE :			List[Any]				,				__SCREAMING_SNAKE_CASE :			int				,				__SCREAMING_SNAKE_CASE :			Dict				,				__SCREAMING_SNAKE_CASE :			List[Any]				,				__SCREAMING_SNAKE_CASE :			List[str]							):



						"""simple docstring"""





						lowercase_     :					Tuple													=	param.size()
						if checkpoint_version == 1.0:
												# version 1.0 stores [num_heads * hidden_size * num_splits, :]
												lowercase_     :					Any													=	(num_heads, hidden_size, num_splits) + input_shape[1:]
												lowercase_     :					Union[str, Any]													=	param.view(*__SCREAMING_SNAKE_CASE							)
												lowercase_     :					Union[str, Any]													=	param.transpose(0				,				2							)
												lowercase_     :					List[Any]													=	param.transpose(1				,				2							).contiguous()
						elif checkpoint_version >= 2.0:
												# other versions store [num_heads * num_splits * hidden_size, :]
												lowercase_     :					List[Any]													=	(num_heads, num_splits, hidden_size) + input_shape[1:]
												lowercase_     :					str													=	param.view(*__SCREAMING_SNAKE_CASE							)
												lowercase_     :					Optional[Any]													=	param.transpose(0				,				1							).contiguous()
						lowercase_     :					List[str]													=	param.view(*__SCREAMING_SNAKE_CASE							)
						return param
def  snake_case_		(    __SCREAMING_SNAKE_CASE :			str				,				__SCREAMING_SNAKE_CASE :			Any				,				__SCREAMING_SNAKE_CASE :			List[Any]							):



						"""simple docstring"""





						lowercase_     :					Optional[Any]													=	{}

						# old versions did not store training args
						lowercase_     :					Dict													=	input_state_dict.get('''args'''				,				__SCREAMING_SNAKE_CASE							)
						if ds_args is not None:
												# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
												# from pprint import pprint
												# pprint(vars(ds_args))

												lowercase_     :					Optional[Any]													=	ds_args.padded_vocab_size
												lowercase_     :					str													=	ds_args.max_position_embeddings
												lowercase_     :					Any													=	ds_args.hidden_size
												lowercase_     :					Tuple													=	ds_args.num_layers
												lowercase_     :					Union[str, Any]													=	ds_args.num_attention_heads
												lowercase_     :					Union[str, Any]													=	ds_args.ffn_hidden_size
												# pprint(config)

						# The number of heads.
						lowercase_     :					Any													=	config.n_head
						# The hidden_size per head.
						lowercase_     :					Optional[Any]													=	config.n_embd // config.n_head
						# Megatron-LM checkpoint version
						if "checkpoint_version" in input_state_dict.keys():
												lowercase_     :					List[str]													=	input_state_dict['''checkpoint_version''']
						else:
												lowercase_     :					Optional[int]													=	0.0

						# The model.
						lowercase_     :					str													=	input_state_dict['''model''']
						# The language model.
						lowercase_     :					Dict													=	model['''language_model''']
						# The embeddings.
						lowercase_     :					Union[str, Any]													=	lm['''embedding''']

						# The word embeddings.
						lowercase_     :					Tuple													=	embeddings['''word_embeddings''']['''weight''']
						# Truncate the embedding table to vocab_size rows.
						lowercase_     :					Optional[int]													=	word_embeddings[: config.vocab_size, :]
						lowercase_     :					Dict													=	word_embeddings

						# The position embeddings.
						lowercase_     :					Optional[int]													=	embeddings['''position_embeddings''']['''weight''']
						# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
						lowercase_     :					str													=	pos_embeddings.size(0							)
						if n_positions != config.n_positions:
												raise ValueError(
												    F'''pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'''							)
						# Store the position embeddings.
						lowercase_     :					List[Any]													=	pos_embeddings

						# The transformer.
						lowercase_     :					List[str]													=	lm['''transformer'''] if '''transformer''' in lm.keys() else lm['''encoder''']

						# The regex to extract layer names.
						lowercase_     :					str													=	re.compile(R'''layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'''							)

						# The simple map of names for "automated" rules.
						lowercase_     :					List[str]													=	{
						    '''attention.dense''': '''.attn.c_proj.''',
						    '''self_attention.dense''': '''.attn.c_proj.''',
						    '''mlp.dense_h_to_4h''': '''.mlp.c_fc.''',
						    '''mlp.dense_4h_to_h''': '''.mlp.c_proj.''',
						}

						# Extract the layers.
						for key, val in transformer.items():
												# Match the name.
												lowercase_     :					int													=	layer_re.match(__SCREAMING_SNAKE_CASE							)

												# Stop if that's not a layer
												if m is None:
																		break

												# The index of the layer.
												lowercase_     :					Optional[Any]													=	int(m.group(1							)							)
												# The name of the operation.
												lowercase_     :					Union[str, Any]													=	m.group(2							)
												# Is it a weight or a bias?
												lowercase_     :					Optional[int]													=	m.group(3							)

												# The name of the layer.
												lowercase_     :					Union[str, Any]													=	F'''transformer.h.{layer_idx}'''

												# For layernorm(s), simply store the layer norm.
												if op_name.endswith('''layernorm'''							):
																		lowercase_     :					Dict													=	'''ln_1''' if op_name.startswith('''input'''							) else '''ln_2'''
																		lowercase_     :					Optional[int]													=	val

												# Transpose the QKV matrix.
												elif (
												    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
												) and weight_or_bias == "weight":
																		# Insert a tensor of 1x1xDxD bias.
																		lowercase_     :					Optional[Any]													=	torch.tril(torch.ones((n_positions, n_positions)				,				dtype=torch.floataa							)							).view(
																		    1				,				1				,				__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE							)
																		lowercase_     :					int													=	causal_mask

																		# Insert a "dummy" tensor for masked_bias.
																		lowercase_     :					Tuple													=	torch.tensor(-1E4				,				dtype=torch.floataa							)
																		lowercase_     :					Tuple													=	masked_bias

																		lowercase_     :					List[Any]													=	fix_query_key_value_ordering(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE				,				3				,				__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE							)
																		# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
																		lowercase_     :					List[Any]													=	out_val.transpose(0				,				1							).contiguous()
																		# Store.
																		lowercase_     :					int													=	out_val

												# Transpose the bias.
												elif (
												    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
												) and weight_or_bias == "bias":
																		lowercase_     :					Optional[Any]													=	fix_query_key_value_ordering(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE				,				3				,				__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE							)
																		# Store. No change of shape.
																		lowercase_     :					Any													=	out_val

												# Transpose the weights.
												elif weight_or_bias == "weight":
																		lowercase_     :					Dict													=	megatron_to_transformers[op_name]
																		lowercase_     :					List[Any]													=	val.transpose(0				,				1							)

												# Copy the bias.
												elif weight_or_bias == "bias":
																		lowercase_     :					Dict													=	megatron_to_transformers[op_name]
																		lowercase_     :					Optional[int]													=	val

    # DEBUG.
						assert config.n_layer == layer_idx + 1

						# The final layernorm.
						lowercase_     :					Optional[int]													=	transformer['''final_layernorm.weight''']
						lowercase_     :					str													=	transformer['''final_layernorm.bias''']

						# For LM head, transformers' wants the matrix to weight embeddings.
						lowercase_     :					List[Any]													=	word_embeddings

						# It should be done!
						return output_state_dict
def  snake_case_		(    ):



						"""simple docstring"""





						lowercase_     :					int													=	argparse.ArgumentParser()
						parser.add_argument('''--print-checkpoint-structure'''				,				action='''store_true'''							)
						parser.add_argument(
						    '''path_to_checkpoint'''				,				type=__SCREAMING_SNAKE_CASE				,				help='''Path to the checkpoint file (.zip archive or direct .pt file)'''				,				)
						parser.add_argument(
						    '''--config_file'''				,				default=''''''				,				type=__SCREAMING_SNAKE_CASE				,				help='''An optional config json file describing the pre-trained model.'''				,				)
						lowercase_     :					List[str]													=	parser.parse_args()

						# Extract the basename.
						lowercase_     :					Dict													=	os.path.dirname(args.path_to_checkpoint							)

						# Load the model.
						# the .zip is very optional, let's keep it for backward compatibility
						print(F'''Extracting PyTorch state dictionary from {args.path_to_checkpoint}'''							)
						if args.path_to_checkpoint.endswith('''.zip'''							):
												with zipfile.ZipFile(args.path_to_checkpoint				,				'''r'''							) as checkpoint:
																		with checkpoint.open('''release/mp_rank_00/model_optim_rng.pt'''							) as pytorch_dict:
																								lowercase_     :					str													=	torch.load(__SCREAMING_SNAKE_CASE				,				map_location='''cpu'''							)
						else:
												lowercase_     :					Any													=	torch.load(args.path_to_checkpoint				,				map_location='''cpu'''							)

						lowercase_     :					Optional[Any]													=	input_state_dict.get('''args'''				,				__SCREAMING_SNAKE_CASE							)

						# Read the config, or default to the model released by NVIDIA.
						if args.config_file == "":
												if ds_args is not None:
																		if ds_args.bias_gelu_fusion:
																								lowercase_     :					Optional[int]													=	'''gelu_fast'''
																		elif ds_args.openai_gelu:
																								lowercase_     :					int													=	'''gelu_new'''
																		else:
																								lowercase_     :					List[Any]													=	'''gelu'''
												else:
																		# in the very early days this used to be "gelu_new"
																		lowercase_     :					str													=	'''gelu_new'''

												# Spell out all parameters in case the defaults change.
												lowercase_     :					Tuple													=	GPTaConfig(
												    vocab_size=50257				,				n_positions=1024				,				n_embd=1024				,				n_layer=24				,				n_head=16				,				n_inner=4096				,				activation_function=__SCREAMING_SNAKE_CASE				,				resid_pdrop=0.1				,				embd_pdrop=0.1				,				attn_pdrop=0.1				,				layer_norm_epsilon=1E-5				,				initializer_range=0.02				,				summary_type='''cls_index'''				,				summary_use_proj=__SCREAMING_SNAKE_CASE				,				summary_activation=__SCREAMING_SNAKE_CASE				,				summary_proj_to_labels=__SCREAMING_SNAKE_CASE				,				summary_first_dropout=0.1				,				scale_attn_weights=__SCREAMING_SNAKE_CASE				,				use_cache=__SCREAMING_SNAKE_CASE				,				bos_token_id=50256				,				eos_token_id=50256				,				)
						else:
												lowercase_     :					Any													=	GPTaConfig.from_json_file(args.config_file							)

						lowercase_     :					List[Any]													=	['''GPT2LMHeadModel''']

						# Convert.
						print('''Converting'''							)
						lowercase_     :					Tuple													=	convert_megatron_checkpoint(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE							)

						# Print the structure of converted state dict.
						if args.print_checkpoint_structure:
												recursive_print(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE							)

						# Add tokenizer class info to config
						# see https://github.com/huggingface/transformers/issues/13906)
						if ds_args is not None:
												lowercase_     :					List[Any]													=	ds_args.tokenizer_type
												if tokenizer_type == "GPT2BPETokenizer":
																		lowercase_     :					Union[str, Any]													=	'''gpt2'''
												elif tokenizer_type == "PretrainedFromHF":
																		lowercase_     :					Any													=	ds_args.tokenizer_name_or_path
												else:
																		raise ValueError(F'''Unrecognized tokenizer_type {tokenizer_type}'''							)
						else:
												lowercase_     :					int													=	'''gpt2'''

						lowercase_     :					Optional[int]													=	AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE							)
						lowercase_     :					Tuple													=	type(__SCREAMING_SNAKE_CASE							).__name__
						lowercase_     :					str													=	tokenizer_class

						# Store the config to file.
						print('''Saving config'''							)
						config.save_pretrained(__SCREAMING_SNAKE_CASE							)

						# Save tokenizer based on args
						print(F'''Adding {tokenizer_class} tokenizer files'''							)
						tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE							)

						# Store the state_dict to file.
						lowercase_     :					Optional[Any]													=	os.path.join(__SCREAMING_SNAKE_CASE				,				'''pytorch_model.bin'''							)
						print(F'''Saving checkpoint to "{output_checkpoint_file}"'''							)
						torch.save(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE							)


####################################################################################################

if __name__ == "__main__":
							main()

####################################################################################################





'''simple docstring'''







import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 			lowerCAmelCase__       (  lowerCamelCase_   ,						unittest.TestCase					):
					lowerCAmelCase_					       =					None
					lowerCAmelCase_					       =					BloomTokenizerFast
					lowerCAmelCase_					       =					BloomTokenizerFast
					lowerCAmelCase_					       =					True
					lowerCAmelCase_					       =					False
					lowerCAmelCase_					       =					'''tokenizer_file'''
					lowerCAmelCase_					       =					{'''bos_token''': '''<s>''', '''eos_token''': '''</s>''', '''unk_token''': '''<unk>''', '''pad_token''': '''<pad>'''}






					def      _snake_case   (       self		):


											"""simple docstring"""





											super().setUp()
											lowercase_     :					List[Any]													=	BloomTokenizerFast.from_pretrained('''bigscience/tokenizer'''		)
											tokenizer.save_pretrained(self.tmpdirname		)






					def      _snake_case   (       self    ,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											kwargs.update(self.special_tokens_map		)
											return BloomTokenizerFast.from_pretrained(self.tmpdirname    ,							**__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Tuple													=	self.get_rust_tokenizer()

											lowercase_     :					Optional[Any]													=	['''The quick brown fox</s>''', '''jumps over the lazy dog</s>''']
											lowercase_     :					Union[str, Any]													=	[[21_75, 2_37_14, 7_31_73, 14_42_52, 2], [77, 13_26_19, 34_78, 3_68, 10_95_86, 3_54_33, 2]]

											lowercase_     :					Dict													=	tokenizer.batch_encode_plus(__SCREAMING_SNAKE_CASE		)['''input_ids''']
											self.assertListEqual(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)

											lowercase_     :					Tuple													=	tokenizer.batch_decode(__SCREAMING_SNAKE_CASE		)
											self.assertListEqual(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE=6		):


											"""simple docstring"""





											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
																							lowercase_     :					int													=	self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)
																							# tokenizer_r.pad_token = None # Hotfixing padding = None
																							# Simple input
																							lowercase_     :					Optional[int]													=	'''This is a simple input'''
																							lowercase_     :					List[str]													=	['''This is a simple input 1''', '''This is a simple input 2''']
																							lowercase_     :					List[str]													=	('''This is a simple input''', '''This is a pair''')
																							lowercase_     :					Union[str, Any]													=	[
																							    ('''This is a simple input 1''', '''This is a simple input 2'''),
																							    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																							]

																							# Simple input tests
																							try:
																													tokenizer_r.encode(__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE		)
																													tokenizer_r.encode_plus(__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE		)

																													tokenizer_r.batch_encode_plus(__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE		)
																													tokenizer_r.encode(__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE		)
																													tokenizer_r.batch_encode_plus(__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE		)
																							except ValueError:
																													self.fail('''Bloom Tokenizer should be able to deal with padding'''		)

																							lowercase_     :					Optional[Any]													=	None  # Hotfixing padding = None
																							self.assertRaises(__SCREAMING_SNAKE_CASE    ,							tokenizer_r.encode    ,							__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE    ,							padding='''max_length'''		)

																							# Simple input
																							self.assertRaises(__SCREAMING_SNAKE_CASE    ,							tokenizer_r.encode_plus    ,							__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE    ,							padding='''max_length'''		)

																							# Simple input
																							self.assertRaises(
																							    __SCREAMING_SNAKE_CASE    ,							tokenizer_r.batch_encode_plus    ,							__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE    ,							padding='''max_length'''    ,							)

																							# Pair input
																							self.assertRaises(__SCREAMING_SNAKE_CASE    ,							tokenizer_r.encode    ,							__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE    ,							padding='''max_length'''		)

																							# Pair input
																							self.assertRaises(__SCREAMING_SNAKE_CASE    ,							tokenizer_r.encode_plus    ,							__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE    ,							padding='''max_length'''		)

																							# Pair input
																							self.assertRaises(
																							    __SCREAMING_SNAKE_CASE    ,							tokenizer_r.batch_encode_plus    ,							__SCREAMING_SNAKE_CASE    ,							max_length=__SCREAMING_SNAKE_CASE    ,							padding='''max_length'''    ,							)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					List[str]													=	self.get_rust_tokenizer()
											lowercase_     :					Tuple													=	load_dataset('''xnli'''    ,							'''all_languages'''    ,							split='''test'''    ,							streaming=__SCREAMING_SNAKE_CASE		)

											lowercase_     :					Optional[int]													=	next(iter(__SCREAMING_SNAKE_CASE		)		)['''premise''']  # pick up one data
											lowercase_     :					List[Any]													=	list(sample_data.values()		)

											lowercase_     :					Tuple													=	list(map(tokenizer.encode    ,							__SCREAMING_SNAKE_CASE		)		)
											lowercase_     :					Any													=	[tokenizer.decode(__SCREAMING_SNAKE_CASE    ,							clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE		) for x in output_tokens]
											self.assertListEqual(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map		)    ,							1		)
											self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()		)[0]		)    ,							1		)



