


"""simple docstring"""


# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig







def  __snake_case       (   UpperCamelCase__ ,			UpperCamelCase__ ,			UpperCamelCase__=0	)      ->		Optional[int]:







		"""simple docstring"""





		if name is None:
				A					       = None
		else:
				A					       = '.' * max(0 ,			spaces - 2	) + '# {:' + str(50 - spaces	) + 's}'
				A					       = fmt.format(UpperCamelCase__	)

		# Print and recurse (if needed).
		if isinstance(UpperCamelCase__ ,			UpperCamelCase__	):
				if msg is not None:
						print(UpperCamelCase__	)
				for k in val.keys():
						recursive_print(UpperCamelCase__ ,			val[k] ,			spaces + 2	)
		elif isinstance(UpperCamelCase__ ,			torch.Tensor	):
				print(UpperCamelCase__ ,			':' ,			val.size()	)
		else:
				print(UpperCamelCase__ ,			':' ,			UpperCamelCase__	)







def  __snake_case       (   UpperCamelCase__ ,			UpperCamelCase__ ,			UpperCamelCase__ ,			UpperCamelCase__ ,			UpperCamelCase__	)      ->		Union[str, Any]:







		"""simple docstring"""





		A					       = param.size()
		if checkpoint_version == 1.0:
				# version 1.0 stores [num_heads * hidden_size * num_splits, :]
				A					       = (num_heads, hidden_size, num_splits) + input_shape[1:]
				A					       = param.view(*UpperCamelCase__	)
				A					       = param.transpose(0 ,			2	)
				A					       = param.transpose(1 ,			2	).contiguous()
		elif checkpoint_version >= 2.0:
				# other versions store [num_heads * num_splits * hidden_size, :]
				A					       = (num_heads, num_splits, hidden_size) + input_shape[1:]
				A					       = param.view(*UpperCamelCase__	)
				A					       = param.transpose(0 ,			1	).contiguous()
		A					       = param.view(*UpperCamelCase__	)
		return param







def  __snake_case       (   UpperCamelCase__ ,			UpperCamelCase__ ,			UpperCamelCase__	)      ->		int:







		"""simple docstring"""





		A					       = {}

		# old versions did not store training args
		A					       = input_state_dict.get('args' ,			UpperCamelCase__	)
		if ds_args is not None:
				# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
				# from pprint import pprint
				# pprint(vars(ds_args))

				A					       = ds_args.padded_vocab_size
				A					       = ds_args.max_position_embeddings
				A					       = ds_args.hidden_size
				A					       = ds_args.num_layers
				A					       = ds_args.num_attention_heads
				A					       = ds_args.ffn_hidden_size
				# pprint(config)

		# The number of heads.
		A					       = config.n_head
		# The hidden_size per head.
		A					       = config.n_embd // config.n_head
		# Megatron-LM checkpoint version
		if "checkpoint_version" in input_state_dict.keys():
				A					       = input_state_dict['checkpoint_version']
		else:
				A					       = 0.0

		# The model.
		A					       = input_state_dict['model']
		# The language model.
		A					       = model['language_model']
		# The embeddings.
		A					       = lm['embedding']

		# The word embeddings.
		A					       = embeddings['word_embeddings']['weight']
		# Truncate the embedding table to vocab_size rows.
		A					       = word_embeddings[: config.vocab_size, :]
		A					       = word_embeddings

		# The position embeddings.
		A					       = embeddings['position_embeddings']['weight']
		# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
		A					       = pos_embeddings.size(0	)
		if n_positions != config.n_positions:
				raise ValueError(
				    f'pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'	)
		# Store the position embeddings.
		A					       = pos_embeddings

		# The transformer.
		A					       = lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

		# The regex to extract layer names.
		A					       = re.compile(r'layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'	)

		# The simple map of names for "automated" rules.
		A					       = {
		    'attention.dense': '.attn.c_proj.',
		    'self_attention.dense': '.attn.c_proj.',
		    'mlp.dense_h_to_4h': '.mlp.c_fc.',
		    'mlp.dense_4h_to_h': '.mlp.c_proj.',
		}

		# Extract the layers.
		for key, val in transformer.items():
				# Match the name.
				A					       = layer_re.match(UpperCamelCase__	)

				# Stop if that's not a layer
				if m is None:
						break

				# The index of the layer.
				A					       = int(m.group(1	)	)
				# The name of the operation.
				A					       = m.group(2	)
				# Is it a weight or a bias?
				A					       = m.group(3	)

				# The name of the layer.
				A					       = f'transformer.h.{layer_idx}'

				# For layernorm(s), simply store the layer norm.
				if op_name.endswith('layernorm'	):
						A					       = 'ln_1' if op_name.startswith('input'	) else 'ln_2'
						A					       = val

				# Transpose the QKV matrix.
				elif (
				    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
				) and weight_or_bias == "weight":
						# Insert a tensor of 1x1xDxD bias.
						A					       = torch.tril(torch.ones((n_positions, n_positions) ,			dtype=torch.floataa	)	).view(
						    1 ,			1 ,			UpperCamelCase__ ,			UpperCamelCase__	)
						A					       = causal_mask

						# Insert a "dummy" tensor for masked_bias.
						A					       = torch.tensor(-1E4 ,			dtype=torch.floataa	)
						A					       = masked_bias

						A					       = fix_query_key_value_ordering(UpperCamelCase__ ,			UpperCamelCase__ ,			3 ,			UpperCamelCase__ ,			UpperCamelCase__	)
						# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
						A					       = out_val.transpose(0 ,			1	).contiguous()
						# Store.
						A					       = out_val

				# Transpose the bias.
				elif (
				    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
				) and weight_or_bias == "bias":
						A					       = fix_query_key_value_ordering(UpperCamelCase__ ,			UpperCamelCase__ ,			3 ,			UpperCamelCase__ ,			UpperCamelCase__	)
						# Store. No change of shape.
						A					       = out_val

				# Transpose the weights.
				elif weight_or_bias == "weight":
						A					       = megatron_to_transformers[op_name]
						A					       = val.transpose(0 ,			1	)

				# Copy the bias.
				elif weight_or_bias == "bias":
						A					       = megatron_to_transformers[op_name]
						A					       = val

    # DEBUG.
		assert config.n_layer == layer_idx + 1

		# The final layernorm.
		A					       = transformer['final_layernorm.weight']
		A					       = transformer['final_layernorm.bias']

		# For LM head, transformers' wants the matrix to weight embeddings.
		A					       = word_embeddings

		# It should be done!
		return output_state_dict







def  __snake_case       (   )      ->		Tuple:







		"""simple docstring"""





		A					       = argparse.ArgumentParser()
		parser.add_argument('--print-checkpoint-structure' ,			action='store_true'	)
		parser.add_argument(
		    'path_to_checkpoint' ,			type=UpperCamelCase__ ,			help='Path to the checkpoint file (.zip archive or direct .pt file)' ,			)
		parser.add_argument(
		    '--config_file' ,			default='' ,			type=UpperCamelCase__ ,			help='An optional config json file describing the pre-trained model.' ,			)
		A					       = parser.parse_args()

		# Extract the basename.
		A					       = os.path.dirname(args.path_to_checkpoint	)

		# Load the model.
		# the .zip is very optional, let's keep it for backward compatibility
		print(f'Extracting PyTorch state dictionary from {args.path_to_checkpoint}'	)
		if args.path_to_checkpoint.endswith('.zip'	):
				with zipfile.ZipFile(args.path_to_checkpoint ,			'r'	) as checkpoint:
						with checkpoint.open('release/mp_rank_00/model_optim_rng.pt'	) as pytorch_dict:
								A					       = torch.load(UpperCamelCase__ ,			map_location='cpu'	)
		else:
				A					       = torch.load(args.path_to_checkpoint ,			map_location='cpu'	)

		A					       = input_state_dict.get('args' ,			UpperCamelCase__	)

		# Read the config, or default to the model released by NVIDIA.
		if args.config_file == "":
				if ds_args is not None:
						if ds_args.bias_gelu_fusion:
								A					       = 'gelu_fast'
						elif ds_args.openai_gelu:
								A					       = 'gelu_new'
						else:
								A					       = 'gelu'
				else:
						# in the very early days this used to be "gelu_new"
						A					       = 'gelu_new'

				# Spell out all parameters in case the defaults change.
				A					       = GPTaConfig(
				    vocab_size=50257 ,			n_positions=1024 ,			n_embd=1024 ,			n_layer=24 ,			n_head=16 ,			n_inner=4096 ,			activation_function=UpperCamelCase__ ,			resid_pdrop=0.1 ,			embd_pdrop=0.1 ,			attn_pdrop=0.1 ,			layer_norm_epsilon=1E-5 ,			initializer_range=0.0_2 ,			summary_type='cls_index' ,			summary_use_proj=UpperCamelCase__ ,			summary_activation=UpperCamelCase__ ,			summary_proj_to_labels=UpperCamelCase__ ,			summary_first_dropout=0.1 ,			scale_attn_weights=UpperCamelCase__ ,			use_cache=UpperCamelCase__ ,			bos_token_id=50256 ,			eos_token_id=50256 ,			)
		else:
				A					       = GPTaConfig.from_json_file(args.config_file	)

		A					       = ['GPT2LMHeadModel']

		# Convert.
		print('Converting'	)
		A					       = convert_megatron_checkpoint(UpperCamelCase__ ,			UpperCamelCase__ ,			UpperCamelCase__	)

		# Print the structure of converted state dict.
		if args.print_checkpoint_structure:
				recursive_print(UpperCamelCase__ ,			UpperCamelCase__	)

		# Add tokenizer class info to config
		# see https://github.com/huggingface/transformers/issues/13906)
		if ds_args is not None:
				A					       = ds_args.tokenizer_type
				if tokenizer_type == "GPT2BPETokenizer":
						A					       = 'gpt2'
				elif tokenizer_type == "PretrainedFromHF":
						A					       = ds_args.tokenizer_name_or_path
				else:
						raise ValueError(f'Unrecognized tokenizer_type {tokenizer_type}'	)
		else:
				A					       = 'gpt2'

		A					       = AutoTokenizer.from_pretrained(UpperCamelCase__	)
		A					       = type(UpperCamelCase__	).__name__
		A					       = tokenizer_class

		# Store the config to file.
		print('Saving config'	)
		config.save_pretrained(UpperCamelCase__	)

		# Save tokenizer based on args
		print(f'Adding {tokenizer_class} tokenizer files'	)
		tokenizer.save_pretrained(UpperCamelCase__	)

		# Store the state_dict to file.
		A					       = os.path.join(UpperCamelCase__ ,			'pytorch_model.bin'	)
		print(f'Saving checkpoint to "{output_checkpoint_file}"'	)
		torch.save(UpperCamelCase__ ,			UpperCamelCase__	)


####################################################################################################

if __name__ == "__main__":
	main()

####################################################################################################









"""simple docstring"""


from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding



class 			lowerCamelCase__						(   UpperCAmelCase_			):
						lowerCAmelCase       =     """ClapFeatureExtractor"""
						lowerCAmelCase       =     ("""RobertaTokenizer""", """RobertaTokenizerFast""")







						def __init__(							self   :    Dict			,			_lowercase   :    List[str]			,			_lowercase   :    Optional[int]		):
								super().__init__(_lowercase			,			_lowercase		)







						def __call__(							self   :    Optional[int]			,			_lowercase   :    List[Any]=None			,			_lowercase   :    Dict=None			,			_lowercase   :    str=None			,			**_lowercase   :    Union[str, Any]		):
								A					       = kwargs.pop('sampling_rate'			,			_lowercase		)

								if text is None and audios is None:
										raise ValueError('You have to specify either text or audios. Both cannot be none.'		)

								if text is not None:
										A					       = self.tokenizer(_lowercase			,			return_tensors=_lowercase			,			**_lowercase		)

								if audios is not None:
										A					       = self.feature_extractor(
										    _lowercase			,			sampling_rate=_lowercase			,			return_tensors=_lowercase			,			**_lowercase		)

								if text is not None and audios is not None:
										A					       = audio_features.input_features
										return encoding
								elif text is not None:
										return encoding
								else:
										return BatchEncoding(data=dict(**_lowercase		)			,			tensor_type=_lowercase		)







						def __a						(							self   :    List[Any]			,			*_lowercase   :    Dict			,			**_lowercase   :    Optional[int]		):
								return self.tokenizer.batch_decode(*_lowercase			,			**_lowercase		)







						def __a						(							self   :    List[str]			,			*_lowercase   :    str			,			**_lowercase   :    Optional[int]		):
								return self.tokenizer.decode(*_lowercase			,			**_lowercase		)







						@property
						def __a						(							self   :    Tuple		):
								A					       = self.tokenizer.model_input_names
								A					       = self.feature_extractor.model_input_names
								return list(dict.fromkeys(tokenizer_input_names + feature_extractor_input_names		)		)





