

'''simple docstring'''




import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig




def 						UpperCamelCase       ( a		,						a		,						a=0 )      ->	Tuple:






     '''simple docstring'''
     # Format the message.
     if name is None:
          __magic_name__						   =				None
     else:
          __magic_name__						   =				'''.''' * max(0		,						spaces - 2 ) + '''# {:''' + str(50 - spaces ) + '''s}'''
          __magic_name__						   =				fmt.format(_SCREAMING_SNAKE_CASE )

     # Print and recurse (if needed).
     if isinstance(_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE ):
          if msg is not None:
               print(_SCREAMING_SNAKE_CASE )
          for k in val.keys():
               recursive_print(_SCREAMING_SNAKE_CASE		,						val[k]		,						spaces + 2 )
     elif isinstance(_SCREAMING_SNAKE_CASE		,						torch.Tensor ):
          print(_SCREAMING_SNAKE_CASE		,						''':'''		,						val.size() )
     else:
          print(_SCREAMING_SNAKE_CASE		,						''':'''		,						_SCREAMING_SNAKE_CASE )




def 						UpperCamelCase       ( a		,						a		,						a		,						a )      ->	str:






     '''simple docstring'''
     __magic_name__						   =				param.size()
     # other versions store [num_heads * num_splits * hidden_size, :]
     __magic_name__						   =				(num_heads, num_splits, hidden_size) + input_shape[1:]
     __magic_name__						   =				param.view(*_SCREAMING_SNAKE_CASE )
     __magic_name__						   =				param.transpose(0		,						1 ).contiguous()
     __magic_name__						   =				param.view(*_SCREAMING_SNAKE_CASE )
     return param




def 						UpperCamelCase       ( a		,						a )      ->	List[str]:






     '''simple docstring'''
     __magic_name__						   =				config.n_positions
     __magic_name__						   =				config.n_layer
     __magic_name__						   =				config.vocab_size
     __magic_name__						   =				config.n_head
     __magic_name__						   =				config.n_embd // config.n_head

     __magic_name__						   =				sd_megatron['''model.language_model.embedding.word_embeddings.weight'''][:vocab_size, :]
     __magic_name__						   =				{
         '''transformer.wte.weight''': word_embeddings,
         '''transformer.wpe.weight''': sd_megatron['''model.language_model.embedding.position_embeddings.weight'''],
         '''transformer.ln_f.weight''': sd_megatron['''model.language_model.encoder.final_layernorm.weight'''],
         '''transformer.ln_f.bias''': sd_megatron['''model.language_model.encoder.final_layernorm.bias'''],
     }

     __magic_name__						   =				'''model.language_model.encoder.layers.'''
     for i in range(_SCREAMING_SNAKE_CASE ):
          __magic_name__						   =				torch.tril(torch.ones((n_positions, n_positions)		,						dtype=torch.bool ) )
          __magic_name__						   =				causal_mask.view(1		,						1		,						_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE )
          __magic_name__						   =				causal_mask
          __magic_name__						   =				torch.tensor(-1e4		,						dtype=torch.bfloataa )

          __magic_name__						   =				sd_megatron[F'''{pf}{i}.input_layernorm.weight''']
          __magic_name__						   =				sd_megatron[F'''{pf}{i}.input_layernorm.bias''']

          __magic_name__						   =				sd_megatron[F'''{pf}{i}.self_attention.query_key_value.weight''']
          __magic_name__						   =				fix_query_key_value_ordering(_SCREAMING_SNAKE_CASE		,						3		,						_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE )
          __magic_name__						   =				vala.transpose(0		,						1 ).contiguous()

          __magic_name__						   =				sd_megatron[F'''{pf}{i}.self_attention.query_key_value.bias''']
          __magic_name__						   =				fix_query_key_value_ordering(_SCREAMING_SNAKE_CASE		,						3		,						_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE )
          __magic_name__						   =				vala

          __magic_name__						   =				sd_megatron[F'''{pf}{i}.self_attention.dense.weight'''].transpose(
              0		,						1 )
          __magic_name__						   =				sd_megatron[F'''{pf}{i}.self_attention.dense.bias''']
          __magic_name__						   =				sd_megatron[F'''{pf}{i}.post_attention_layernorm.weight''']
          __magic_name__						   =				sd_megatron[F'''{pf}{i}.post_attention_layernorm.bias''']
          __magic_name__						   =				sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.weight'''].transpose(0		,						1 )
          __magic_name__						   =				sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.bias''']
          __magic_name__						   =				sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.weight'''].transpose(
              0		,						1 )
          __magic_name__						   =				sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.bias''']

     # For LM head, transformers' wants the matrix to weight embeddings.
     __magic_name__						   =				word_embeddings

     return sd_hf




def 						UpperCamelCase       ( a		,						a )      ->	Dict:






     '''simple docstring'''
     __magic_name__						   =				6_4000
     __magic_name__						   =				config_megatron['''encoder_seq_length''']
     __magic_name__						   =				config_megatron['''hidden_size''']
     __magic_name__						   =				config_megatron['''num_layers''']
     __magic_name__						   =				config_megatron['''num_attention_heads''']
     __magic_name__						   =				config_megatron['''ffn_hidden_size''']
     __magic_name__						   =				'''gelu'''
     __magic_name__						   =				0.1
     __magic_name__						   =				0.1
     __magic_name__						   =				0.1
     __magic_name__						   =				config_megatron['''layernorm_epsilon''']  # 1e-5
     __magic_name__						   =				config_megatron['''init_method_std''']  # 0.02
     __magic_name__						   =				config_megatron['''apply_query_key_layer_scaling''']  # True
     __magic_name__						   =				True
     __magic_name__						   =				True

     # This identifies the 6.7B (7B) model which uses a different tokenizer
     if config_megatron["hidden_size"] == 4096:
          __magic_name__						   =				1  # <|endoftext|>
          __magic_name__						   =				1  # <|endoftext|>
          __magic_name__						   =				0  # <unk>
     else:
          __magic_name__						   =				2  # <s>
          __magic_name__						   =				3  # <|endoftext|>
          __magic_name__						   =				0  # <pad>

     return config_hf




def 						UpperCamelCase       ( a )      ->	List[str]:






     '''simple docstring'''
     print(_SCREAMING_SNAKE_CASE )

     __magic_name__						   =				args.checkpoint_path
     __magic_name__						   =				args.save_path
     if isfile(_SCREAMING_SNAKE_CASE ):
          raise FileNotFoundError(F'''ERROR! could not find file {checkpoint_path}''' )

     # Load the model.
     __magic_name__						   =				torch.load(_SCREAMING_SNAKE_CASE		,						map_location='''cpu''' )

     # Load the config.
     __magic_name__						   =				checkpoint['''hyper_parameters''']['''cfg''']
     __magic_name__						   =				GPTaConfig()
     __magic_name__						   =				copy_config(config_hf=_SCREAMING_SNAKE_CASE		,						config_megatron=_SCREAMING_SNAKE_CASE )
     __magic_name__						   =				['''GPT2LMHeadModel''']

     __magic_name__						   =				checkpoint['''state_dict''']

     # Convert.
     print('''Converting''' )
     __magic_name__						   =				convert_megatron_checkpoint(_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE )

     # Print the structure of converted state dict.
     if args.print_checkpoint_structure:
          recursive_print(_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE )

     __magic_name__						   =				'''GPTSw3Tokenizer'''

     # Store the config to file.
     print('''Saving config''' )
     config_hf.save_pretrained(_SCREAMING_SNAKE_CASE )

     # Store the state_dict to file.
     __magic_name__						   =				os.path.join(_SCREAMING_SNAKE_CASE		,						'''pytorch_model.bin''' )
     print(F'''Saving checkpoint to "{output_checkpoint_file}"''' )
     torch.save(_SCREAMING_SNAKE_CASE		,						_SCREAMING_SNAKE_CASE )


if __name__ == "__main__":
      _lowerCAmelCase    			=						argparse.ArgumentParser()
      parser.add_argument(
          "--checkpoint_path",
          type=str,
          required=True,
          help="e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000",
      )
      parser.add_argument("--save_path", type=str, required=True, help="e.g. /home/user/gpt-sw3/hf")
      parser.add_argument("--print-checkpoint-structure", action="store_true")
      _lowerCAmelCase    			=						parser.parse_args()
      main(_args)











'''simple docstring'''

from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


UpperCAmelCase			=    logging.get_logger(__name__)

UpperCAmelCase			=    {
    'google/bigbird-pegasus-large-arxiv': (
        'https://huggingface.co/google/bigbird-pegasus-large-arxiv/resolve/main/config.json'
    ),
    'google/bigbird-pegasus-large-pubmed': (
        'https://huggingface.co/google/bigbird-pegasus-large-pubmed/resolve/main/config.json'
    ),
    'google/bigbird-pegasus-large-bigpatent': (
        'https://huggingface.co/google/bigbird-pegasus-large-bigpatent/resolve/main/config.json'
    ),
    # See all BigBirdPegasus models at https://huggingface.co/models?filter=bigbird_pegasus
}

class __snake_case(      _lowerCAmelCase					):


						'''simple docstring'''



						UpperCAmelCase							:    Union[str, Any]           =		"bigbird_pegasus"
						UpperCAmelCase							:    Optional[Any]           =		["past_key_values"]
						UpperCAmelCase							:    Optional[Any]           =		{
						    "num_attention_heads": "encoder_attention_heads",
						    "hidden_size": "d_model",
						    "attention_probs_dropout_prob": "attention_dropout",
						}


						def __init__(					self	,       A_=9_6103	,       A_=4096	,       A_=16	,       A_=4096	,       A_=16	,       A_=16	,       A_=4096	,       A_=16	,       A_=0.0	,       A_=0.0	,       A_=True	,       A_=True	,       A_="gelu_new"	,       A_=1024	,       A_=0.1	,       A_=0.0	,       A_=0.0	,       A_=0.0_2	,       A_=2	,       A_=0.0	,       A_=True	,       A_=0	,       A_=2	,       A_=1	,       A_="block_sparse"	,       A_=64	,       A_=3	,       A_=False	,       **A_	,       )	->	List[str]:
							lowerCAmelCase    =	vocab_size
							lowerCAmelCase    =	max_position_embeddings
							lowerCAmelCase    =	d_model
							lowerCAmelCase    =	encoder_ffn_dim
							lowerCAmelCase    =	encoder_layers
							lowerCAmelCase    =	encoder_attention_heads
							lowerCAmelCase    =	decoder_ffn_dim
							lowerCAmelCase    =	decoder_layers
							lowerCAmelCase    =	decoder_attention_heads
							lowerCAmelCase    =	dropout
							lowerCAmelCase    =	attention_dropout
							lowerCAmelCase    =	activation_dropout
							lowerCAmelCase    =	activation_function
							lowerCAmelCase    =	init_std
							lowerCAmelCase    =	encoder_layerdrop
							lowerCAmelCase    =	decoder_layerdrop
							lowerCAmelCase    =	classifier_dropout
							lowerCAmelCase    =	use_cache
							lowerCAmelCase    =	encoder_layers
							lowerCAmelCase    =	scale_embedding  # scale factor will be sqrt(d_model) if True

							# extra config
							lowerCAmelCase    =	attention_type
							lowerCAmelCase    =	block_size
							lowerCAmelCase    =	num_random_blocks
							lowerCAmelCase    =	use_bias

							super().__init__(
							    pad_token_id=A_	,       bos_token_id=A_	,       eos_token_id=A_	,       is_encoder_decoder=A_	,       decoder_start_token_id=A_	,       **A_	,       )







class __snake_case(      _lowerCAmelCase					):


						'''simple docstring'''



						@property
						def 				__snake_case    (					self      )	->	Mapping[str, Mapping[int, str]]:
							if self.task in ["default", "seq2seq-lm"]:
								lowerCAmelCase    =	OrderedDict(
								    [
								        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
								        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
								    ]      )

								if self.use_past:
									lowerCAmelCase    =	{0: """batch"""}
									lowerCAmelCase    =	{0: """batch""", 1: """past_decoder_sequence + sequence"""}
								else:
									lowerCAmelCase    =	{0: """batch""", 1: """decoder_sequence"""}
									lowerCAmelCase    =	{0: """batch""", 1: """decoder_sequence"""}

								if self.use_past:
									self.fill_with_past_key_values_(A_	,       direction="""inputs"""      )
							elif self.task == "causal-lm":
								# TODO: figure this case out.
								lowerCAmelCase    =	OrderedDict(
								    [
								        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
								        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
								    ]      )
								if self.use_past:
									lowerCAmelCase,   lowerCAmelCase    =	self.num_layers
									for i in range(A_      ):
										lowerCAmelCase    =	{0: """batch""", 2: """past_sequence + sequence"""}
										lowerCAmelCase    =	{0: """batch""", 2: """past_sequence + sequence"""}
							else:
								lowerCAmelCase    =	OrderedDict(
								    [
								        ("""input_ids""", {0: """batch""", 1: """encoder_sequence"""}),
								        ("""attention_mask""", {0: """batch""", 1: """encoder_sequence"""}),
								        ("""decoder_input_ids""", {0: """batch""", 1: """decoder_sequence"""}),
								        ("""decoder_attention_mask""", {0: """batch""", 1: """decoder_sequence"""}),
								    ]      )

							return common_inputs


						@property
						def 				__snake_case    (					self      )	->	Mapping[str, Mapping[int, str]]:
							if self.task in ["default", "seq2seq-lm"]:
								lowerCAmelCase    =	super().outputs
							else:
								lowerCAmelCase    =	super(A_	,       self      ).outputs
								if self.use_past:
									lowerCAmelCase,   lowerCAmelCase    =	self.num_layers
									for i in range(A_      ):
										lowerCAmelCase    =	{0: """batch""", 2: """past_sequence + sequence"""}
										lowerCAmelCase    =	{0: """batch""", 2: """past_sequence + sequence"""}
							return common_outputs


						def 				__snake_case    (					self	,       A_	,       A_ = -1	,       A_ = -1	,       A_ = False	,       A_ = None	,       )	->	Mapping[str, Any]:
							lowerCAmelCase    =	self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
							    A_	,       A_	,       A_	,       A_	,       A_      )

							# Generate decoder inputs
							lowerCAmelCase    =	seq_length if not self.use_past else 1
							lowerCAmelCase    =	self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
							    A_	,       A_	,       A_	,       A_	,       A_      )
							lowerCAmelCase    =	{f'decoder_{name}': tensor for name, tensor in decoder_inputs.items()}
							lowerCAmelCase    =	dict(**A_	,       **A_      )

							if self.use_past:
								if not is_torch_available():
									raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""      )
								else:
									import torch
								lowerCAmelCase,   lowerCAmelCase    =	common_inputs["""input_ids"""].shape
								lowerCAmelCase    =	common_inputs["""decoder_input_ids"""].shape[1]
								lowerCAmelCase,   lowerCAmelCase    =	self.num_attention_heads
								lowerCAmelCase    =	(
								    batch,
								    num_encoder_attention_heads,
								    encoder_seq_length,
								    self._config.hidden_size // num_encoder_attention_heads,
								)
								lowerCAmelCase    =	decoder_seq_length + 3
								lowerCAmelCase    =	(
								    batch,
								    num_decoder_attention_heads,
								    decoder_past_length,
								    self._config.hidden_size // num_decoder_attention_heads,
								)

								lowerCAmelCase    =	torch.cat(
								    [common_inputs["""decoder_attention_mask"""], torch.ones(A_	,       A_      )]	,       dim=1      )

								lowerCAmelCase    =	[]
								# If the number of encoder and decoder layers are present in the model configuration, both are considered
								lowerCAmelCase,   lowerCAmelCase    =	self.num_layers
								lowerCAmelCase    =	min(A_	,       A_      )
								lowerCAmelCase    =	max(A_	,       A_      ) - min_num_layers
								lowerCAmelCase    =	"""encoder""" if num_encoder_layers > num_decoder_layers else """decoder"""

								for _ in range(A_      ):
									common_inputs["past_key_values"].append(
									    (
									        torch.zeros(A_      ),
									        torch.zeros(A_      ),
									        torch.zeros(A_      ),
									        torch.zeros(A_      ),
									    )      )
								# TODO: test this.
								lowerCAmelCase    =	encoder_shape if remaining_side_name == """encoder""" else decoder_shape
								for _ in range(A_	,       A_      ):
									common_inputs["past_key_values"].append((torch.zeros(A_      ), torch.zeros(A_      ))      )
							return common_inputs


						def 				__snake_case    (					self	,       A_	,       A_ = -1	,       A_ = -1	,       A_ = False	,       A_ = None	,       )	->	Mapping[str, Any]:
							lowerCAmelCase    =	self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
							    A_	,       A_	,       A_	,       A_	,       A_      )

							if self.use_past:
								if not is_torch_available():
									raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""      )
								else:
									import torch
								lowerCAmelCase,   lowerCAmelCase    =	common_inputs["""input_ids"""].shape
								# Not using the same length for past_key_values
								lowerCAmelCase    =	seqlen + 2
								lowerCAmelCase,   lowerCAmelCase    =	self.num_layers
								lowerCAmelCase,   lowerCAmelCase    =	self.num_attention_heads
								lowerCAmelCase    =	(
								    batch,
								    num_encoder_attention_heads,
								    past_key_values_length,
								    self._config.hidden_size // num_encoder_attention_heads,
								)

								lowerCAmelCase    =	common_inputs["""attention_mask"""].dtype
								lowerCAmelCase    =	torch.cat(
								    [common_inputs["""attention_mask"""], torch.ones(A_	,       A_	,       dtype=A_      )]	,       dim=1      )
								lowerCAmelCase    =	[
								    (torch.zeros(A_      ), torch.zeros(A_      )) for _ in range(A_      )
								]
							return common_inputs


						def 				__snake_case    (					self	,       A_	,       A_ = -1	,       A_ = -1	,       A_ = False	,       A_ = None	,       )	->	Mapping[str, Any]:
							# Copied from OnnxConfig.generate_dummy_inputs
							# Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
							# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
							lowerCAmelCase    =	compute_effective_axis_dimension(
							    A_	,       fixed_dimension=OnnxConfig.default_fixed_batch	,       num_token_to_add=0      )

							# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
							lowerCAmelCase    =	tokenizer.num_special_tokens_to_add(A_      )
							lowerCAmelCase    =	compute_effective_axis_dimension(
							    A_	,       fixed_dimension=OnnxConfig.default_fixed_sequence	,       num_token_to_add=A_      )

							# Generate dummy inputs according to compute batch and sequence
							lowerCAmelCase    =	[""" """.join([tokenizer.unk_token]      ) * seq_length] * batch_size
							lowerCAmelCase    =	dict(tokenizer(A_	,       return_tensors=A_      )      )
							return common_inputs


						def 				__snake_case    (					self	,       A_	,       A_ = -1	,       A_ = -1	,       A_ = False	,       A_ = None	,       )	->	Mapping[str, Any]:
							if self.task in ["default", "seq2seq-lm"]:
								lowerCAmelCase    =	self._generate_dummy_inputs_for_default_and_seqaseq_lm(
								    A_	,       batch_size=A_	,       seq_length=A_	,       is_pair=A_	,       framework=A_      )

							elif self.task == "causal-lm":
								lowerCAmelCase    =	self._generate_dummy_inputs_for_causal_lm(
								    A_	,       batch_size=A_	,       seq_length=A_	,       is_pair=A_	,       framework=A_      )
							else:
								lowerCAmelCase    =	self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
								    A_	,       batch_size=A_	,       seq_length=A_	,       is_pair=A_	,       framework=A_      )

							return common_inputs




						def 				__snake_case    (					self	,       A_	,       A_	,       A_	,       A_      )	->	Tuple:
							if self.task in ["default", "seq2seq-lm"]:
								lowerCAmelCase    =	super()._flatten_past_key_values_(A_	,       A_	,       A_	,       A_      )
							else:
								lowerCAmelCase    =	super(A_	,       self      )._flatten_past_key_values_(
								    A_	,       A_	,       A_	,       A_      )