





"""simple docstring"""





import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig




def 		lowerCAmelCase_	(  lowercase_   :							Any ,      lowercase_   :							Union[str, Any] ,      lowercase_   :							Dict=0		):

 '''simple docstring'''






 if name is None:
  __SCREAMING_SNAKE_CASE :			List[Any]												=       None
 else:
  __SCREAMING_SNAKE_CASE :			Optional[Any]												=       '''.''' * max(0 ,      spaces - 2		) + '''# {:''' + str(50 - spaces		) + '''s}'''
  __SCREAMING_SNAKE_CASE :			Optional[int]												=       fmt.format(lowercase_		)

 # Print and recurse (if needed).
 if isinstance(lowercase_ ,      lowercase_		):
  if msg is not None:
   print(lowercase_		)
  for k in val.keys():
   recursive_print(lowercase_ ,      val[k] ,      spaces + 2		)
 elif isinstance(lowercase_ ,      torch.Tensor		):
  print(lowercase_ ,      ''':''' ,      val.size()		)
 else:
  print(lowercase_ ,      ''':''' ,      lowercase_		)




def 		lowerCAmelCase_	(  lowercase_   :							int ,      lowercase_   :							Optional[Any] ,      lowercase_   :							str ,      lowercase_   :							Union[str, Any]		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			List[str]												=       param.size()
 # other versions store [num_heads * num_splits * hidden_size, :]
 __SCREAMING_SNAKE_CASE :			List[Any]												=       (num_heads, num_splits, hidden_size) + input_shape[1:]
 __SCREAMING_SNAKE_CASE :			str												=       param.view(*lowercase_		)
 __SCREAMING_SNAKE_CASE :			int												=       param.transpose(0 ,      1		).contiguous()
 __SCREAMING_SNAKE_CASE :			Optional[int]												=       param.view(*lowercase_		)
 return param




def 		lowerCAmelCase_	(  lowercase_   :							List[Any] ,      lowercase_   :							Tuple		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			List[str]												=       config.n_positions
 __SCREAMING_SNAKE_CASE :			str												=       config.n_layer
 __SCREAMING_SNAKE_CASE :			List[str]												=       config.vocab_size
 __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       config.n_head
 __SCREAMING_SNAKE_CASE :			str												=       config.n_embd // config.n_head

 __SCREAMING_SNAKE_CASE :			int												=       sd_megatron['''model.language_model.embedding.word_embeddings.weight'''][:vocab_size, :]
 __SCREAMING_SNAKE_CASE :			Any												=       {
     '''transformer.wte.weight''': word_embeddings,
     '''transformer.wpe.weight''': sd_megatron['''model.language_model.embedding.position_embeddings.weight'''],
     '''transformer.ln_f.weight''': sd_megatron['''model.language_model.encoder.final_layernorm.weight'''],
     '''transformer.ln_f.bias''': sd_megatron['''model.language_model.encoder.final_layernorm.bias'''],
 }

 __SCREAMING_SNAKE_CASE :			Any												=       '''model.language_model.encoder.layers.'''
 for i in range(lowercase_		):
  __SCREAMING_SNAKE_CASE :			List[str]												=       torch.tril(torch.ones((n_positions, n_positions) ,      dtype=torch.bool		)		)
  __SCREAMING_SNAKE_CASE :			Tuple												=       causal_mask.view(1 ,      1 ,      lowercase_ ,      lowercase_		)
  __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       causal_mask
  __SCREAMING_SNAKE_CASE :			Any												=       torch.tensor(-1E4 ,      dtype=torch.bfloataa		)

  __SCREAMING_SNAKE_CASE :			Dict												=       sd_megatron[F'''{pf}{i}.input_layernorm.weight''']
  __SCREAMING_SNAKE_CASE :			Optional[int]												=       sd_megatron[F'''{pf}{i}.input_layernorm.bias''']

  __SCREAMING_SNAKE_CASE :			Optional[Any]												=       sd_megatron[F'''{pf}{i}.self_attention.query_key_value.weight''']
  __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       fix_query_key_value_ordering(lowercase_ ,      3 ,      lowercase_ ,      lowercase_		)
  __SCREAMING_SNAKE_CASE :			List[str]												=       vala.transpose(0 ,      1		).contiguous()

  __SCREAMING_SNAKE_CASE :			Optional[Any]												=       sd_megatron[F'''{pf}{i}.self_attention.query_key_value.bias''']
  __SCREAMING_SNAKE_CASE :			List[str]												=       fix_query_key_value_ordering(lowercase_ ,      3 ,      lowercase_ ,      lowercase_		)
  __SCREAMING_SNAKE_CASE :			int												=       vala

  __SCREAMING_SNAKE_CASE :			str												=       sd_megatron[F'''{pf}{i}.self_attention.dense.weight'''].transpose(
      0 ,      1		)
  __SCREAMING_SNAKE_CASE :			Any												=       sd_megatron[F'''{pf}{i}.self_attention.dense.bias''']
  __SCREAMING_SNAKE_CASE :			List[Any]												=       sd_megatron[F'''{pf}{i}.post_attention_layernorm.weight''']
  __SCREAMING_SNAKE_CASE :			Optional[Any]												=       sd_megatron[F'''{pf}{i}.post_attention_layernorm.bias''']
  __SCREAMING_SNAKE_CASE :			Dict												=       sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.weight'''].transpose(0 ,      1		)
  __SCREAMING_SNAKE_CASE :			Tuple												=       sd_megatron[F'''{pf}{i}.mlp.dense_h_to_4h.bias''']
  __SCREAMING_SNAKE_CASE :			Optional[Any]												=       sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.weight'''].transpose(
      0 ,      1		)
  __SCREAMING_SNAKE_CASE :			List[str]												=       sd_megatron[F'''{pf}{i}.mlp.dense_4h_to_h.bias''']

 # For LM head, transformers' wants the matrix to weight embeddings.
 __SCREAMING_SNAKE_CASE :			Optional[int]												=       word_embeddings

 return sd_hf




def 		lowerCAmelCase_	(  lowercase_   :							Dict ,      lowercase_   :							Union[str, Any]		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			List[Any]												=       6_4000
 __SCREAMING_SNAKE_CASE :			Optional[Any]												=       config_megatron['''encoder_seq_length''']
 __SCREAMING_SNAKE_CASE :			Tuple												=       config_megatron['''hidden_size''']
 __SCREAMING_SNAKE_CASE :			Optional[int]												=       config_megatron['''num_layers''']
 __SCREAMING_SNAKE_CASE :			List[Any]												=       config_megatron['''num_attention_heads''']
 __SCREAMING_SNAKE_CASE :			Any												=       config_megatron['''ffn_hidden_size''']
 __SCREAMING_SNAKE_CASE :			Optional[Any]												=       '''gelu'''
 __SCREAMING_SNAKE_CASE :			Optional[Any]												=       0.1
 __SCREAMING_SNAKE_CASE :			Any												=       0.1
 __SCREAMING_SNAKE_CASE :			Any												=       0.1
 __SCREAMING_SNAKE_CASE :			List[str]												=       config_megatron['''layernorm_epsilon''']  # 1e-5
 __SCREAMING_SNAKE_CASE :			str												=       config_megatron['''init_method_std''']  # 0.02
 __SCREAMING_SNAKE_CASE :			Dict												=       config_megatron['''apply_query_key_layer_scaling''']  # True
 __SCREAMING_SNAKE_CASE :			List[Any]												=       True
 __SCREAMING_SNAKE_CASE :			Any												=       True

 # This identifies the 6.7B (7B) model which uses a different tokenizer
 if config_megatron["hidden_size"] == 4096:
  __SCREAMING_SNAKE_CASE :			str												=       1  # <|endoftext|>
  __SCREAMING_SNAKE_CASE :			Optional[Any]												=       1  # <|endoftext|>
  __SCREAMING_SNAKE_CASE :			List[str]												=       0  # <unk>
 else:
  __SCREAMING_SNAKE_CASE :			str												=       2  # <s>
  __SCREAMING_SNAKE_CASE :			Optional[Any]												=       3  # <|endoftext|>
  __SCREAMING_SNAKE_CASE :			Dict												=       0  # <pad>

 return config_hf




def 		lowerCAmelCase_	(  lowercase_   :							Any		):

 '''simple docstring'''






 print(lowercase_		)

 __SCREAMING_SNAKE_CASE :			Any												=       args.checkpoint_path
 __SCREAMING_SNAKE_CASE :			Optional[Any]												=       args.save_path
 if isfile(lowercase_		):
  raise FileNotFoundError(F'''ERROR! could not find file {checkpoint_path}'''		)

 # Load the model.
 __SCREAMING_SNAKE_CASE :			Any												=       torch.load(lowercase_ ,      map_location='''cpu'''		)

 # Load the config.
 __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       checkpoint['''hyper_parameters''']['''cfg''']
 __SCREAMING_SNAKE_CASE :			str												=       GPTaConfig()
 __SCREAMING_SNAKE_CASE :			Optional[Any]												=       copy_config(config_hf=lowercase_ ,      config_megatron=lowercase_		)
 __SCREAMING_SNAKE_CASE :			str												=       ['''GPT2LMHeadModel''']

 __SCREAMING_SNAKE_CASE :			Tuple												=       checkpoint['''state_dict''']

 # Convert.
 print('''Converting'''		)
 __SCREAMING_SNAKE_CASE :			Any												=       convert_megatron_checkpoint(lowercase_ ,      lowercase_		)

 # Print the structure of converted state dict.
 if args.print_checkpoint_structure:
  recursive_print(lowercase_ ,      lowercase_		)

 __SCREAMING_SNAKE_CASE :			List[str]												=       '''GPTSw3Tokenizer'''

 # Store the config to file.
 print('''Saving config'''		)
 config_hf.save_pretrained(lowercase_		)

 # Store the state_dict to file.
 __SCREAMING_SNAKE_CASE :			int												=       os.path.join(lowercase_ ,      '''pytorch_model.bin'''		)
 print(F'''Saving checkpoint to "{output_checkpoint_file}"'''		)
 torch.save(lowercase_ ,      lowercase_		)


if __name__ == "__main__":
    _lowerCamelCase       =	argparse.ArgumentParser()
    parser.add_argument(
        '''--checkpoint_path''',
        type=str,
        required=True,
        help='''e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000''',
    )
    parser.add_argument('''--save_path''', type=str, required=True, help='''e.g. /home/user/gpt-sw3/hf''')
    parser.add_argument('''--print-checkpoint-structure''', action='''store_true''')
    _lowerCamelCase       =	parser.parse_args()
    main(_args)








"""simple docstring"""





import numpy as np




def 		lowerCAmelCase_	(  lowercase_   :							str		):

 '''simple docstring'''






 __SCREAMING_SNAKE_CASE :			Dict												=       np.exp(lowercase_		)

 # Add up the all the exponentials
 __SCREAMING_SNAKE_CASE :			Tuple												=       np.sum(lowercase_		)

 # Divide every exponent by the sum of all exponents
 __SCREAMING_SNAKE_CASE :			List[Any]												=       exponent_vector / sum_of_exponents

 return softmax_vector


if __name__ == "__main__":
    print(softmax((0,)))

