






import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig







def       lowercase__	(					A_:						List[Any]  ,      A_:						Optional[int]  ,      A_:						Optional[Any]=0		)	->				Dict:







       """simple docstring"""






       if name is None:
              __UpperCAmelCase     				=None
       else:
              __UpperCAmelCase     				='.' * max(0  ,      spaces - 2		) + '# {:' + str(50 - spaces		) + 's}'
              __UpperCAmelCase     				=fmt.format(_SCREAMING_SNAKE_CASE		)

       # Print and recurse (if needed).
       if isinstance(_SCREAMING_SNAKE_CASE  ,      _SCREAMING_SNAKE_CASE		):
              if msg is not None:
                     print(_SCREAMING_SNAKE_CASE		)
              for k in val.keys():
                     recursive_print(_SCREAMING_SNAKE_CASE  ,      val[k]  ,      spaces + 2		)
       elif isinstance(_SCREAMING_SNAKE_CASE  ,      torch.Tensor		):
              print(_SCREAMING_SNAKE_CASE  ,      """:"""  ,      val.size()		)
       else:
              print(_SCREAMING_SNAKE_CASE  ,      """:"""  ,      _SCREAMING_SNAKE_CASE		)







def       lowercase__	(					A_:						List[str]  ,      A_:						Optional[Any]  ,      A_:						int  ,      A_:						str  ,      A_:						int		)	->				Optional[int]:







       """simple docstring"""






       __UpperCAmelCase     				=param.size()
       if checkpoint_version == 1.0:
              # version 1.0 stores [num_heads * hidden_size * num_splits, :]
              __UpperCAmelCase     				=(num_heads, hidden_size, num_splits) + input_shape[1:]
              __UpperCAmelCase     				=param.view(*_SCREAMING_SNAKE_CASE		)
              __UpperCAmelCase     				=param.transpose(0  ,      2		)
              __UpperCAmelCase     				=param.transpose(1  ,      2		).contiguous()
       elif checkpoint_version >= 2.0:
              # other versions store [num_heads * num_splits * hidden_size, :]
              __UpperCAmelCase     				=(num_heads, num_splits, hidden_size) + input_shape[1:]
              __UpperCAmelCase     				=param.view(*_SCREAMING_SNAKE_CASE		)
              __UpperCAmelCase     				=param.transpose(0  ,      1		).contiguous()
       __UpperCAmelCase     				=param.view(*_SCREAMING_SNAKE_CASE		)
       return param







def       lowercase__	(					A_:						Tuple  ,      A_:						int  ,      A_:						List[str]		)	->				Tuple:







       """simple docstring"""






       __UpperCAmelCase     				={}

       # old versions did not store training args
       __UpperCAmelCase     				=input_state_dict.get("""args"""  ,      _SCREAMING_SNAKE_CASE		)
       if ds_args is not None:
              # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
              # from pprint import pprint
              # pprint(vars(ds_args))

              __UpperCAmelCase     				=ds_args.padded_vocab_size
              __UpperCAmelCase     				=ds_args.max_position_embeddings
              __UpperCAmelCase     				=ds_args.hidden_size
              __UpperCAmelCase     				=ds_args.num_layers
              __UpperCAmelCase     				=ds_args.num_attention_heads
              __UpperCAmelCase     				=ds_args.ffn_hidden_size
              # pprint(config)

       # The number of heads.
       __UpperCAmelCase     				=config.n_head
       # The hidden_size per head.
       __UpperCAmelCase     				=config.n_embd // config.n_head
       # Megatron-LM checkpoint version
       if "checkpoint_version" in input_state_dict.keys():
              __UpperCAmelCase     				=input_state_dict['checkpoint_version']
       else:
              __UpperCAmelCase     				=0.0

       # The model.
       __UpperCAmelCase     				=input_state_dict['model']
       # The language model.
       __UpperCAmelCase     				=model['language_model']
       # The embeddings.
       __UpperCAmelCase     				=lm['embedding']

       # The word embeddings.
       __UpperCAmelCase     				=embeddings['word_embeddings']['weight']
       # Truncate the embedding table to vocab_size rows.
       __UpperCAmelCase     				=word_embeddings[: config.vocab_size, :]
       __UpperCAmelCase     				=word_embeddings

       # The position embeddings.
       __UpperCAmelCase     				=embeddings['position_embeddings']['weight']
       # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
       __UpperCAmelCase     				=pos_embeddings.size(0		)
       if n_positions != config.n_positions:
              raise ValueError(
                  F'''pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'''		)
       # Store the position embeddings.
       __UpperCAmelCase     				=pos_embeddings

       # The transformer.
       __UpperCAmelCase     				=lm['transformer'] if 'transformer' in lm.keys() else lm['encoder']

       # The regex to extract layer names.
       __UpperCAmelCase     				=re.compile(r"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"""		)

       # The simple map of names for "automated" rules.
       __UpperCAmelCase     				={
           'attention.dense': '.attn.c_proj.',
           'self_attention.dense': '.attn.c_proj.',
           'mlp.dense_h_to_4h': '.mlp.c_fc.',
           'mlp.dense_4h_to_h': '.mlp.c_proj.',
       }

       # Extract the layers.
       for key, val in transformer.items():
              # Match the name.
              __UpperCAmelCase     				=layer_re.match(_SCREAMING_SNAKE_CASE		)

              # Stop if that's not a layer
              if m is None:
                     break

              # The index of the layer.
              __UpperCAmelCase     				=int(m.group(1		)		)
              # The name of the operation.
              __UpperCAmelCase     				=m.group(2		)
              # Is it a weight or a bias?
              __UpperCAmelCase     				=m.group(3		)

              # The name of the layer.
              __UpperCAmelCase     				=F'''transformer.h.{layer_idx}'''

              # For layernorm(s), simply store the layer norm.
              if op_name.endswith("""layernorm"""		):
                     __UpperCAmelCase     				='ln_1' if op_name.startswith("""input"""		) else 'ln_2'
                     __UpperCAmelCase     				=val

              # Transpose the QKV matrix.
              elif (
                  op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
              ) and weight_or_bias == "weight":
                     # Insert a tensor of 1x1xDxD bias.
                     __UpperCAmelCase     				=torch.tril(torch.ones((n_positions, n_positions)  ,      dtype=torch.floataa		)		).view(
                         1  ,      1  ,      _SCREAMING_SNAKE_CASE  ,      _SCREAMING_SNAKE_CASE		)
                     __UpperCAmelCase     				=causal_mask

                     # Insert a "dummy" tensor for masked_bias.
                     __UpperCAmelCase     				=torch.tensor(-1e4  ,      dtype=torch.floataa		)
                     __UpperCAmelCase     				=masked_bias

                     __UpperCAmelCase     				=fix_query_key_value_ordering(_SCREAMING_SNAKE_CASE  ,      _SCREAMING_SNAKE_CASE  ,      3  ,      _SCREAMING_SNAKE_CASE  ,      _SCREAMING_SNAKE_CASE		)
                     # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
                     __UpperCAmelCase     				=out_val.transpose(0  ,      1		).contiguous()
                     # Store.
                     __UpperCAmelCase     				=out_val

              # Transpose the bias.
              elif (
                  op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
              ) and weight_or_bias == "bias":
                     __UpperCAmelCase     				=fix_query_key_value_ordering(_SCREAMING_SNAKE_CASE  ,      _SCREAMING_SNAKE_CASE  ,      3  ,      _SCREAMING_SNAKE_CASE  ,      _SCREAMING_SNAKE_CASE		)
                     # Store. No change of shape.
                     __UpperCAmelCase     				=out_val

              # Transpose the weights.
              elif weight_or_bias == "weight":
                     __UpperCAmelCase     				=megatron_to_transformers[op_name]
                     __UpperCAmelCase     				=val.transpose(0  ,      1		)

              # Copy the bias.
              elif weight_or_bias == "bias":
                     __UpperCAmelCase     				=megatron_to_transformers[op_name]
                     __UpperCAmelCase     				=val

              # DEBUG.
       assert config.n_layer == layer_idx + 1

       # The final layernorm.
       __UpperCAmelCase     				=transformer['final_layernorm.weight']
       __UpperCAmelCase     				=transformer['final_layernorm.bias']

       # For LM head, transformers' wants the matrix to weight embeddings.
       __UpperCAmelCase     				=word_embeddings

       # It should be done!
       return output_state_dict







def       lowercase__	(					)	->				int:







       """simple docstring"""






       __UpperCAmelCase     				=argparse.ArgumentParser()
       parser.add_argument("""--print-checkpoint-structure"""  ,      action="""store_true"""		)
       parser.add_argument(
           """path_to_checkpoint"""  ,      type=_SCREAMING_SNAKE_CASE  ,      help="""Path to the checkpoint file (.zip archive or direct .pt file)"""  ,      )
       parser.add_argument(
           """--config_file"""  ,      default=""""""  ,      type=_SCREAMING_SNAKE_CASE  ,      help="""An optional config json file describing the pre-trained model."""  ,      )
       __UpperCAmelCase     				=parser.parse_args()

       # Extract the basename.
       __UpperCAmelCase     				=os.path.dirname(args.path_to_checkpoint		)

       # Load the model.
       # the .zip is very optional, let's keep it for backward compatibility
       print(F'''Extracting PyTorch state dictionary from {args.path_to_checkpoint}'''		)
       if args.path_to_checkpoint.endswith(""".zip"""		):
              with zipfile.ZipFile(args.path_to_checkpoint  ,      """r"""		) as checkpoint:
                     with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt"""		) as pytorch_dict:
                            __UpperCAmelCase     				=torch.load(_SCREAMING_SNAKE_CASE  ,      map_location="""cpu"""		)
       else:
              __UpperCAmelCase     				=torch.load(args.path_to_checkpoint  ,      map_location="""cpu"""		)

       __UpperCAmelCase     				=input_state_dict.get("""args"""  ,      _SCREAMING_SNAKE_CASE		)

       # Read the config, or default to the model released by NVIDIA.
       if args.config_file == "":
              if ds_args is not None:
                     if ds_args.bias_gelu_fusion:
                            __UpperCAmelCase     				='gelu_fast'
                     elif ds_args.openai_gelu:
                            __UpperCAmelCase     				='gelu_new'
                     else:
                            __UpperCAmelCase     				='gelu'
              else:
                     # in the very early days this used to be "gelu_new"
                     __UpperCAmelCase     				='gelu_new'

              # Spell out all parameters in case the defaults change.
              __UpperCAmelCase     				=GPTaConfig(
                  vocab_size=50257  ,      n_positions=1024  ,      n_embd=1024  ,      n_layer=24  ,      n_head=16  ,      n_inner=4096  ,      activation_function=_SCREAMING_SNAKE_CASE  ,      resid_pdrop=0.1  ,      embd_pdrop=0.1  ,      attn_pdrop=0.1  ,      layer_norm_epsilon=1e-5  ,      initializer_range=0.0_2  ,      summary_type="""cls_index"""  ,      summary_use_proj=_SCREAMING_SNAKE_CASE  ,      summary_activation=_SCREAMING_SNAKE_CASE  ,      summary_proj_to_labels=_SCREAMING_SNAKE_CASE  ,      summary_first_dropout=0.1  ,      scale_attn_weights=_SCREAMING_SNAKE_CASE  ,      use_cache=_SCREAMING_SNAKE_CASE  ,      bos_token_id=50256  ,      eos_token_id=50256  ,      )
       else:
              __UpperCAmelCase     				=GPTaConfig.from_json_file(args.config_file		)

       __UpperCAmelCase     				=['GPT2LMHeadModel']

       # Convert.
       print("""Converting"""		)
       __UpperCAmelCase     				=convert_megatron_checkpoint(_SCREAMING_SNAKE_CASE  ,      _SCREAMING_SNAKE_CASE  ,      _SCREAMING_SNAKE_CASE		)

       # Print the structure of converted state dict.
       if args.print_checkpoint_structure:
              recursive_print(_SCREAMING_SNAKE_CASE  ,      _SCREAMING_SNAKE_CASE		)

       # Add tokenizer class info to config
       # see https://github.com/huggingface/transformers/issues/13906)
       if ds_args is not None:
              __UpperCAmelCase     				=ds_args.tokenizer_type
              if tokenizer_type == "GPT2BPETokenizer":
                     __UpperCAmelCase     				='gpt2'
              elif tokenizer_type == "PretrainedFromHF":
                     __UpperCAmelCase     				=ds_args.tokenizer_name_or_path
              else:
                     raise ValueError(F'''Unrecognized tokenizer_type {tokenizer_type}'''		)
       else:
              __UpperCAmelCase     				='gpt2'

       __UpperCAmelCase     				=AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE		)
       __UpperCAmelCase     				=type(_SCREAMING_SNAKE_CASE		).__name__
       __UpperCAmelCase     				=tokenizer_class

       # Store the config to file.
       print("""Saving config"""		)
       config.save_pretrained(_SCREAMING_SNAKE_CASE		)

       # Save tokenizer based on args
       print(F'''Adding {tokenizer_class} tokenizer files'''		)
       tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE		)

       # Store the state_dict to file.
       __UpperCAmelCase     				=os.path.join(_SCREAMING_SNAKE_CASE  ,      """pytorch_model.bin"""		)
       print(F'''Saving checkpoint to \"{output_checkpoint_file}\"'''		)
       torch.save(_SCREAMING_SNAKE_CASE  ,      _SCREAMING_SNAKE_CASE		)


####################################################################################################

if __name__ == "__main__":
   main()

####################################################################################################











"""simple docstring"""





import tempfile
import unittest

from make_student import create_student_by_copying_alternating_layers

from transformers import AutoConfig
from transformers.file_utils import cached_property
from transformers.testing_utils import require_torch


__A											=						"""sshleifer/bart-tiny-random"""
__A											=						"""patrickvonplaten/t5-tiny-random"""






@require_torch
class    _lowerCAmelCase  (	unittest.TestCase			):
       """simple docstring"""




       @cached_property
       def 	snake_case		(       self ):



         '''simple docstring'''

         return AutoConfig.from_pretrained(__UpperCAmelCase )






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__				,							*lowerCAmelCase__     :List[str]											=				create_student_by_copying_alternating_layers(__UpperCAmelCase       ,					tempfile.mkdtemp()       ,					e=1       ,					d=1 )
         self.assertEqual(student.config.num_hidden_layers       ,					1 )






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__				,							*lowerCAmelCase__     :Optional[int]											=				create_student_by_copying_alternating_layers(__UpperCAmelCase       ,					tempfile.mkdtemp()       ,					e=1       ,					d=__UpperCAmelCase )






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__				,							*lowerCAmelCase__     :List[Any]											=				create_student_by_copying_alternating_layers(__UpperCAmelCase       ,					tempfile.mkdtemp()       ,					e=1       ,					d=__UpperCAmelCase )
         self.assertEqual(student.config.encoder_layers       ,					1 )
         self.assertEqual(student.config.decoder_layers       ,					self.teacher_config.encoder_layers )






       def 	snake_case		(       self ):



         '''simple docstring'''

         lowerCAmelCase__				,							*lowerCAmelCase__     :Optional[int]											=				create_student_by_copying_alternating_layers(__UpperCAmelCase       ,					tempfile.mkdtemp()       ,					e=1       ,					d=1 )
         self.assertEqual(student.config.encoder_layers       ,					1 )
         self.assertEqual(student.config.decoder_layers       ,					1 )






       def 	snake_case		(       self ):



         '''simple docstring'''

         with self.assertRaises(__UpperCAmelCase ):
           create_student_by_copying_alternating_layers(__UpperCAmelCase       ,					tempfile.mkdtemp()       ,					e=__UpperCAmelCase       ,					d=__UpperCAmelCase )
