






'''simple docstring'''

# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig



def 							snake_case					(     a_							: Optional[Any]    ,   a_							: Dict    ,   a_							: Dict=0	)       ->				List[Any]:





      """simple docstring"""

      if name is None:
            UpperCamelCase_				:   Tuple					    =							None
      else:
            UpperCamelCase_				:   Any					    =							""".""" * max(0    ,   spaces - 2	) + """# {:""" + str(50 - spaces	) + """s}"""
            UpperCamelCase_				:   List[Any]					    =							fmt.format(__A	)

      # Print and recurse (if needed).
      if isinstance(__A    ,   __A	):
            if msg is not None:
                  print(__A	)
            for k in val.keys():
                  recursive_print(__A    ,   val[k]    ,   spaces + 2	)
      elif isinstance(__A    ,   torch.Tensor	):
            print(__A    ,   """:"""    ,   val.size()	)
      else:
            print(__A    ,   """:"""    ,   __A	)



def 							snake_case					(     a_							: int    ,   a_							: Union[str, Any]    ,   a_							: int    ,   a_							: int    ,   a_							: str	)       ->				int:





      """simple docstring"""

      UpperCamelCase_				:   Tuple					    =							param.size()
      if checkpoint_version == 1.0:
            # version 1.0 stores [num_heads * hidden_size * num_splits, :]
            UpperCamelCase_				:   Optional[int]					    =							(num_heads, hidden_size, num_splits) + input_shape[1:]
            UpperCamelCase_				:   str					    =							param.view(*__A	)
            UpperCamelCase_				:   Union[str, Any]					    =							param.transpose(0    ,   2	)
            UpperCamelCase_				:   int					    =							param.transpose(1    ,   2	).contiguous()
      elif checkpoint_version >= 2.0:
            # other versions store [num_heads * num_splits * hidden_size, :]
            UpperCamelCase_				:   Union[str, Any]					    =							(num_heads, num_splits, hidden_size) + input_shape[1:]
            UpperCamelCase_				:   Any					    =							param.view(*__A	)
            UpperCamelCase_				:   Tuple					    =							param.transpose(0    ,   1	).contiguous()
      UpperCamelCase_				:   Any					    =							param.view(*__A	)
      return param



def 							snake_case					(     a_							: List[Any]    ,   a_							: Tuple    ,   a_							: Tuple	)       ->				Any:





      """simple docstring"""

      UpperCamelCase_				:   Optional[Any]					    =							{}

      # old versions did not store training args
      UpperCamelCase_				:   List[Any]					    =							input_state_dict.get("""args"""    ,   __A	)
      if ds_args is not None:
            # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
            # from pprint import pprint
            # pprint(vars(ds_args))

            UpperCamelCase_				:   Optional[Any]					    =							ds_args.padded_vocab_size
            UpperCamelCase_				:   Optional[int]					    =							ds_args.max_position_embeddings
            UpperCamelCase_				:   Any					    =							ds_args.hidden_size
            UpperCamelCase_				:   Any					    =							ds_args.num_layers
            UpperCamelCase_				:   Optional[int]					    =							ds_args.num_attention_heads
            UpperCamelCase_				:   Union[str, Any]					    =							ds_args.ffn_hidden_size
            # pprint(config)

      # The number of heads.
      UpperCamelCase_				:   Dict					    =							config.n_head
      # The hidden_size per head.
      UpperCamelCase_				:   Union[str, Any]					    =							config.n_embd // config.n_head
      # Megatron-LM checkpoint version
      if "checkpoint_version" in input_state_dict.keys():
            UpperCamelCase_				:   Any					    =							input_state_dict["""checkpoint_version"""]
      else:
            UpperCamelCase_				:   List[Any]					    =							0.0

      # The model.
      UpperCamelCase_				:   Any					    =							input_state_dict["""model"""]
      # The language model.
      UpperCamelCase_				:   str					    =							model["""language_model"""]
      # The embeddings.
      UpperCamelCase_				:   Optional[Any]					    =							lm["""embedding"""]

      # The word embeddings.
      UpperCamelCase_				:   int					    =							embeddings["""word_embeddings"""]["""weight"""]
      # Truncate the embedding table to vocab_size rows.
      UpperCamelCase_				:   int					    =							word_embeddings[: config.vocab_size, :]
      UpperCamelCase_				:   List[str]					    =							word_embeddings

      # The position embeddings.
      UpperCamelCase_				:   Dict					    =							embeddings["""position_embeddings"""]["""weight"""]
      # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
      UpperCamelCase_				:   Union[str, Any]					    =							pos_embeddings.size(0	)
      if n_positions != config.n_positions:
            raise ValueError(
                f"pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match"	)
      # Store the position embeddings.
      UpperCamelCase_				:   Optional[int]					    =							pos_embeddings

      # The transformer.
      UpperCamelCase_				:   Union[str, Any]					    =							lm["""transformer"""] if """transformer""" in lm.keys() else lm["""encoder"""]

      # The regex to extract layer names.
      UpperCamelCase_				:   Tuple					    =							re.compile(r"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"""	)

      # The simple map of names for "automated" rules.
      UpperCamelCase_				:   str					    =							{
          """attention.dense""": """.attn.c_proj.""",
          """self_attention.dense""": """.attn.c_proj.""",
          """mlp.dense_h_to_4h""": """.mlp.c_fc.""",
          """mlp.dense_4h_to_h""": """.mlp.c_proj.""",
      }

      # Extract the layers.
      for key, val in transformer.items():
            # Match the name.
            UpperCamelCase_				:   Union[str, Any]					    =							layer_re.match(__A	)

            # Stop if that's not a layer
            if m is None:
                  break

            # The index of the layer.
            UpperCamelCase_				:   Union[str, Any]					    =							int(m.group(1	)	)
            # The name of the operation.
            UpperCamelCase_				:   Optional[Any]					    =							m.group(2	)
            # Is it a weight or a bias?
            UpperCamelCase_				:   str					    =							m.group(3	)

            # The name of the layer.
            UpperCamelCase_				:   List[str]					    =							f"transformer.h.{layer_idx}"

            # For layernorm(s), simply store the layer norm.
            if op_name.endswith("""layernorm"""	):
                  UpperCamelCase_				:   List[Any]					    =							"""ln_1""" if op_name.startswith("""input"""	) else """ln_2"""
                  UpperCamelCase_				:   Optional[Any]					    =							val

            # Transpose the QKV matrix.
            elif (
                op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
            ) and weight_or_bias == "weight":
                  # Insert a tensor of 1x1xDxD bias.
                  UpperCamelCase_				:   Dict					    =							torch.tril(torch.ones((n_positions, n_positions)    ,   dtype=torch.floataa	)	).view(
                      1    ,   1    ,   __A    ,   __A	)
                  UpperCamelCase_				:   Optional[int]					    =							causal_mask

                  # Insert a "dummy" tensor for masked_bias.
                  UpperCamelCase_				:   List[str]					    =							torch.tensor(-1e4    ,   dtype=torch.floataa	)
                  UpperCamelCase_				:   Tuple					    =							masked_bias

                  UpperCamelCase_				:   Union[str, Any]					    =							fix_query_key_value_ordering(__A    ,   __A    ,   3    ,   __A    ,   __A	)
                  # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
                  UpperCamelCase_				:   List[str]					    =							out_val.transpose(0    ,   1	).contiguous()
                  # Store.
                  UpperCamelCase_				:   List[Any]					    =							out_val

            # Transpose the bias.
            elif (
                op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
            ) and weight_or_bias == "bias":
                  UpperCamelCase_				:   Any					    =							fix_query_key_value_ordering(__A    ,   __A    ,   3    ,   __A    ,   __A	)
                  # Store. No change of shape.
                  UpperCamelCase_				:   List[Any]					    =							out_val

            # Transpose the weights.
            elif weight_or_bias == "weight":
                  UpperCamelCase_				:   int					    =							megatron_to_transformers[op_name]
                  UpperCamelCase_				:   Any					    =							val.transpose(0    ,   1	)

            # Copy the bias.
            elif weight_or_bias == "bias":
                  UpperCamelCase_				:   List[Any]					    =							megatron_to_transformers[op_name]
                  UpperCamelCase_				:   List[Any]					    =							val

            # DEBUG.
      assert config.n_layer == layer_idx + 1

      # The final layernorm.
      UpperCamelCase_				:   Optional[Any]					    =							transformer["""final_layernorm.weight"""]
      UpperCamelCase_				:   List[str]					    =							transformer["""final_layernorm.bias"""]

      # For LM head, transformers' wants the matrix to weight embeddings.
      UpperCamelCase_				:   Dict					    =							word_embeddings

      # It should be done!
      return output_state_dict



def 							snake_case					(     )       ->				List[str]:





      """simple docstring"""

      UpperCamelCase_				:   Union[str, Any]					    =							argparse.ArgumentParser()
      parser.add_argument("""--print-checkpoint-structure"""    ,   action="""store_true"""	)
      parser.add_argument(
          """path_to_checkpoint"""    ,   type=__A    ,   help="""Path to the checkpoint file (.zip archive or direct .pt file)"""    ,   )
      parser.add_argument(
          """--config_file"""    ,   default=""""""    ,   type=__A    ,   help="""An optional config json file describing the pre-trained model."""    ,   )
      UpperCamelCase_				:   int					    =							parser.parse_args()

      # Extract the basename.
      UpperCamelCase_				:   List[str]					    =							os.path.dirname(args.path_to_checkpoint	)

      # Load the model.
      # the .zip is very optional, let's keep it for backward compatibility
      print(f"Extracting PyTorch state dictionary from {args.path_to_checkpoint}"	)
      if args.path_to_checkpoint.endswith(""".zip"""	):
            with zipfile.ZipFile(args.path_to_checkpoint    ,   """r"""	) as checkpoint:
                  with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt"""	) as pytorch_dict:
                        UpperCamelCase_				:   Tuple					    =							torch.load(__A    ,   map_location="""cpu"""	)
      else:
            UpperCamelCase_				:   Union[str, Any]					    =							torch.load(args.path_to_checkpoint    ,   map_location="""cpu"""	)

      UpperCamelCase_				:   Any					    =							input_state_dict.get("""args"""    ,   __A	)

      # Read the config, or default to the model released by NVIDIA.
      if args.config_file == "":
            if ds_args is not None:
                  if ds_args.bias_gelu_fusion:
                        UpperCamelCase_				:   Optional[Any]					    =							"""gelu_fast"""
                  elif ds_args.openai_gelu:
                        UpperCamelCase_				:   Optional[int]					    =							"""gelu_new"""
                  else:
                        UpperCamelCase_				:   List[Any]					    =							"""gelu"""
            else:
                  # in the very early days this used to be "gelu_new"
                  UpperCamelCase_				:   str					    =							"""gelu_new"""

            # Spell out all parameters in case the defaults change.
            UpperCamelCase_				:   Union[str, Any]					    =							GPTaConfig(
                vocab_size=50_257    ,   n_positions=1_024    ,   n_embd=1_024    ,   n_layer=24    ,   n_head=16    ,   n_inner=4_096    ,   activation_function=__A    ,   resid_pdrop=0.1    ,   embd_pdrop=0.1    ,   attn_pdrop=0.1    ,   layer_norm_epsilon=1e-5    ,   initializer_range=0.02    ,   summary_type="""cls_index"""    ,   summary_use_proj=__A    ,   summary_activation=__A    ,   summary_proj_to_labels=__A    ,   summary_first_dropout=0.1    ,   scale_attn_weights=__A    ,   use_cache=__A    ,   bos_token_id=50_256    ,   eos_token_id=50_256    ,   )
      else:
            UpperCamelCase_				:   List[Any]					    =							GPTaConfig.from_json_file(args.config_file	)

      UpperCamelCase_				:   int					    =							["""GPT2LMHeadModel"""]

      # Convert.
      print("""Converting"""	)
      UpperCamelCase_				:   Optional[Any]					    =							convert_megatron_checkpoint(__A    ,   __A    ,   __A	)

      # Print the structure of converted state dict.
      if args.print_checkpoint_structure:
            recursive_print(__A    ,   __A	)

      # Add tokenizer class info to config
      # see https://github.com/huggingface/transformers/issues/13906)
      if ds_args is not None:
            UpperCamelCase_				:   List[str]					    =							ds_args.tokenizer_type
            if tokenizer_type == "GPT2BPETokenizer":
                  UpperCamelCase_				:   Union[str, Any]					    =							"""gpt2"""
            elif tokenizer_type == "PretrainedFromHF":
                  UpperCamelCase_				:   List[Any]					    =							ds_args.tokenizer_name_or_path
            else:
                  raise ValueError(f"Unrecognized tokenizer_type {tokenizer_type}"	)
      else:
            UpperCamelCase_				:   Union[str, Any]					    =							"""gpt2"""

      UpperCamelCase_				:   str					    =							AutoTokenizer.from_pretrained(__A	)
      UpperCamelCase_				:   Any					    =							type(__A	).__name__
      UpperCamelCase_				:   int					    =							tokenizer_class

      # Store the config to file.
      print("""Saving config"""	)
      config.save_pretrained(__A	)

      # Save tokenizer based on args
      print(f"Adding {tokenizer_class} tokenizer files"	)
      tokenizer.save_pretrained(__A	)

      # Store the state_dict to file.
      UpperCamelCase_				:   Dict					    =							os.path.join(__A    ,   """pytorch_model.bin"""	)
      print(f"Saving checkpoint to \"{output_checkpoint_file}\""	)
      torch.save(__A    ,   __A	)


####################################################################################################

if __name__ == "__main__":
      main()

####################################################################################################














'''simple docstring'''

import inspect
import os
import unittest
from dataclasses import dataclass

import torch

from accelerate import Accelerator, DistributedDataParallelKwargs, GradScalerKwargs
from accelerate.state import AcceleratorState
from accelerate.test_utils import execute_subprocess_async, require_cuda, require_multi_gpu
from accelerate.utils import KwargsHandler




@dataclass
class     A    (      SCREAMING_SNAKE_CASE__						):






 """simple docstring"""





 __a			:						int      =						0
 __a			:						bool      =						False
 __a			:						float      =						3.0






class     A    (      unittest.TestCase						):






 """simple docstring"""





 def   _UpperCAmelCase       (							self ):
       # If no defaults are changed, `to_kwargs` returns an empty dict.
       self.assertDictEqual(MockClass().to_kwargs()			,     {} )
       self.assertDictEqual(MockClass(a=2 ).to_kwargs()			,     {"""a""": 2} )
       self.assertDictEqual(MockClass(a=2			,     b=__lowerCAmelCase ).to_kwargs()			,     {"""a""": 2, """b""": True} )
       self.assertDictEqual(MockClass(a=2			,     c=2.25 ).to_kwargs()			,     {"""a""": 2, """c""": 2.25} )
 @require_cuda
 def   _UpperCAmelCase       (							self ):
       # If no defaults are changed, `to_kwargs` returns an empty dict.
       UpperCamelCase_				:   int					    =							GradScalerKwargs(init_scale=10_24			,     growth_factor=2 )
       AcceleratorState._reset_state()
       UpperCamelCase_				:   Dict					    =							Accelerator(mixed_precision="""fp16"""			,     kwargs_handlers=[scaler_handler] )
       print(accelerator.use_fpaa )
       UpperCamelCase_				:   Any					    =							accelerator.scaler

       # Check the kwargs have been applied
       self.assertEqual(scaler._init_scale			,     10_24.0 )
       self.assertEqual(scaler._growth_factor			,     2.0 )

       # Check the other values are at the default
       self.assertEqual(scaler._backoff_factor			,     0.5 )
       self.assertEqual(scaler._growth_interval			,     20_00 )
       self.assertEqual(scaler._enabled			,     __lowerCAmelCase )





 @require_multi_gpu
 def   _UpperCAmelCase       (							self ):
       UpperCamelCase_				:   Optional[int]					    =							["""torchrun""", F"--nproc_per_node={torch.cuda.device_count()}", inspect.getfile(self.__class__ )]
       execute_subprocess_async(__lowerCAmelCase			,     env=os.environ.copy() )


if __name__ == "__main__":
      UpperCamelCase							=DistributedDataParallelKwargs(bucket_cap_mb=15, find_unused_parameters=True)
      UpperCamelCase							=Accelerator(kwargs_handlers=[ddp_scaler])
      UpperCamelCase							=torch.nn.Linear(100, 200)
      UpperCamelCase							=accelerator.prepare(model)

      # Check the values changed in kwargs
      UpperCamelCase							=""
      UpperCamelCase							=model.bucket_bytes_cap // (1024 * 1024)
      if observed_bucket_cap_map != 15:
            error_msg += f"Kwargs badly passed, should have `15` but found {observed_bucket_cap_map}.\n"
      if model.find_unused_parameters is not True:
            error_msg += f"Kwargs badly passed, should have `True` but found {model.find_unused_parameters}.\n"

      # Check the values of the defaults
      if model.dim != 0:
            error_msg += f"Default value not respected, should have `0` but found {model.dim}.\n"
      if model.broadcast_buffers is not True:
            error_msg += f"Default value not respected, should have `True` but found {model.broadcast_buffers}.\n"
      if model.gradient_as_bucket_view is not False:
            error_msg += f"Default value not respected, should have `False` but found {model.gradient_as_bucket_view}.\n"

      # Raise error at the end to make sure we don't stop at the first failure.
      if len(error_msg) > 0:
            raise ValueError(error_msg)






