






# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig





def       __lowerCAmelCase      (						__snake_case					, __snake_case					, __snake_case=0       ):
   # Format the message.
   if name is None:
      __lowerCAmelCase								=					None
   else:
      __lowerCAmelCase								=					"." * max(0					, spaces - 2       ) + "# {:" + str(50 - spaces       ) + "s}"
      __lowerCAmelCase								=					fmt.format(__snake_case       )

   # Print and recurse (if needed).
   if isinstance(__snake_case					, __snake_case       ):
      if msg is not None:
         print(__snake_case       )
      for k in val.keys():
         recursive_print(__snake_case					, val[k]					, spaces + 2       )
   elif isinstance(__snake_case					, torch.Tensor       ):
      print(__snake_case					, ":"					, val.size()       )
   else:
      print(__snake_case					, ":"					, __snake_case       )





def       __lowerCAmelCase      (						__snake_case					, __snake_case					, __snake_case					, __snake_case					, __snake_case       ):
   # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
   # for compatibility with later versions of NVIDIA Megatron-LM.
   # The inverse operation is performed inside Megatron-LM to read checkpoints:
   # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
   # If param is the weight tensor of the self-attention block, the returned tensor
   # will have to be transposed one more time to be read by HuggingFace GPT2.
   __lowerCAmelCase								=					param.size()
   if checkpoint_version == 1.0:
      # version 1.0 stores [num_heads * hidden_size * num_splits, :]
      __lowerCAmelCase								=					(num_heads, hidden_size, num_splits) + input_shape[1:]
      __lowerCAmelCase								=					param.view(*__snake_case       )
      __lowerCAmelCase								=					param.transpose(0					, 2       )
      __lowerCAmelCase								=					param.transpose(1					, 2       ).contiguous()
   elif checkpoint_version >= 2.0:
      # other versions store [num_heads * num_splits * hidden_size, :]
      __lowerCAmelCase								=					(num_heads, num_splits, hidden_size) + input_shape[1:]
      __lowerCAmelCase								=					param.view(*__snake_case       )
      __lowerCAmelCase								=					param.transpose(0					, 1       ).contiguous()
   __lowerCAmelCase								=					param.view(*__snake_case       )
   return param





def       __lowerCAmelCase      (						__snake_case					, __snake_case					, __snake_case       ):
   # The converted output model.
   __lowerCAmelCase								=					{}

   # old versions did not store training args
   __lowerCAmelCase								=					input_state_dict.get("args"					, __snake_case       )
   if ds_args is not None:
      # do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
      # from pprint import pprint
      # pprint(vars(ds_args))

      __lowerCAmelCase								=					ds_args.padded_vocab_size
      __lowerCAmelCase								=					ds_args.max_position_embeddings
      __lowerCAmelCase								=					ds_args.hidden_size
      __lowerCAmelCase								=					ds_args.num_layers
      __lowerCAmelCase								=					ds_args.num_attention_heads
      __lowerCAmelCase								=					ds_args.ffn_hidden_size
      # pprint(config)

   # The number of heads.
   __lowerCAmelCase								=					config.n_head
   # The hidden_size per head.
   __lowerCAmelCase								=					config.n_embd // config.n_head
   # Megatron-LM checkpoint version
   if "checkpoint_version" in input_state_dict.keys():
      __lowerCAmelCase								=					input_state_dict["checkpoint_version"]
   else:
      __lowerCAmelCase								=					0.0

   # The model.
   __lowerCAmelCase								=					input_state_dict["model"]
   # The language model.
   __lowerCAmelCase								=					model["language_model"]
   # The embeddings.
   __lowerCAmelCase								=					lm["embedding"]

   # The word embeddings.
   __lowerCAmelCase								=					embeddings["word_embeddings"]["weight"]
   # Truncate the embedding table to vocab_size rows.
   __lowerCAmelCase								=					word_embeddings[: config.vocab_size, :]
   __lowerCAmelCase								=					word_embeddings

   # The position embeddings.
   __lowerCAmelCase								=					embeddings["position_embeddings"]["weight"]
   # Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
   __lowerCAmelCase								=					pos_embeddings.size(0       )
   if n_positions != config.n_positions:
      raise ValueError(
          F"""pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don't match"""       )
   # Store the position embeddings.
   __lowerCAmelCase								=					pos_embeddings

   # The transformer.
   __lowerCAmelCase								=					lm["transformer"] if "transformer" in lm.keys() else lm["encoder"]

   # The regex to extract layer names.
   __lowerCAmelCase								=					re.compile(r"layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"       )

   # The simple map of names for "automated" rules.
   __lowerCAmelCase								=					{
       "attention.dense": ".attn.c_proj.",
       "self_attention.dense": ".attn.c_proj.",
       "mlp.dense_h_to_4h": ".mlp.c_fc.",
       "mlp.dense_4h_to_h": ".mlp.c_proj.",
   }

   # Extract the layers.
   for key, val in transformer.items():
      # Match the name.
      __lowerCAmelCase								=					layer_re.match(__snake_case       )

      # Stop if that's not a layer
      if m is None:
         break

      # The index of the layer.
      __lowerCAmelCase								=					int(m.group(1       )       )
      # The name of the operation.
      __lowerCAmelCase								=					m.group(2       )
      # Is it a weight or a bias?
      __lowerCAmelCase								=					m.group(3       )

      # The name of the layer.
      __lowerCAmelCase								=					F"""transformer.h.{layer_idx}"""

      # For layernorm(s), simply store the layer norm.
      if op_name.endswith("layernorm"       ):
         __lowerCAmelCase								=					"ln_1" if op_name.startswith("input"       ) else "ln_2"
         __lowerCAmelCase								=					val

      # Transpose the QKV matrix.
      elif (
          op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
      ) and weight_or_bias == "weight":
         # Insert a tensor of 1x1xDxD bias.
         __lowerCAmelCase								=					torch.tril(torch.ones((n_positions, n_positions)					, dtype=torch.floataa       )       ).view(
             1					, 1					, __snake_case					, __snake_case       )
         __lowerCAmelCase								=					causal_mask

         # Insert a "dummy" tensor for masked_bias.
         __lowerCAmelCase								=					torch.tensor(-1E4					, dtype=torch.floataa       )
         __lowerCAmelCase								=					masked_bias

         __lowerCAmelCase								=					fix_query_key_value_ordering(__snake_case					, __snake_case					, 3					, __snake_case					, __snake_case       )
         # Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
         __lowerCAmelCase								=					out_val.transpose(0					, 1       ).contiguous()
         # Store.
         __lowerCAmelCase								=					out_val

      # Transpose the bias.
      elif (
          op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
      ) and weight_or_bias == "bias":
         __lowerCAmelCase								=					fix_query_key_value_ordering(__snake_case					, __snake_case					, 3					, __snake_case					, __snake_case       )
         # Store. No change of shape.
         __lowerCAmelCase								=					out_val

      # Transpose the weights.
      elif weight_or_bias == "weight":
         __lowerCAmelCase								=					megatron_to_transformers[op_name]
         __lowerCAmelCase								=					val.transpose(0					, 1       )

      # Copy the bias.
      elif weight_or_bias == "bias":
         __lowerCAmelCase								=					megatron_to_transformers[op_name]
         __lowerCAmelCase								=					val

    # DEBUG.
   assert config.n_layer == layer_idx + 1

   # The final layernorm.
   __lowerCAmelCase								=					transformer["final_layernorm.weight"]
   __lowerCAmelCase								=					transformer["final_layernorm.bias"]

   # For LM head, transformers' wants the matrix to weight embeddings.
   __lowerCAmelCase								=					word_embeddings

   # It should be done!
   return output_state_dict





def       __lowerCAmelCase      (						):
   # Create the argument parser.
   __lowerCAmelCase								=					argparse.ArgumentParser()
   parser.add_argument("--print-checkpoint-structure"					, action="store_true"       )
   parser.add_argument(
       "path_to_checkpoint"					, type=__snake_case					, help="Path to the checkpoint file (.zip archive or direct .pt file)"					, )
   parser.add_argument(
       "--config_file"					, default=""					, type=__snake_case					, help="An optional config json file describing the pre-trained model."					, )
   __lowerCAmelCase								=					parser.parse_args()

   # Extract the basename.
   __lowerCAmelCase								=					os.path.dirname(args.path_to_checkpoint       )

   # Load the model.
   # the .zip is very optional, let's keep it for backward compatibility
   print(F"""Extracting PyTorch state dictionary from {args.path_to_checkpoint}"""       )
   if args.path_to_checkpoint.endswith(".zip"       ):
      with zipfile.ZipFile(args.path_to_checkpoint					, "r"       ) as checkpoint:
         with checkpoint.open("release/mp_rank_00/model_optim_rng.pt"       ) as pytorch_dict:
            __lowerCAmelCase								=					torch.load(__snake_case					, map_location="cpu"       )
   else:
      __lowerCAmelCase								=					torch.load(args.path_to_checkpoint					, map_location="cpu"       )

   __lowerCAmelCase								=					input_state_dict.get("args"					, __snake_case       )

   # Read the config, or default to the model released by NVIDIA.
   if args.config_file == "":
      if ds_args is not None:
         if ds_args.bias_gelu_fusion:
            __lowerCAmelCase								=					"gelu_fast"
         elif ds_args.openai_gelu:
            __lowerCAmelCase								=					"gelu_new"
         else:
            __lowerCAmelCase								=					"gelu"
      else:
         # in the very early days this used to be "gelu_new"
         __lowerCAmelCase								=					"gelu_new"

      # Spell out all parameters in case the defaults change.
      __lowerCAmelCase								=					GPTaConfig(
          vocab_size=5_0257					, n_positions=1024					, n_embd=1024					, n_layer=24					, n_head=16					, n_inner=4096					, activation_function=__snake_case					, resid_pdrop=0.1					, embd_pdrop=0.1					, attn_pdrop=0.1					, layer_norm_epsilon=1E-5					, initializer_range=0.02					, summary_type="cls_index"					, summary_use_proj=__snake_case					, summary_activation=__snake_case					, summary_proj_to_labels=__snake_case					, summary_first_dropout=0.1					, scale_attn_weights=__snake_case					, use_cache=__snake_case					, bos_token_id=5_0256					, eos_token_id=5_0256					, )
   else:
      __lowerCAmelCase								=					GPTaConfig.from_json_file(args.config_file       )

   __lowerCAmelCase								=					["GPT2LMHeadModel"]

   # Convert.
   print("Converting"       )
   __lowerCAmelCase								=					convert_megatron_checkpoint(__snake_case					, __snake_case					, __snake_case       )

   # Print the structure of converted state dict.
   if args.print_checkpoint_structure:
      recursive_print(__snake_case					, __snake_case       )

   # Add tokenizer class info to config
   # see https://github.com/huggingface/transformers/issues/13906)
   if ds_args is not None:
      __lowerCAmelCase								=					ds_args.tokenizer_type
      if tokenizer_type == "GPT2BPETokenizer":
         __lowerCAmelCase								=					"gpt2"
      elif tokenizer_type == "PretrainedFromHF":
         __lowerCAmelCase								=					ds_args.tokenizer_name_or_path
      else:
         raise ValueError(F"""Unrecognized tokenizer_type {tokenizer_type}"""       )
   else:
      __lowerCAmelCase								=					"gpt2"

   __lowerCAmelCase								=					AutoTokenizer.from_pretrained(__snake_case       )
   __lowerCAmelCase								=					type(__snake_case       ).__name__
   __lowerCAmelCase								=					tokenizer_class

   # Store the config to file.
   print("Saving config"       )
   config.save_pretrained(__snake_case       )

   # Save tokenizer based on args
   print(F"""Adding {tokenizer_class} tokenizer files"""       )
   tokenizer.save_pretrained(__snake_case       )

   # Store the state_dict to file.
   __lowerCAmelCase								=					os.path.join(__snake_case					, "pytorch_model.bin"       )
   print(F"""Saving checkpoint to \"{output_checkpoint_file}\""""       )
   torch.save(__snake_case					, __snake_case       )


####################################################################################################

if __name__ == "__main__":
 main()

####################################################################################################














from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_batched,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


lowerCamelCase			:					Optional[Any]         =	logging.get_logger(__name__)



class 							_UpperCamelCase			(a_			):
      snake_case_  =							["""pixel_values"""]


      def __init__(	self					,						__UpperCamelCase = True					,						__UpperCamelCase = None					,						__UpperCamelCase = PILImageResampling.BICUBIC					,						__UpperCamelCase = True					,						__UpperCamelCase = True					,						__UpperCamelCase = 1 / 2_5_5					,						__UpperCamelCase = None					,						__UpperCamelCase = True					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						**__UpperCamelCase					,						)->	None:
           super().__init__(**__UpperCamelCase	)
           __lowerCAmelCase								=					size if size is not None else {"height": 2_2_4, "width": 2_2_4}
           __lowerCAmelCase								=					get_size_dict(__UpperCamelCase	)
           __lowerCAmelCase								=					crop_size if crop_size is not None else {"height": 2_2_4, "width": 2_2_4}
           __lowerCAmelCase								=					get_size_dict(__UpperCamelCase					,						default_to_square=__UpperCamelCase					,						param_name="crop_size"	)

           __lowerCAmelCase								=					do_resize
           __lowerCAmelCase								=					do_rescale
           __lowerCAmelCase								=					do_normalize
           __lowerCAmelCase								=					do_center_crop
           __lowerCAmelCase								=					crop_size
           __lowerCAmelCase								=					size
           __lowerCAmelCase								=					resample
           __lowerCAmelCase								=					rescale_factor
           __lowerCAmelCase								=					image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
           __lowerCAmelCase								=					image_std if image_std is not None else IMAGENET_DEFAULT_STD


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase					,						__UpperCamelCase					,						__UpperCamelCase = PILImageResampling.BILINEAR					,						__UpperCamelCase = None					,						**__UpperCamelCase					,						)->	np.ndarray:
           __lowerCAmelCase								=					get_size_dict(__UpperCamelCase	)

           if "shortest_edge" in size:
                __lowerCAmelCase								=					get_resize_output_image_size(__UpperCamelCase					,						size=size["shortest_edge"]					,						default_to_square=__UpperCamelCase	)
                # size = get_resize_output_image_size(image, size["shortest_edge"], size["longest_edge"])
           elif "height" in size and "width" in size:
                __lowerCAmelCase								=					(size["height"], size["width"])
           else:
                raise ValueError(F"""Size must contain 'height' and 'width' keys or 'shortest_edge' key. Got {size.keys()}"""	)
           return resize(__UpperCamelCase					,						size=__UpperCamelCase					,						resample=__UpperCamelCase					,						data_format=__UpperCamelCase					,						**__UpperCamelCase	)


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase					,						__UpperCamelCase					,						__UpperCamelCase = None					,						**__UpperCamelCase					,						)->	np.ndarray:
           __lowerCAmelCase								=					get_size_dict(__UpperCamelCase	)
           if "height" not in size or "width" not in size:
                raise ValueError(F"""The `size` parameter must contain the keys (height, width). Got {size.keys()}"""	)
           return center_crop(__UpperCamelCase					,						size=(size["height"], size["width"])					,						data_format=__UpperCamelCase					,						**__UpperCamelCase	)


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase					,						__UpperCamelCase					,						__UpperCamelCase = None					,						**__UpperCamelCase	)->	np.ndarray:
           return rescale(__UpperCamelCase					,						scale=__UpperCamelCase					,						data_format=__UpperCamelCase					,						**__UpperCamelCase	)


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase					,						__UpperCamelCase					,						__UpperCamelCase					,						__UpperCamelCase = None					,						**__UpperCamelCase					,						)->	np.ndarray:
           return normalize(__UpperCamelCase					,						mean=__UpperCamelCase					,						std=__UpperCamelCase					,						data_format=__UpperCamelCase					,						**__UpperCamelCase	)


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = None					,						__UpperCamelCase = ChannelDimension.FIRST					,						**__UpperCamelCase					,						)->	BatchFeature:
           __lowerCAmelCase								=					do_resize if do_resize is not None else self.do_resize
           __lowerCAmelCase								=					do_rescale if do_rescale is not None else self.do_rescale
           __lowerCAmelCase								=					do_normalize if do_normalize is not None else self.do_normalize
           __lowerCAmelCase								=					do_center_crop if do_center_crop is not None else self.do_center_crop
           __lowerCAmelCase								=					crop_size if crop_size is not None else self.crop_size
           __lowerCAmelCase								=					get_size_dict(__UpperCamelCase					,						param_name="crop_size"					,						default_to_square=__UpperCamelCase	)
           __lowerCAmelCase								=					resample if resample is not None else self.resample
           __lowerCAmelCase								=					rescale_factor if rescale_factor is not None else self.rescale_factor
           __lowerCAmelCase								=					image_mean if image_mean is not None else self.image_mean
           __lowerCAmelCase								=					image_std if image_std is not None else self.image_std

           __lowerCAmelCase								=					size if size is not None else self.size
           __lowerCAmelCase								=					get_size_dict(__UpperCamelCase	)

           if not is_batched(__UpperCamelCase	):
                __lowerCAmelCase								=					[images]

           if not valid_images(__UpperCamelCase	):
                raise ValueError(
                    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
                    "torch.Tensor, tf.Tensor or jax.ndarray."	)

           if do_resize and size is None:
                raise ValueError("Size must be specified if do_resize is True."	)

           if do_center_crop and crop_size is None:
                raise ValueError("Crop size must be specified if do_center_crop is True."	)

           if do_rescale and rescale_factor is None:
                raise ValueError("Rescale factor must be specified if do_rescale is True."	)

           # All transformations expect numpy arrays.
           __lowerCAmelCase								=					[to_numpy_array(__UpperCamelCase	) for image in images]

           if do_resize:
                __lowerCAmelCase								=					[self.resize(image=__UpperCamelCase					,						size=__UpperCamelCase					,						resample=__UpperCamelCase	) for image in images]

           if do_center_crop:
                __lowerCAmelCase								=					[self.center_crop(image=__UpperCamelCase					,						size=__UpperCamelCase	) for image in images]

           if do_rescale:
                __lowerCAmelCase								=					[self.rescale(image=__UpperCamelCase					,						scale=__UpperCamelCase	) for image in images]

           if do_normalize:
                __lowerCAmelCase								=					[self.normalize(image=__UpperCamelCase					,						mean=__UpperCamelCase					,						std=__UpperCamelCase	) for image in images]

           __lowerCAmelCase								=					[to_channel_dimension_format(__UpperCamelCase					,						__UpperCamelCase	) for image in images]

           __lowerCAmelCase								=					{"pixel_values": images}
           return BatchFeature(data=__UpperCamelCase					,						tensor_type=__UpperCamelCase	)






