
# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig



def 							lowerCamelCase_	( __UpperCamelCase			,    __UpperCamelCase			,    __UpperCamelCase=0  ):


				if name is None:
								A_													=	None
				else:
								A_													=	'''.''' * max(0			,    spaces - 2  ) + '''# {:''' + str(50 - spaces  ) + '''s}'''
								A_													=	fmt.format(_lowercase  )

				# Print and recurse (if needed).
				if isinstance(_lowercase			,    _lowercase  ):
								if msg is not None:
												print(_lowercase  )
								for k in val.keys():
												recursive_print(_lowercase			,    val[k]			,    spaces + 2  )
				elif isinstance(_lowercase			,    torch.Tensor  ):
								print(_lowercase			,    ''':'''			,    val.size()  )
				else:
								print(_lowercase			,    ''':'''			,    _lowercase  )



def 							lowerCamelCase_	( __UpperCamelCase			,    __UpperCamelCase			,    __UpperCamelCase			,    __UpperCamelCase			,    __UpperCamelCase  ):
				A_													=	param.size()
				if checkpoint_version == 1.0:
								# version 1.0 stores [num_heads * hidden_size * num_splits, :]
								A_													=	(num_heads, hidden_size, num_splits) + input_shape[1:]
								A_													=	param.view(*_lowercase  )
								A_													=	param.transpose(0			,    2  )
								A_													=	param.transpose(1			,    2  ).contiguous()
				elif checkpoint_version >= 2.0:
								# other versions store [num_heads * num_splits * hidden_size, :]
								A_													=	(num_heads, num_splits, hidden_size) + input_shape[1:]
								A_													=	param.view(*_lowercase  )
								A_													=	param.transpose(0			,    1  ).contiguous()
				A_													=	param.view(*_lowercase  )
				return param



def 							lowerCamelCase_	( __UpperCamelCase			,    __UpperCamelCase			,    __UpperCamelCase  ):
				A_													=	{}

				# old versions did not store training args
				A_													=	input_state_dict.get('''args'''			,    _lowercase  )
				if ds_args is not None:
								# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
								# from pprint import pprint
								# pprint(vars(ds_args))

								A_													=	ds_args.padded_vocab_size
								A_													=	ds_args.max_position_embeddings
								A_													=	ds_args.hidden_size
								A_													=	ds_args.num_layers
								A_													=	ds_args.num_attention_heads
								A_													=	ds_args.ffn_hidden_size
								# pprint(config)

				# The number of heads.
				A_													=	config.n_head
				# The hidden_size per head.
				A_													=	config.n_embd // config.n_head
				# Megatron-LM checkpoint version
				if "checkpoint_version" in input_state_dict.keys():
								A_													=	input_state_dict['''checkpoint_version''']
				else:
								A_													=	0.0

				# The model.
				A_													=	input_state_dict['''model''']
				# The language model.
				A_													=	model['''language_model''']
				# The embeddings.
				A_													=	lm['''embedding''']

				# The word embeddings.
				A_													=	embeddings['''word_embeddings''']['''weight''']
				# Truncate the embedding table to vocab_size rows.
				A_													=	word_embeddings[: config.vocab_size, :]
				A_													=	word_embeddings

				# The position embeddings.
				A_													=	embeddings['''position_embeddings''']['''weight''']
				# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
				A_													=	pos_embeddings.size(0  )
				if n_positions != config.n_positions:
								raise ValueError(
								    F"pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match"  )
				# Store the position embeddings.
				A_													=	pos_embeddings

				# The transformer.
				A_													=	lm['''transformer'''] if '''transformer''' in lm.keys() else lm['''encoder''']

				# The regex to extract layer names.
				A_													=	re.compile(r'''layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)'''  )

				# The simple map of names for "automated" rules.
				A_													=	{
				    '''attention.dense''': '''.attn.c_proj.''',
				    '''self_attention.dense''': '''.attn.c_proj.''',
				    '''mlp.dense_h_to_4h''': '''.mlp.c_fc.''',
				    '''mlp.dense_4h_to_h''': '''.mlp.c_proj.''',
				}

				# Extract the layers.
				for key, val in transformer.items():
								# Match the name.
								A_													=	layer_re.match(_lowercase  )

								# Stop if that's not a layer
								if m is None:
												break

								# The index of the layer.
								A_													=	int(m.group(1  )  )
								# The name of the operation.
								A_													=	m.group(2  )
								# Is it a weight or a bias?
								A_													=	m.group(3  )

								# The name of the layer.
								A_													=	F"transformer.h.{layer_idx}"

								# For layernorm(s), simply store the layer norm.
								if op_name.endswith('''layernorm'''  ):
												A_													=	'''ln_1''' if op_name.startswith('''input'''  ) else '''ln_2'''
												A_													=	val

								# Transpose the QKV matrix.
								elif (
								    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
								) and weight_or_bias == "weight":
												# Insert a tensor of 1x1xDxD bias.
												A_													=	torch.tril(torch.ones((n_positions, n_positions)			,    dtype=torch.floataa  )  ).view(
												    1			,    1			,    _lowercase			,    _lowercase  )
												A_													=	causal_mask

												# Insert a "dummy" tensor for masked_bias.
												A_													=	torch.tensor(-1e4			,    dtype=torch.floataa  )
												A_													=	masked_bias

												A_													=	fix_query_key_value_ordering(_lowercase			,    _lowercase			,    3			,    _lowercase			,    _lowercase  )
												# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
												A_													=	out_val.transpose(0			,    1  ).contiguous()
												# Store.
												A_													=	out_val

								# Transpose the bias.
								elif (
								    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
								) and weight_or_bias == "bias":
												A_													=	fix_query_key_value_ordering(_lowercase			,    _lowercase			,    3			,    _lowercase			,    _lowercase  )
												# Store. No change of shape.
												A_													=	out_val

								# Transpose the weights.
								elif weight_or_bias == "weight":
												A_													=	megatron_to_transformers[op_name]
												A_													=	val.transpose(0			,    1  )

								# Copy the bias.
								elif weight_or_bias == "bias":
												A_													=	megatron_to_transformers[op_name]
												A_													=	val

    # DEBUG.
				assert config.n_layer == layer_idx + 1

				# The final layernorm.
				A_													=	transformer['''final_layernorm.weight''']
				A_													=	transformer['''final_layernorm.bias''']

				# For LM head, transformers' wants the matrix to weight embeddings.
				A_													=	word_embeddings

				# It should be done!
				return output_state_dict



def 							lowerCamelCase_	( ):
				A_													=	argparse.ArgumentParser()
				parser.add_argument('''--print-checkpoint-structure'''			,    action='''store_true'''  )
				parser.add_argument(
				    '''path_to_checkpoint'''			,    type=_lowercase			,    help='''Path to the checkpoint file (.zip archive or direct .pt file)'''			,    )
				parser.add_argument(
				    '''--config_file'''			,    default=''''''			,    type=_lowercase			,    help='''An optional config json file describing the pre-trained model.'''			,    )
				A_													=	parser.parse_args()

				# Extract the basename.
				A_													=	os.path.dirname(args.path_to_checkpoint  )

				# Load the model.
				# the .zip is very optional, let's keep it for backward compatibility
				print(F"Extracting PyTorch state dictionary from {args.path_to_checkpoint}"  )
				if args.path_to_checkpoint.endswith('''.zip'''  ):
								with zipfile.ZipFile(args.path_to_checkpoint			,    '''r'''  ) as checkpoint:
												with checkpoint.open('''release/mp_rank_00/model_optim_rng.pt'''  ) as pytorch_dict:
																A_													=	torch.load(_lowercase			,    map_location='''cpu'''  )
				else:
								A_													=	torch.load(args.path_to_checkpoint			,    map_location='''cpu'''  )

				A_													=	input_state_dict.get('''args'''			,    _lowercase  )

				# Read the config, or default to the model released by NVIDIA.
				if args.config_file == "":
								if ds_args is not None:
												if ds_args.bias_gelu_fusion:
																A_													=	'''gelu_fast'''
												elif ds_args.openai_gelu:
																A_													=	'''gelu_new'''
												else:
																A_													=	'''gelu'''
								else:
												# in the very early days this used to be "gelu_new"
												A_													=	'''gelu_new'''

								# Spell out all parameters in case the defaults change.
								A_													=	GPTaConfig(
								    vocab_size=5_02_57			,    n_positions=10_24			,    n_embd=10_24			,    n_layer=24			,    n_head=16			,    n_inner=40_96			,    activation_function=_lowercase			,    resid_pdrop=0.1			,    embd_pdrop=0.1			,    attn_pdrop=0.1			,    layer_norm_epsilon=1e-5			,    initializer_range=0.02			,    summary_type='''cls_index'''			,    summary_use_proj=_lowercase			,    summary_activation=_lowercase			,    summary_proj_to_labels=_lowercase			,    summary_first_dropout=0.1			,    scale_attn_weights=_lowercase			,    use_cache=_lowercase			,    bos_token_id=5_02_56			,    eos_token_id=5_02_56			,    )
				else:
								A_													=	GPTaConfig.from_json_file(args.config_file  )

				A_													=	['''GPT2LMHeadModel''']

				# Convert.
				print('''Converting'''  )
				A_													=	convert_megatron_checkpoint(_lowercase			,    _lowercase			,    _lowercase  )

				# Print the structure of converted state dict.
				if args.print_checkpoint_structure:
								recursive_print(_lowercase			,    _lowercase  )

				# Add tokenizer class info to config
				# see https://github.com/huggingface/transformers/issues/13906)
				if ds_args is not None:
								A_													=	ds_args.tokenizer_type
								if tokenizer_type == "GPT2BPETokenizer":
												A_													=	'''gpt2'''
								elif tokenizer_type == "PretrainedFromHF":
												A_													=	ds_args.tokenizer_name_or_path
								else:
												raise ValueError(F"Unrecognized tokenizer_type {tokenizer_type}"  )
				else:
								A_													=	'''gpt2'''

				A_													=	AutoTokenizer.from_pretrained(_lowercase  )
				A_													=	type(_lowercase  ).__name__
				A_													=	tokenizer_class

				# Store the config to file.
				print('''Saving config'''  )
				config.save_pretrained(_lowercase  )

				# Save tokenizer based on args
				print(F"Adding {tokenizer_class} tokenizer files"  )
				tokenizer.save_pretrained(_lowercase  )

				# Store the state_dict to file.
				A_													=	os.path.join(_lowercase			,    '''pytorch_model.bin'''  )
				print(F"Saving checkpoint to \"{output_checkpoint_file}\""  )
				torch.save(_lowercase			,    _lowercase  )


####################################################################################################

if __name__ == "__main__":
						main()

####################################################################################################



import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import XLMRobertaTokenizerFast

from diffusers import DDIMScheduler, KandinskyInpaintPipeline, KandinskyPriorPipeline, UNetaDConditionModel, VQModel
from diffusers.pipelines.kandinsky.text_encoder import MCLIPConfig, MultilingualCLIP
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference


enable_full_determinism()



class 	__lowerCamelCase			(_a				,      unittest.TestCase						):
       _lowercase											=     KandinskyInpaintPipeline
       _lowercase											=     ["""prompt""", """image_embeds""", """negative_image_embeds""", """image""", """mask_image"""]
       _lowercase											=     [
           """prompt""",
           """negative_prompt""",
           """image_embeds""",
           """negative_image_embeds""",
           """image""",
           """mask_image""",
       ]
       _lowercase											=     [
           """generator""",
           """height""",
           """width""",
           """latents""",
           """guidance_scale""",
           """negative_prompt""",
           """num_inference_steps""",
           """return_dict""",
           """guidance_scale""",
           """num_images_per_prompt""",
           """output_type""",
           """return_dict""",
       ]
       _lowercase											=     False







       @property
       def 		snake_case_	(			self: int					):






            '''simple docstring'''




            return 32







       @property
       def 		snake_case_	(			self: str					):






            '''simple docstring'''




            return 32







       @property
       def 		snake_case_	(			self: Tuple					):






            '''simple docstring'''




            return self.time_input_dim







       @property
       def 		snake_case_	(			self: Union[str, Any]					):






            '''simple docstring'''




            return self.time_input_dim * 4







       @property
       def 		snake_case_	(			self: Optional[int]					):






            '''simple docstring'''




            return 100







       @property
       def 		snake_case_	(			self: str					):






            '''simple docstring'''




            __UpperCamelCase =			XLMRobertaTokenizerFast.from_pretrained('YiYiXu/tiny-random-mclip-base'					)
            return tokenizer







       @property
       def 		snake_case_	(			self: Any					):






            '''simple docstring'''




            torch.manual_seed(0					)
            __UpperCamelCase =			MCLIPConfig(
                numDims=self.cross_attention_dim,transformerDimensions=self.text_embedder_hidden_size,hidden_size=self.text_embedder_hidden_size,intermediate_size=37,num_attention_heads=4,num_hidden_layers=5,vocab_size=1005,)

            __UpperCamelCase =			MultilingualCLIP(A_					)
            __UpperCamelCase =			text_encoder.eval()

            return text_encoder







       @property
       def 		snake_case_	(			self: Any					):






            '''simple docstring'''




            torch.manual_seed(0					)

            __UpperCamelCase =			{
                'in_channels': 9,
                # Out channels is double in channels because predicts mean and variance
                'out_channels': 8,
                'addition_embed_type': 'text_image',
                'down_block_types': ('ResnetDownsampleBlock2D', 'SimpleCrossAttnDownBlock2D'),
                'up_block_types': ('SimpleCrossAttnUpBlock2D', 'ResnetUpsampleBlock2D'),
                'mid_block_type': 'UNetMidBlock2DSimpleCrossAttn',
                'block_out_channels': (self.block_out_channels_a, self.block_out_channels_a * 2),
                'layers_per_block': 1,
                'encoder_hid_dim': self.text_embedder_hidden_size,
                'encoder_hid_dim_type': 'text_image_proj',
                'cross_attention_dim': self.cross_attention_dim,
                'attention_head_dim': 4,
                'resnet_time_scale_shift': 'scale_shift',
                'class_embed_type': None,
            }

            __UpperCamelCase =			UNetaDConditionModel(**A_					)
            return model







       @property
       def 		snake_case_	(			self: str					):






            '''simple docstring'''




            return {
                "block_out_channels": [32, 64],
                "down_block_types": ["DownEncoderBlock2D", "AttnDownEncoderBlock2D"],
                "in_channels": 3,
                "latent_channels": 4,
                "layers_per_block": 1,
                "norm_num_groups": 8,
                "norm_type": "spatial",
                "num_vq_embeddings": 12,
                "out_channels": 3,
                "up_block_types": [
                    "AttnUpDecoderBlock2D",
                    "UpDecoderBlock2D",
                ],
                "vq_embed_dim": 4,
            }







       @property
       def 		snake_case_	(			self: str					):






            '''simple docstring'''




            torch.manual_seed(0					)
            __UpperCamelCase =			VQModel(**self.dummy_movq_kwargs					)
            return model







       def 		snake_case_	(			self: Dict					):






            '''simple docstring'''




            __UpperCamelCase =			self.dummy_text_encoder
            __UpperCamelCase =			self.dummy_tokenizer
            __UpperCamelCase =			self.dummy_unet
            __UpperCamelCase =			self.dummy_movq

            __UpperCamelCase =			DDIMScheduler(
                num_train_timesteps=1000,beta_schedule='linear',beta_start=0.0_0_0_8_5,beta_end=0.0_1_2,clip_sample=A_,set_alpha_to_one=A_,steps_offset=1,prediction_type='epsilon',thresholding=A_,)

            __UpperCamelCase =			{
                'text_encoder': text_encoder,
                'tokenizer': tokenizer,
                'unet': unet,
                'scheduler': scheduler,
                'movq': movq,
            }

            return components







       def 		snake_case_	(			self: Tuple,A_: Optional[int],A_: Dict=0					):






            '''simple docstring'''




            __UpperCamelCase =			floats_tensor((1, self.cross_attention_dim),rng=random.Random(A_					)					).to(A_					)
            __UpperCamelCase =			floats_tensor((1, self.cross_attention_dim),rng=random.Random(seed + 1					)					).to(A_					)
            # create init_image
            __UpperCamelCase =			floats_tensor((1, 3, 64, 64),rng=random.Random(A_					)					).to(A_					)
            __UpperCamelCase =			image.cpu().permute(0,2,3,1					)[0]
            __UpperCamelCase =			Image.fromarray(np.uinta(A_					)					).convert('RGB'					).resize((256, 256)					)
            # create mask
            __UpperCamelCase =			np.ones((64, 64),dtype=np.floataa					)
            __UpperCamelCase =			0

            if str(A_					).startswith('mps'					):
                 __UpperCamelCase =			torch.manual_seed(A_					)
            else:
                 __UpperCamelCase =			torch.Generator(device=A_					).manual_seed(A_					)
            __UpperCamelCase =			{
                'prompt': 'horse',
                'image': init_image,
                'mask_image': mask,
                'image_embeds': image_embeds,
                'negative_image_embeds': negative_image_embeds,
                'generator': generator,
                'height': 64,
                'width': 64,
                'num_inference_steps': 2,
                'guidance_scale': 4.0,
                'output_type': 'np',
            }
            return inputs







       def 		snake_case_	(			self: Any					):






            '''simple docstring'''




            __UpperCamelCase =			'cpu'

            __UpperCamelCase =			self.get_dummy_components()

            __UpperCamelCase =			self.pipeline_class(**A_					)
            __UpperCamelCase =			pipe.to(A_					)

            pipe.set_progress_bar_config(disable=A_					)

            __UpperCamelCase =			pipe(**self.get_dummy_inputs(A_					)					)
            __UpperCamelCase =			output.images

            __UpperCamelCase =			pipe(
                **self.get_dummy_inputs(A_					),return_dict=A_,)[0]

            __UpperCamelCase =			image[0, -3:, -3:, -1]
            __UpperCamelCase =			image_from_tuple[0, -3:, -3:, -1]

            print(F'''image.shape {image.shape}'''					)

            assert image.shape == (1, 64, 64, 3)

            __UpperCamelCase =			np.array(
                [0.8_3_2_6_9_1_9, 0.7_3_7_9_0_4_6_7, 0.2_0_9_1_8_5_8_1, 0.9_3_0_9_6_1_2, 0.5_5_1_1_7_9_1, 0.4_3_7_1_3_3_2_8, 0.5_5_1_3_3_2_1, 0.4_9_9_2_2_9_3_4, 0.5_9_4_9_7_7_8_6]					)

            assert (
                np.abs(image_slice.flatten() - expected_slice					).max() < 1E-2
            ), F''' expected_slice {expected_slice}, but got {image_slice.flatten()}'''
            assert (
                np.abs(image_from_tuple_slice.flatten() - expected_slice					).max() < 1E-2
            ), F''' expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}'''







       def 		snake_case_	(			self: Optional[Any]					):






            '''simple docstring'''




            super().test_inference_batch_single_identical(expected_max_diff=3E-3					)



@slow
@require_torch_gpu
class 	__lowerCamelCase			(unittest.TestCase						):







       def 		snake_case_	(			self: Tuple					):






            '''simple docstring'''




            super().tearDown()
            gc.collect()
            torch.cuda.empty_cache()







       def 		snake_case_	(			self: Any					):






            '''simple docstring'''




            __UpperCamelCase =			load_numpy(
                'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
                '/kandinsky/kandinsky_inpaint_cat_with_hat_fp16.npy'					)

            __UpperCamelCase =			load_image(
                'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main' '/kandinsky/cat.png'					)
            __UpperCamelCase =			np.ones((768, 768),dtype=np.floataa					)
            __UpperCamelCase =			0

            __UpperCamelCase =			'a hat'

            __UpperCamelCase =			KandinskyPriorPipeline.from_pretrained(
                'kandinsky-community/kandinsky-2-1-prior',torch_dtype=torch.floataa					)
            pipe_prior.to(A_					)

            __UpperCamelCase =			KandinskyInpaintPipeline.from_pretrained(
                'kandinsky-community/kandinsky-2-1-inpaint',torch_dtype=torch.floataa					)
            __UpperCamelCase =			pipeline.to(A_					)
            pipeline.set_progress_bar_config(disable=A_					)

            __UpperCamelCase =			torch.Generator(device='cpu'					).manual_seed(0					)
            __UpperCamelCase,    __UpperCamelCase =			pipe_prior(
                A_,generator=A_,num_inference_steps=5,negative_prompt='',).to_tuple()

            __UpperCamelCase =			pipeline(
                A_,image=A_,mask_image=A_,image_embeds=A_,negative_image_embeds=A_,generator=A_,num_inference_steps=100,height=768,width=768,output_type='np',)

            __UpperCamelCase =			output.images[0]

            assert image.shape == (768, 768, 3)

            assert_mean_pixel_difference(A_,A_					)


