



'''simple docstring'''

import unittest

import numpy as np
import torch
from torch import nn
from transformers import (
    CLIPImageProcessor,
    CLIPTextConfig,
    CLIPTextModelWithProjection,
    CLIPTokenizer,
    CLIPVisionConfig,
    CLIPVisionModelWithProjection,
)

from diffusers import KandinskyPriorPipeline, PriorTransformer, UnCLIPScheduler
from diffusers.utils import torch_device
from diffusers.utils.testing_utils import enable_full_determinism, skip_mps

from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()

class __snake_case(      _lowerCAmelCase		,							unittest.TestCase					):


						'''simple docstring'''



						UpperCAmelCase							:    List[Any]           =		KandinskyPriorPipeline
						UpperCAmelCase							:    Union[str, Any]           =		["prompt"]
						UpperCAmelCase							:    List[Any]           =		["prompt", "negative_prompt"]
						UpperCAmelCase							:    str           =		[
						    "num_images_per_prompt",
						    "generator",
						    "num_inference_steps",
						    "latents",
						    "negative_prompt",
						    "guidance_scale",
						    "output_type",
						    "return_dict",
						]
						UpperCAmelCase							:    Any           =		False


						@property
						def 				__snake_case    (					self      )	->	int:
							return 32


						@property
						def 				__snake_case    (					self      )	->	List[Any]:
							return 32


						@property
						def 				__snake_case    (					self      )	->	int:
							return self.time_input_dim


						@property
						def 				__snake_case    (					self      )	->	List[str]:
							return self.time_input_dim * 4


						@property
						def 				__snake_case    (					self      )	->	List[Any]:
							return 100


						@property
						def 				__snake_case    (					self      )	->	int:
							lowerCAmelCase    =	CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""      )
							return tokenizer


						@property
						def 				__snake_case    (					self      )	->	Any:
							torch.manual_seed(0      )
							lowerCAmelCase    =	CLIPTextConfig(
							    bos_token_id=0	,       eos_token_id=2	,       hidden_size=self.text_embedder_hidden_size	,       projection_dim=self.text_embedder_hidden_size	,       intermediate_size=37	,       layer_norm_eps=1e-05	,       num_attention_heads=4	,       num_hidden_layers=5	,       pad_token_id=1	,       vocab_size=1000	,       )
							return CLIPTextModelWithProjection(A_      )


						@property
						def 				__snake_case    (					self      )	->	Tuple:
							torch.manual_seed(0      )

							lowerCAmelCase    =	{
							    """num_attention_heads""": 2,
							    """attention_head_dim""": 12,
							    """embedding_dim""": self.text_embedder_hidden_size,
							    """num_layers""": 1,
							}

							lowerCAmelCase    =	PriorTransformer(**A_      )
							# clip_std and clip_mean is initialized to be 0 so PriorTransformer.post_process_latents will always return 0 - set clip_std to be 1 so it won't return 0
							lowerCAmelCase    =	nn.Parameter(torch.ones(model.clip_std.shape      )      )
							return model


						@property
						def 				__snake_case    (					self      )	->	List[str]:
							torch.manual_seed(0      )
							lowerCAmelCase    =	CLIPVisionConfig(
							    hidden_size=self.text_embedder_hidden_size	,       image_size=224	,       projection_dim=self.text_embedder_hidden_size	,       intermediate_size=37	,       num_attention_heads=4	,       num_channels=3	,       num_hidden_layers=5	,       patch_size=14	,       )

							lowerCAmelCase    =	CLIPVisionModelWithProjection(A_      )
							return model


						@property
						def 				__snake_case    (					self      )	->	Optional[int]:
							lowerCAmelCase    =	CLIPImageProcessor(
							    crop_size=224	,       do_center_crop=A_	,       do_normalize=A_	,       do_resize=A_	,       image_mean=[0.4_8_1_4_5_4_6_6, 0.4_5_7_8_2_7_5, 0.4_0_8_2_1_0_7_3]	,       image_std=[0.2_6_8_6_2_9_5_4, 0.2_6_1_3_0_2_5_8, 0.2_7_5_7_7_7_1_1]	,       resample=3	,       size=224	,       )

							return image_processor


						def 				__snake_case    (					self      )	->	Tuple:
							lowerCAmelCase    =	self.dummy_prior
							lowerCAmelCase    =	self.dummy_image_encoder
							lowerCAmelCase    =	self.dummy_text_encoder
							lowerCAmelCase    =	self.dummy_tokenizer
							lowerCAmelCase    =	self.dummy_image_processor

							lowerCAmelCase    =	UnCLIPScheduler(
							    variance_type="""fixed_small_log"""	,       prediction_type="""sample"""	,       num_train_timesteps=1000	,       clip_sample=A_	,       clip_sample_range=1_0.0	,       )

							lowerCAmelCase    =	{
							    """prior""": prior,
							    """image_encoder""": image_encoder,
							    """text_encoder""": text_encoder,
							    """tokenizer""": tokenizer,
							    """scheduler""": scheduler,
							    """image_processor""": image_processor,
							}

							return components


						def 				__snake_case    (					self	,       A_	,       A_=0      )	->	Optional[Any]:
							if str(A_      ).startswith("""mps"""      ):
								lowerCAmelCase    =	torch.manual_seed(A_      )
							else:
								lowerCAmelCase    =	torch.Generator(device=A_      ).manual_seed(A_      )
							lowerCAmelCase    =	{
							    """prompt""": """horse""",
							    """generator""": generator,
							    """guidance_scale""": 4.0,
							    """num_inference_steps""": 2,
							    """output_type""": """np""",
							}
							return inputs


						def 				__snake_case    (					self      )	->	Dict:
							lowerCAmelCase    =	"""cpu"""

							lowerCAmelCase    =	self.get_dummy_components()

							lowerCAmelCase    =	self.pipeline_class(**A_      )
							lowerCAmelCase    =	pipe.to(A_      )

							pipe.set_progress_bar_config(disable=A_      )

							lowerCAmelCase    =	pipe(**self.get_dummy_inputs(A_      )      )
							lowerCAmelCase    =	output.image_embeds

							lowerCAmelCase    =	pipe(
							    **self.get_dummy_inputs(A_      )	,       return_dict=A_	,       )[0]

							lowerCAmelCase    =	image[0, -10:]
							lowerCAmelCase    =	image_from_tuple[0, -10:]

							assert image.shape == (1, 32)

							lowerCAmelCase    =	np.array(
							    [-0.0_5_3_2, 1.7_1_2_0, 0.3_6_5_6, -1.0_8_5_2, -0.8_9_4_6, -1.1_7_5_6, 0.4_3_4_8, 0.2_4_8_2, 0.5_1_4_6, -0.1_1_5_6]      )

							assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1e-2
							assert np.abs(image_from_tuple_slice.flatten() - expected_slice      ).max() < 1e-2


						@skip_mps
						def 				__snake_case    (					self      )	->	Optional[int]:
							lowerCAmelCase    =	torch_device == """cpu"""
							lowerCAmelCase    =	True
							lowerCAmelCase    =	False

							self._test_inference_batch_single_identical(
							    test_max_difference=A_	,       relax_max_difference=A_	,       test_mean_pixel_difference=A_	,       )




						@skip_mps
						def 				__snake_case    (					self      )	->	str:
							lowerCAmelCase    =	torch_device == """cpu"""
							lowerCAmelCase    =	False

							self._test_attention_slicing_forward_pass(
							    test_max_difference=A_	,       test_mean_pixel_difference=A_	,       )




'''simple docstring'''

import argparse

# utilities to import the model weights and config file
import os
import pickle as pkl

# PyTorch + new model classes
import torch
from torch import nn

from transformers import AutoTokenizer, MegaConfig, MegaForMaskedLM


# import the EncoderLayer class used to pretrain
# !! NOTE !! this requires the version of fairseq that is built when you install the Mega source
try:
				from fairseq.modules.mega_layer import MegaEncoderLayer
except ImportError:
				raise ImportError('You need to install the version of fairseq from the Mega repo!')

class __snake_case(      nn.Module					):


						'''simple docstring'''



						def __init__(					self	,       A_	,       A_	,       A_      )	->	Dict:
							super().__init__()
							lowerCAmelCase    =	mega_args
							lowerCAmelCase    =	nn.Embedding(A_	,       self.mega_args.encoder_embed_dim      )
							lowerCAmelCase    =	nn.ModuleList([MegaEncoderLayer(self.mega_args      ) for _ in range(A_      )]      )
							lowerCAmelCase    =	depth




						def 				__snake_case    (					self	,       A_	,       A_	,       A_=True	,       A_=0      )	->	List[Any]:

							# Mega expects embeddings to be (time, batch, embedding size), but
							# Hugging Face returns tokens as (batch, time)
							if batch_first:
								lowerCAmelCase    =	input_ids.T

							# to make things more confusing, Mega expects the attention mask to
							# be (batch, time), but with values of 0 (normal token) and 1 (ignore token)
							# which is the opposite of what HF returns
							if ignore_mask_value == 0:
								lowerCAmelCase    =	1 - attention_mask

							# get token embeddings from IDs
							lowerCAmelCase    =	self.embedding_layer(A_      )

							# pass through the Mega layers
							# input is (time, batch, encoder dim) and output is the same
							for encoder in self.encoders:
								lowerCAmelCase    =	encoder(A_	,       A_      )

							# return according to the shape specified
							if batch_first:
								# (T, B, H) --> (B, T, H)
								return torch.transpose(A_	,       0	,       1      )
							else:
								return embeds







class __snake_case(      nn.Module					):


						'''simple docstring'''



						def __init__(					self	,       A_	,       A_	,       A_      )	->	Optional[Any]:
							super().__init__()
							lowerCAmelCase    =	MegaLM(A_	,       A_	,       A_      )
							lowerCAmelCase    =	nn.Linear(mega_args.encoder_embed_dim	,       A_      )
							lowerCAmelCase    =	nn.Dropout(p=0.1      )




						def 				__snake_case    (					self	,       A_	,       A_	,       A_=True	,       A_=0      )	->	List[Any]:
							lowerCAmelCase    =	self.mega(A_	,       A_	,       A_	,       A_      )
							return self.mlm_head(self.dropout(A_      )      )







def 		_snake_case		(					_SCREAMING_SNAKE_CASE     :						Optional[int]							,		_SCREAMING_SNAKE_CASE     :						List[str]							,		_SCREAMING_SNAKE_CASE     :						Union[str, Any]							)			->						List[Any]:







	"""simple docstring"""
	with open(os.path.join(_SCREAMING_SNAKE_CASE							,		"""model_args.pkl"""							)							,		"""rb"""							) as f:
		lowerCAmelCase    =	pkl.load(_SCREAMING_SNAKE_CASE							)

	# load the original encoder
	lowerCAmelCase    =	OriginalMegaForMaskedLM(**_SCREAMING_SNAKE_CASE							).eval()

	# load its weights
	print(
	    """Original Mega encoder:"""							,		original_mlm.mega.load_state_dict(
	        torch.load(os.path.join(_SCREAMING_SNAKE_CASE							,		"""encoder_weights.pt"""							)							,		map_location="""cpu"""							)							)							,		)
	print(
	    """Original Mega MLM layer:"""							,		original_mlm.mlm_head.load_state_dict(
	        torch.load(os.path.join(_SCREAMING_SNAKE_CASE							,		"""mlm_head_weights.pt"""							)							,		map_location="""cpu"""							)							)							,		)

	# create a new config from the old one
	lowerCAmelCase    =	MegaConfig(
	    num_hidden_layers=mega_original_args["""depth"""]							,		vocab_size=mega_original_args["""vocab_size"""]							,		hidden_size=mega_original_args["""mega_args"""].encoder_embed_dim							,		shared_representation_size=mega_original_args["""mega_args"""].encoder_z_dim							,		intermediate_size=mega_original_args["""mega_args"""].encoder_hidden_dim							,		ema_projection_size=mega_original_args["""mega_args"""].encoder_n_dim							,		dropout_prob=mega_original_args["""mega_args"""].dropout							,		attention_probs_dropout_prob=mega_original_args["""mega_args"""].attention_dropout							,		hidden_dropout_prob=mega_original_args["""mega_args"""].hidden_dropout							,		activation=mega_original_args["""mega_args"""].activation_fn							,		attention_activation=mega_original_args["""mega_args"""].attention_activation_fn							,		bidirectional=mega_original_args["""mega_args"""].bidirectional							,		use_chunking=mega_original_args["""mega_args"""].encoder_chunk_size > 0							,		chunk_size=mega_original_args["""mega_args"""].encoder_chunk_size							,		truncation=mega_original_args["""mega_args"""].truncation_length							,		normalization_type=mega_original_args["""mega_args"""].normalization_type							,		normalize_before_mega=_SCREAMING_SNAKE_CASE							,		norm_affine=_SCREAMING_SNAKE_CASE							,		use_feature_dropout=mega_original_args["""mega_args"""].feature_dropout							,		relative_positional_bias=mega_original_args["""mega_args"""].rel_pos_bias							,		max_positions=mega_original_args["""mega_args"""].max_source_positions							,		nffn_hidden_size=mega_original_args["""mega_args"""].encoder_ffn_embed_dim							,		normalize_before_ffn=mega_original_args["""mega_args"""].normalize_before							,		nffn_activation_dropout_prob=0.0							,		add_token_type_embeddings=_SCREAMING_SNAKE_CASE							,		add_lm_hidden_dense_layer=_SCREAMING_SNAKE_CASE							,		)

	lowerCAmelCase    =	MegaForMaskedLM(_SCREAMING_SNAKE_CASE							).eval()

	# the originl checkpoint just uses nn.Embedding for the word embeddings
	# we use a wrapper module for embeddings to add support for positional embeddings
	lowerCAmelCase    =	original_mlm.mega.embedding_layer.weight

	# modify the state dictionary of the original checkpoint to account for naming issues in the Hugging Face
	# ecosystem -- any names containing "beta" or "gamma" aren't safe to use and are renamed upon _load_pretrained,
	# also renaming previously confusing parameter names
	lowerCAmelCase    =	original_mlm.mega.encoders.state_dict()
	lowerCAmelCase    =	{}
	for module_name in original_state_dict.keys():
		lowerCAmelCase    =	None
		# have to handle gamma, beta, and alpha differently due to their use
		# in multiple modules within the original repository;
		# beta is used in EMA, MovingAverageGatedAttention, and RotaryRelativePositionalBias, and must be renamed due to flax/tf weights
		# the EMA sublayer was renamed from "move" to "ema_gate" for readability, so that is also done here
		if "beta" in module_name:
			# EMA sub-layers were always called "move" in the original repo
			if "move.beta" in module_name:
				lowerCAmelCase    =	module_name.replace("""move.beta"""							,		"""ema_gate.ema_expansion_matrix"""							)
			elif "mega_layer.beta" in module_name:
				lowerCAmelCase    =	module_name.replace("""beta"""							,		"""qk_bias"""							)
			else:
				lowerCAmelCase    =	module_name.replace("""beta"""							,		"""b_param"""							)
        # beta is used in EMA and MovingAverageGatedAttention, and must be renamed due to flax/tf weights
		elif "gamma" in module_name:
			if "move.gamma" in module_name:
				lowerCAmelCase    =	module_name.replace("""move.gamma"""							,		"""ema_gate.kernel_projection_matrix"""							)
			elif "mega_layer.gamma" in module_name:
				lowerCAmelCase    =	module_name.replace("""gamma"""							,		"""qk_weight"""							)
			else:
				lowerCAmelCase    =	module_name.replace("""gamma"""							,		"""g_param"""							)
        # alpha is used in EMA and positional bias; renaming to improve readability
		elif "move.alpha" in module_name:
			lowerCAmelCase    =	module_name.replace("""move.alpha"""							,		"""ema_gate.decay_factor"""							)
		# delta is only used in EMA; renaming to improve readability
		elif "move.delta" in module_name:
			lowerCAmelCase    =	module_name.replace("""move.delta"""							,		"""ema_gate.damping_factor"""							)
		# omega is only used in EMA; renaming to improve readability
		elif "omega" in module_name:
			lowerCAmelCase    =	module_name.replace("""move.omega"""							,		"""ema_gate.residual_weight"""							)

		if new_module_name:
			lowerCAmelCase    =	new_module_name

	if len(_SCREAMING_SNAKE_CASE							) != 0:
		print(f'Renaming these keys: {updated_keys.keys()}'							)
	else:
		print("""No need to rename state dict entries"""							)
	for old, new in updated_keys.items():
		lowerCAmelCase    =	original_state_dict.pop(_SCREAMING_SNAKE_CASE							)

	# now attempt to load the state dictionary with updated names
	# note that we now call it `mega.layers` instead of `mega.encoders` due to hugging face style
	print("""HF Mega encoder:"""							,		hf_mlm.mega.layers.load_state_dict(_SCREAMING_SNAKE_CASE							)							)

	# load the MLM head weights directly
	print(
	    """HF Mega MLM layer:"""							,		hf_mlm.mlm_head.load_state_dict(
	        torch.load(os.path.join(_SCREAMING_SNAKE_CASE							,		"""mlm_head_weights.pt"""							)							,		map_location="""cpu"""							)							)							,		)

	# test on a randomly generated input sequence
	lowerCAmelCase    =	torch.randint(0							,		hf_config.vocab_size							,		size=(4, 256)							)
	lowerCAmelCase    =	torch.ones_like(_SCREAMING_SNAKE_CASE							)
	# mask a few tokens to make sure masking is applied appropriately :)
	lowerCAmelCase    =	0

	# run forward passes
	lowerCAmelCase    =	original_mlm(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							,		batch_first=_SCREAMING_SNAKE_CASE							,		ignore_mask_value=0							)
	lowerCAmelCase    =	hf_mlm(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							)[0]

	# print shapes and diff
	print(f'original output {original_output.shape}'							)
	print(f'hf output {hf_output.shape}'							)
	print(f'max diff: {(original_output - hf_output).max()}'							)  # 0.0
	lowerCAmelCase    =	torch.allclose(_SCREAMING_SNAKE_CASE							,		_SCREAMING_SNAKE_CASE							,		atol=1E-3							)

	if success:
		print("""Yay!"""							)
		hf_mlm.save_pretrained(_SCREAMING_SNAKE_CASE							)
	else:
		raise RuntimeError(f'Something\'s broken :(\nOriginal:\n{original_output}\n\nHF\n{hf_output}\n{hf_mlm}'							)

	if includes_tokenizer:
		print("""Transferring tokenizer"""							)
		lowerCAmelCase    =	AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE							)
		tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE							)


if __name__ == "__main__":
				UpperCAmelCase			=    argparse.ArgumentParser()

				parser.add_argument(
				    '--pretrained_checkpoint_path',
				    default=None,
				    type=str,
				    required=True,
				    help='Point to the directory containing your model weights using the official Mega repo',
				)

				parser.add_argument(
				    '--output_path', default=None, type=str, required=True, help='Location to save the Hugging Face version'
				)

				parser.add_argument(
				    '--includes_tokenizer',
				    action='store_true',
				    help='Use this flag if there is a Hugging Face tokenizer in the original checkpoint repo',
				)

				UpperCAmelCase			=    parser.parse_args()

				convert_checkpoint_to_huggingface(args.pretrained_checkpoint_path, args.output_path, args.includes_tokenizer)