


'''simple docstring'''






import gc
import random
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DiffusionPipeline,
    EulerDiscreteScheduler,
    StableDiffusionXLImgaImgPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import floats_tensor, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import (
    IMAGE_TO_IMAGE_IMAGE_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_PARAMS,
)
from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()
class 						__UpperCamelCase	(						lowercase__						,  lowercase__						,  unittest.TestCase							):
							lowercase			:		Any   		=  StableDiffusionXLImgaImgPipeline
							lowercase			:		str   		=  TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {'height', 'width'}
							lowercase			:		Tuple   		=  PipelineTesterMixin.required_optional_params - {'latents'}
							lowercase			:		List[str]   		=  TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS
							lowercase			:		Dict   		=  IMAGE_TO_IMAGE_IMAGE_PARAMS
							lowercase			:		Optional[Any]   		=  IMAGE_TO_IMAGE_IMAGE_PARAMS





							def 		a__   (  self		:Tuple			):
									torch.manual_seed(0			)
									snake_case_		:							Tuple			       =						UNetaDConditionModel(
									    block_out_channels=(3_2, 6_4)  ,layers_per_block=2  ,sample_size=3_2  ,in_channels=4  ,out_channels=4  ,down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")  ,up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")  ,attention_head_dim=(2, 4)  ,use_linear_projection=_UpperCamelCase  ,addition_embed_type="""text_time"""  ,addition_time_embed_dim=8  ,transformer_layers_per_block=(1, 2)  ,projection_class_embeddings_input_dim=8_0  ,cross_attention_dim=6_4  ,)
									snake_case_		:							Union[str, Any]			       =						EulerDiscreteScheduler(
									    beta_start=0.0_00_85  ,beta_end=0.0_12  ,steps_offset=1  ,beta_schedule="""scaled_linear"""  ,timestep_spacing="""leading"""  ,)
									torch.manual_seed(0			)
									snake_case_		:							List[Any]			       =						AutoencoderKL(
									    block_out_channels=[3_2, 6_4]  ,in_channels=3  ,out_channels=3  ,down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]  ,up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]  ,latent_channels=4  ,sample_size=1_2_8  ,)
									torch.manual_seed(0			)
									snake_case_		:							Any			       =						CLIPTextConfig(
									    bos_token_id=0  ,eos_token_id=2  ,hidden_size=3_2  ,intermediate_size=3_7  ,layer_norm_eps=1E-0_5  ,num_attention_heads=4  ,num_hidden_layers=5  ,pad_token_id=1  ,vocab_size=1_0_0_0  ,hidden_act="""gelu"""  ,projection_dim=3_2  ,)
									snake_case_		:							Union[str, Any]			       =						CLIPTextModel(_UpperCamelCase			)
									snake_case_		:							List[str]			       =						CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""  ,local_files_only=_UpperCamelCase			)

									snake_case_		:							int			       =						CLIPTextModelWithProjection(_UpperCamelCase			)
									snake_case_		:							Optional[int]			       =						CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""  ,local_files_only=_UpperCamelCase			)

									snake_case_		:							Dict			       =						{
									    """unet""": unet,
									    """scheduler""": scheduler,
									    """vae""": vae,
									    """text_encoder""": text_encoder,
									    """tokenizer""": tokenizer,
									    """text_encoder_2""": text_encoder_a,
									    """tokenizer_2""": tokenizer_a,
									    # "safety_checker": None,
									    # "feature_extractor": None,
									}
									return components





							def 		a__   (  self		:Any  ,_UpperCamelCase		:int  ,_UpperCamelCase		:Optional[int]=0			):
									snake_case_		:							Tuple			       =						floats_tensor((1, 3, 3_2, 3_2)  ,rng=random.Random(_UpperCamelCase			)			).to(_UpperCamelCase			)
									snake_case_		:							str			       =						image / 2 + 0.5
									if str(_UpperCamelCase			).startswith("""mps"""			):
											snake_case_		:							Optional[Any]			       =						torch.manual_seed(_UpperCamelCase			)
									else:
											snake_case_		:							str			       =						torch.Generator(device=_UpperCamelCase			).manual_seed(_UpperCamelCase			)
									snake_case_		:							List[Any]			       =						{
									    """prompt""": """A painting of a squirrel eating a burger""",
									    """image""": image,
									    """generator""": generator,
									    """num_inference_steps""": 2,
									    """guidance_scale""": 5.0,
									    """output_type""": """numpy""",
									    """strength""": 0.75,
									}
									return inputs





							def 		a__   (  self		:Tuple			):
									snake_case_		:							Optional[Any]			       =						"""cpu"""  # ensure determinism for the device-dependent torch.Generator
									snake_case_		:							List[Any]			       =						self.get_dummy_components()
									snake_case_		:							Optional[Any]			       =						StableDiffusionXLImgaImgPipeline(**_UpperCamelCase			)
									snake_case_		:							Union[str, Any]			       =						sd_pipe.to(_UpperCamelCase			)
									sd_pipe.set_progress_bar_config(disable=_UpperCamelCase			)

									snake_case_		:							str			       =						self.get_dummy_inputs(_UpperCamelCase			)
									snake_case_		:							Optional[Any]			       =						sd_pipe(**_UpperCamelCase			).images
									snake_case_		:							int			       =						image[0, -3:, -3:, -1]

									assert image.shape == (1, 3_2, 3_2, 3)

									snake_case_		:							List[str]			       =						np.array([0.46_56, 0.48_40, 0.44_39, 0.66_98, 0.55_74, 0.45_24, 0.57_99, 0.59_43, 0.51_65]			)

									assert np.abs(image_slice.flatten() - expected_slice			).max() < 1E-2





							def 		a__   (  self		:Optional[int]			):
									super().test_attention_slicing_forward_pass(expected_max_diff=3E-3			)





							def 		a__   (  self		:Optional[Any]			):
									super().test_inference_batch_single_identical(expected_max_diff=3E-3			)





							def 		a__   (  self		:Any			):
									pass





							def 		a__   (  self		:List[str]			):
									snake_case_		:							Optional[Any]			       =						self.get_dummy_components()
									snake_case_		:							Optional[Any]			       =						StableDiffusionXLImgaImgPipeline(**_UpperCamelCase			)
									snake_case_		:							Union[str, Any]			       =						sd_pipe.to(_UpperCamelCase			)
									snake_case_		:							str			       =						sd_pipe.to(_UpperCamelCase			)
									sd_pipe.set_progress_bar_config(disable=_UpperCamelCase			)

									# forward without prompt embeds
									snake_case_		:							int			       =						self.get_dummy_inputs(_UpperCamelCase			)
									snake_case_		:							Dict			       =						3 * ["""this is a negative prompt"""]
									snake_case_		:							Any			       =						negative_prompt
									snake_case_		:							Dict			       =						3 * [inputs["""prompt"""]]

									snake_case_		:							Any			       =						sd_pipe(**_UpperCamelCase			)
									snake_case_		:							Any			       =						output.images[0, -3:, -3:, -1]

									# forward with prompt embeds
									snake_case_		:							Dict			       =						self.get_dummy_inputs(_UpperCamelCase			)
									snake_case_		:							Dict			       =						3 * ["""this is a negative prompt"""]
									snake_case_		:							int			       =						3 * [inputs.pop("""prompt"""			)]

									(
									    (
									    snake_case_
									)       ,      (
									    snake_case_
									)       ,      (
									    snake_case_
									)       ,      (
									    snake_case_
									)       ,      
									)		:							Union[str, Any]			       =						sd_pipe.encode_prompt(_UpperCamelCase  ,negative_prompt=_UpperCamelCase			)

									snake_case_		:							Optional[int]			       =						sd_pipe(
									    **_UpperCamelCase  ,prompt_embeds=_UpperCamelCase  ,negative_prompt_embeds=_UpperCamelCase  ,pooled_prompt_embeds=_UpperCamelCase  ,negative_pooled_prompt_embeds=_UpperCamelCase  ,)
									snake_case_		:							List[str]			       =						output.images[0, -3:, -3:, -1]

									# make sure that it's equal
									assert np.abs(image_slice_a.flatten() - image_slice_a.flatten()			).max() < 1E-4





@slow
@require_torch_gpu
class 						__UpperCamelCase	(						unittest.TestCase							):





							def 		a__   (  self		:Optional[int]			):
									super().tearDown()
									gc.collect()
									torch.cuda.empty_cache()





							def 		a__   (  self		:List[str]  ,_UpperCamelCase		:Any  ,_UpperCamelCase		:Tuple="cpu"  ,_UpperCamelCase		:List[str]=torch.floataa  ,_UpperCamelCase		:List[str]=0			):
									snake_case_		:							Tuple			       =						torch.Generator(device=_UpperCamelCase			).manual_seed(_UpperCamelCase			)
									snake_case_		:							Dict			       =						np.random.RandomState(_UpperCamelCase			).standard_normal((1, 4, 6_4, 6_4)			)
									snake_case_		:							int			       =						torch.from_numpy(_UpperCamelCase			).to(device=_UpperCamelCase  ,dtype=_UpperCamelCase			)
									snake_case_		:							List[str]			       =						{
									    """prompt""": """a photograph of an astronaut riding a horse""",
									    """latents""": latents,
									    """generator""": generator,
									    """num_inference_steps""": 3,
									    """guidance_scale""": 7.5,
									    """output_type""": """numpy""",
									}
									return inputs





							def 		a__   (  self		:str			):
									snake_case_		:							Dict			       =						DiffusionPipeline.from_pretrained("""stabilityai/stable-diffusion-2-base"""			)
									pipe.to(_UpperCamelCase			)
									pipe.set_progress_bar_config(disable=_UpperCamelCase			)

									snake_case_		:							int			       =						self.get_inputs(_UpperCamelCase			)
									snake_case_		:							Optional[Any]			       =						pipe(**_UpperCamelCase			).images
									snake_case_		:							int			       =						image[0, -3:, -3:, -1].flatten()

									assert image.shape == (1, 5_1_2, 5_1_2, 3)
									snake_case_		:							Any			       =						np.array([0.4_94_93, 0.4_78_96, 0.4_07_98, 0.5_42_14, 0.5_32_12, 0.4_82_02, 0.4_76_56, 0.4_63_29, 0.4_85_06]			)
									assert np.abs(image_slice - expected_slice			).max() < 7E-3



'''simple docstring'''






import argparse
import json
import os

import torch
from torch import nn

from transformers import NllbMoeConfig, NllbMoeModel
from transformers.modeling_utils import dtype_byte_size
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME





def      UpperCAmelCase      (		lowerCamelCase_  :List[Any] ):







		'''simple docstring'''
		snake_case_		:							Tuple			       =						[
		    """encoder.version""",
		    """decoder.version""",
		    """model.encoder.version""",
		    """model.decoder.version""",
		    """decoder.output_projection.weight""",
		    """_float_tensor""",
		    """encoder.embed_positions._float_tensor""",
		    """decoder.embed_positions._float_tensor""",
		]
		for k in ignore_keys:
				state_dict.pop(lowerCamelCase_				,      lowerCamelCase_ )





def      UpperCAmelCase      (		lowerCamelCase_  :List[Any] ):







		'''simple docstring'''
		snake_case_       ,      snake_case_		:							Dict			       =						emb.weight.shape
		snake_case_		:							str			       =						nn.Linear(lowerCamelCase_				,      lowerCamelCase_				,      bias=lowerCamelCase_ )
		snake_case_		:							str			       =						emb.weight.data
		return lin_layer





def      UpperCAmelCase      (		lowerCamelCase_  :List[Any]				,      lowerCamelCase_  :List[str]=None ):







		'''simple docstring'''
		snake_case_		:							Union[str, Any]			       =						{}
		for old_key in state_dict.keys():
				snake_case_		:							Optional[int]			       =						old_key
				if "moe_layer.experts." in key:
						if expert_idx is not None:
								snake_case_		:							Dict			       =						key.replace("""moe_layer.experts.0"""				,      F'''ffn.experts.expert_{expert_idx}''' )
						else:
								snake_case_		:							str			       =						key.replace("""moe_layer.experts."""				,      """ffn.experts.expert_""" )
				if "gate" in key:
						snake_case_		:							List[str]			       =						key.replace(""".moe_layer.gate.wg"""				,      """.ffn.router.classifier""" )
				if "fc2" and "experts" not in key:
						snake_case_		:							Any			       =						key.replace(""".fc2."""				,      """.ffn.fc2.""" )
				if "fc1" and "experts" not in key:
						snake_case_		:							str			       =						key.replace(""".fc1."""				,      """.ffn.fc1.""" )
				if ".encoder_attn." in key:
						snake_case_		:							str			       =						key.replace(""".encoder_attn."""				,      """.cross_attention.""" )
				if "encoder_attn_layer_norm" in key:
						snake_case_		:							List[str]			       =						key.replace("""encoder_attn_layer_norm"""				,      """cross_attention_layer_norm""" )
				if "final_layer_norm" in key:
						snake_case_		:							Dict			       =						key.replace("""final_layer_norm"""				,      """ff_layer_norm""" )
				snake_case_		:							Dict			       =						state_dict[old_key]
		return new_dict





def      UpperCAmelCase      (		lowerCamelCase_  :Tuple				,      lowerCamelCase_  :Optional[Any]				,      lowerCamelCase_  :Union[str, Any]				,      lowerCamelCase_  :Any				,      lowerCamelCase_  :str = WEIGHTS_NAME ):







		'''simple docstring'''
		snake_case_		:							Tuple			       =						[]
		snake_case_		:							Dict			       =						0
		os.makedirs(lowerCamelCase_				,      exist_ok=lowerCamelCase_ )

		for expert in range(lowerCamelCase_ ):
				snake_case_		:							Optional[Any]			       =						switch_checkpoint_path + F'''-rank-{expert}.pt'''
				if os.path.isfile(lowerCamelCase_ ):
						snake_case_		:							List[Any]			       =						torch.load(lowerCamelCase_ )["""model"""]
						remove_ignore_keys_(lowerCamelCase_ )
						snake_case_		:							List[str]			       =						rename_fairseq_keys(lowerCamelCase_				,      lowerCamelCase_ )
						snake_case_		:							List[str]			       =						os.path.join(
						    lowerCamelCase_				,      weights_name.replace(""".bin"""				,      F'''-{len(lowerCamelCase_ )+1:05d}-of-???.bin''' ) )
						torch.save(lowerCamelCase_				,      lowerCamelCase_ )
						sharded_state_dicts.append(expert_state.keys() )
						total_size += sum([value.numel() for key, value in expert_state.items()] ) * dtype_byte_size(
						    expert_state[list(lowerCamelCase_ )[0]].dtype )

    # Add the last block
		snake_case_		:							Tuple			       =						os.path.join(lowerCamelCase_				,      weights_name.replace(""".bin"""				,      F'''-{len(lowerCamelCase_ )+1:05d}-of-???.bin''' ) )
		snake_case_		:							Tuple			       =						torch.load(switch_checkpoint_path + """-shared.pt""" )["""model"""]
		remove_ignore_keys_(lowerCamelCase_ )
		snake_case_		:							Tuple			       =						rename_fairseq_keys(lowerCamelCase_				,      lowerCamelCase_ )
		snake_case_		:							List[str]			       =						shared_weights["""decoder.embed_tokens.weight"""]
		sharded_state_dicts.append(shared_weights.keys() )

		# If we only have the shared weights (dummy model/experts saved on the same file)
		if len(lowerCamelCase_ ) == 1:
				snake_case_		:							Union[str, Any]			       =						os.path.join(lowerCamelCase_				,      lowerCamelCase_ )
				torch.save(lowerCamelCase_				,      lowerCamelCase_ )
				return {weights_name: sharded_state_dicts[0]}, None
		else:
				torch.save(lowerCamelCase_				,      lowerCamelCase_ )
		# Otherwise, let's build the index
		snake_case_		:							str			       =						{}
		for idx, shard in enumerate(lowerCamelCase_ ):
				snake_case_		:							List[str]			       =						weights_name.replace(""".bin"""				,      F'''-{idx+1:05d}-of-{len(lowerCamelCase_ ):05d}.bin''' )
				snake_case_		:							Optional[int]			       =						os.path.join(lowerCamelCase_				,      weights_name.replace(""".bin"""				,      F'''-{idx+1:05d}-of-???.bin''' ) )
				os.rename(lowerCamelCase_				,      os.path.join(lowerCamelCase_				,      lowerCamelCase_ ) )
				for key in shard:
						snake_case_		:							Optional[int]			       =						shard_file

    # Add the metadata
		snake_case_		:							Any			       =						{"""total_size""": total_size}
		snake_case_		:							int			       =						{"""metadata""": metadata, """weight_map""": weight_map}

		with open(os.path.join(lowerCamelCase_				,      lowerCamelCase_ )				,      """w"""				,      encoding="""utf-8""" ) as f:
				snake_case_		:							List[str]			       =						json.dumps(lowerCamelCase_				,      indent=2				,      sort_keys=lowerCamelCase_ ) + """\n"""
				f.write(lowerCamelCase_ )

		return metadata, index


if __name__ == "__main__":
							__A	:	int     =     argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '--nllb_moe_checkpoint_path',
							    default='/home/arthur_huggingface_co/fairseq/weights/checkpoints/model_moe_54b/checkpoint_2_300000',
							    type=str,
							    required=False,
							    help='Path to a directory containing a folder per layer. Follows the original Google format.',
							)
							parser.add_argument('--dtype', default='float32', type=str, required=False, help='dtype of the saved model')
							parser.add_argument(
							    '--pytorch_dump_folder_path',
							    default='/home/arthur_huggingface_co/fairseq/weights/checkpoints/hf-converted-moe-54b',
							    type=str,
							    required=False,
							    help='Path to the output pytorch model.',
							)
							__A	:	List[str]     =     parser.parse_args()
							__A,   __A	:	List[Any]     =     shard_on_the_fly(
							    args.nllb_moe_checkpoint_path,
							    args.pytorch_dump_folder_path,
							    128,
							    args.dtype,
							)

							__A	:	Tuple     =     NllbMoeConfig.from_pretrained(
							    'facebook/nllb-200-3.3B', encoder_sparse_step=4, decoder_sparse_step=4, num_experts=128
							)
							config.save_pretrained(args.pytorch_dump_folder_path)
							__A	:	List[str]     =     NllbMoeModel.from_pretrained(args.pytorch_dump_folder_path)
							print('Done')
							model.save_pretrained(args.pytorch_dump_folder_path)