

import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image

from diffusers import (
    DDIMScheduler,
    KandinskyVaaControlnetImgaImgPipeline,
    KandinskyVaaPriorEmbaEmbPipeline,
    UNetaDConditionModel,
    VQModel,
)
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference


enable_full_determinism()



class        _a		(_SCREAMING_SNAKE_CASE						,  unittest.TestCase							):


					'''simple docstring'''






					UpperCAmelCase__:  Dict    				=			KandinskyVaaControlnetImgaImgPipeline
					UpperCAmelCase__:  int    				=			['''image_embeds''', '''negative_image_embeds''', '''image''', '''hint''']
					UpperCAmelCase__:  Tuple    				=			['''image_embeds''', '''negative_image_embeds''', '''image''', '''hint''']
					UpperCAmelCase__:  Dict    				=			[
					    '''generator''',
					    '''height''',
					    '''width''',
					    '''strength''',
					    '''guidance_scale''',
					    '''num_inference_steps''',
					    '''return_dict''',
					    '''guidance_scale''',
					    '''num_images_per_prompt''',
					    '''output_type''',
					    '''return_dict''',
					]
					UpperCAmelCase__:  str    				=			False






					@property
					def     __A     (					self     ):
						return 32






					@property
					def     __A     (					self     ):
						return 32






					@property
					def     __A     (					self     ):
						return self.time_input_dim






					@property
					def     __A     (					self     ):
						return self.time_input_dim * 4






					@property
					def     __A     (					self     ):
						return 100






					@property
					def     __A     (					self     ):
						torch.manual_seed(0     )

						A__				:       Union[str, Any]	  =						{
						    """in_channels""": 8,
						    # Out channels is double in channels because predicts mean and variance
						    """out_channels""": 8,
						    """addition_embed_type""": """image_hint""",
						    """down_block_types""": ("""ResnetDownsampleBlock2D""", """SimpleCrossAttnDownBlock2D"""),
						    """up_block_types""": ("""SimpleCrossAttnUpBlock2D""", """ResnetUpsampleBlock2D"""),
						    """mid_block_type""": """UNetMidBlock2DSimpleCrossAttn""",
						    """block_out_channels""": (self.block_out_channels_a, self.block_out_channels_a * 2),
						    """layers_per_block""": 1,
						    """encoder_hid_dim""": self.text_embedder_hidden_size,
						    """encoder_hid_dim_type""": """image_proj""",
						    """cross_attention_dim""": self.cross_attention_dim,
						    """attention_head_dim""": 4,
						    """resnet_time_scale_shift""": """scale_shift""",
						    """class_embed_type""": None,
						}

						A__				:       List[Any]	  =						UNetaDConditionModel(**_lowercase     )
						return model






					@property
					def     __A     (					self     ):
						return {
						    "block_out_channels": [32, 32, 64, 64],
						    "down_block_types": [
						        "DownEncoderBlock2D",
						        "DownEncoderBlock2D",
						        "DownEncoderBlock2D",
						        "AttnDownEncoderBlock2D",
						    ],
						    "in_channels": 3,
						    "latent_channels": 4,
						    "layers_per_block": 1,
						    "norm_num_groups": 8,
						    "norm_type": "spatial",
						    "num_vq_embeddings": 12,
						    "out_channels": 3,
						    "up_block_types": ["AttnUpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D"],
						    "vq_embed_dim": 4,
						}






					@property
					def     __A     (					self     ):
						torch.manual_seed(0     )
						A__				:       Optional[Any]	  =						VQModel(**self.dummy_movq_kwargs     )
						return model






					def     __A     (					self     ):
						A__				:       int	  =						self.dummy_unet
						A__				:       Dict	  =						self.dummy_movq

						A__				:       List[str]	  =						{
						    """num_train_timesteps""": 1000,
						    """beta_schedule""": """linear""",
						    """beta_start""": 0.0_0_0_8_5,
						    """beta_end""": 0.0_1_2,
						    """clip_sample""": False,
						    """set_alpha_to_one""": False,
						    """steps_offset""": 0,
						    """prediction_type""": """epsilon""",
						    """thresholding""": False,
						}

						A__				:       str	  =						DDIMScheduler(**_lowercase     )

						A__				:       int	  =						{
						    """unet""": unet,
						    """scheduler""": scheduler,
						    """movq""": movq,
						}

						return components






					def     __A     (					self       ,			A__       ,			A__=0     ):
						A__				:       Optional[int]	  =						floats_tensor((1, self.text_embedder_hidden_size)       ,			rng=random.Random(_lowercase     )     ).to(_lowercase     )
						A__				:       List[Any]	  =						floats_tensor((1, self.text_embedder_hidden_size)       ,			rng=random.Random(seed + 1     )     ).to(
						    _lowercase     )
						# create init_image
						A__				:       int	  =						floats_tensor((1, 3, 64, 64)       ,			rng=random.Random(_lowercase     )     ).to(_lowercase     )
						A__				:       Dict	  =						image.cpu().permute(0       ,			2       ,			3       ,			1     )[0]
						A__				:       Optional[int]	  =						Image.fromarray(np.uinta(_lowercase     )     ).convert("""RGB"""     ).resize((256, 256)     )
						# create hint
						A__				:       List[str]	  =						floats_tensor((1, 3, 64, 64)       ,			rng=random.Random(_lowercase     )     ).to(_lowercase     )

						if str(_lowercase     ).startswith("""mps"""     ):
							A__				:       str	  =						torch.manual_seed(_lowercase     )
						else:
							A__				:       Dict	  =						torch.Generator(device=_lowercase     ).manual_seed(_lowercase     )
						A__				:       List[Any]	  =						{
						    """image""": init_image,
						    """image_embeds""": image_embeds,
						    """negative_image_embeds""": negative_image_embeds,
						    """hint""": hint,
						    """generator""": generator,
						    """height""": 64,
						    """width""": 64,
						    """num_inference_steps""": 10,
						    """guidance_scale""": 7.0,
						    """strength""": 0.2,
						    """output_type""": """np""",
						}
						return inputs






					def     __A     (					self     ):
						A__				:       Tuple	  =						"""cpu"""

						A__				:       int	  =						self.get_dummy_components()

						A__				:       Tuple	  =						self.pipeline_class(**_lowercase     )
						A__				:       str	  =						pipe.to(_lowercase     )

						pipe.set_progress_bar_config(disable=_lowercase     )

						A__				:       Tuple	  =						pipe(**self.get_dummy_inputs(_lowercase     )     )
						A__				:       Union[str, Any]	  =						output.images

						A__				:       List[Any]	  =						pipe(
						    **self.get_dummy_inputs(_lowercase     )       ,			return_dict=_lowercase       ,			)[0]

						A__				:       Optional[Any]	  =						image[0, -3:, -3:, -1]
						A__				:       List[str]	  =						image_from_tuple[0, -3:, -3:, -1]

						assert image.shape == (1, 64, 64, 3)

						A__				:       List[str]	  =						np.array(
						    [0.5_4_9_8_5_0_3_4, 0.5_5_5_0_9_3_6_5, 0.5_2_5_6_1_5_0_4, 0.5_5_7_0_4_9_4, 0.5_5_9_3_8_1_8, 0.5_2_6_3_9_7_9, 0.5_0_2_8_5_6_4_3, 0.5_0_6_9_8_4_6, 0.5_1_1_9_6_7_3_6]     )
						assert (
						    np.abs(image_slice.flatten() - expected_slice     ).max() < 1e-2
						), F""" expected_slice {expected_slice}, but got {image_slice.flatten()}"""
						assert (
						    np.abs(image_from_tuple_slice.flatten() - expected_slice     ).max() < 1e-2
						), F""" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}"""



@slow
@require_torch_gpu
class        _a		(unittest.TestCase							):


					'''simple docstring'''






					def     __A     (					self     ):
						super().tearDown()
						gc.collect()
						torch.cuda.empty_cache()






					def     __A     (					self     ):
						A__				:       Dict	  =						load_numpy(
						    """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"""
						    """/kandinskyv22/kandinskyv22_controlnet_img2img_robotcat_fp16.npy"""     )

						A__				:       Dict	  =						load_image(
						    """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main""" """/kandinsky/cat.png"""     )
						A__				:       Union[str, Any]	  =						init_image.resize((512, 512)     )

						A__				:       List[Any]	  =						load_image(
						    """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"""
						    """/kandinskyv22/hint_image_cat.png"""     )
						A__				:       Any	  =						torch.from_numpy(np.array(_lowercase     )     ).float() / 2_5_5.0
						A__				:       List[Any]	  =						hint.permute(2       ,			0       ,			1     ).unsqueeze(0     )

						A__				:       List[Any]	  =						"""A robot, 4k photo"""

						A__				:       Tuple	  =						KandinskyVaaPriorEmbaEmbPipeline.from_pretrained(
						    """kandinsky-community/kandinsky-2-2-prior"""       ,			torch_dtype=torch.floataa     )
						pipe_prior.to(_lowercase     )

						A__				:       Optional[int]	  =						KandinskyVaaControlnetImgaImgPipeline.from_pretrained(
						    """kandinsky-community/kandinsky-2-2-controlnet-depth"""       ,			torch_dtype=torch.floataa     )
						A__				:       Dict	  =						pipeline.to(_lowercase     )

						pipeline.set_progress_bar_config(disable=_lowercase     )

						A__				:       List[str]	  =						torch.Generator(device="""cpu"""     ).manual_seed(0     )

						A__  ,  A__				:       Tuple	  =						pipe_prior(
						    _lowercase       ,			image=_lowercase       ,			strength=0.8_5       ,			generator=_lowercase       ,			negative_prompt=""""""       ,			).to_tuple()

						A__				:       List[str]	  =						pipeline(
						    image=_lowercase       ,			image_embeds=_lowercase       ,			negative_image_embeds=_lowercase       ,			hint=_lowercase       ,			generator=_lowercase       ,			num_inference_steps=100       ,			height=512       ,			width=512       ,			strength=0.5       ,			output_type="""np"""       ,			)

						A__				:       List[Any]	  =						output.images[0]

						assert image.shape == (512, 512, 3)

						assert_mean_pixel_difference(_lowercase       ,			_lowercase     )





'''simple docstring'''



import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
						from transformers import LlamaTokenizerFast
except ImportError as e:
						warnings.warn(e)
						warnings.warn(
						    """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
						)
						_lowercase          =						None
_lowercase          =						{
    """7B""": 11008,
    """13B""": 13824,
    """30B""": 17920,
    """65B""": 22016,
    """70B""": 28672,
}
_lowercase          =						{
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}


def A (__lowerCamelCase :int					,      __lowerCamelCase :Optional[Any]=1					,      __lowerCamelCase :List[Any]=256						):
						return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3						)						) + multiple_of - 1) // multiple_of)


def A (__lowerCamelCase :Any						):
						with open(__lowerCamelCase					,      """r"""						) as f:
												return json.load(__lowerCamelCase						)


def A (__lowerCamelCase :List[Any]					,      __lowerCamelCase :int						):
						with open(__lowerCamelCase					,      """w"""						) as f:
												json.dump(__lowerCamelCase					,      __lowerCamelCase						)


def A (__lowerCamelCase :Optional[Any]					,      __lowerCamelCase :Tuple					,      __lowerCamelCase :Optional[Any]					,      __lowerCamelCase :Tuple=True						):
						os.makedirs(__lowerCamelCase					,      exist_ok=__lowerCamelCase						)
						_lowerCAmelCase    		=					os.path.join(__lowerCamelCase					,      """tmp"""						)
						os.makedirs(__lowerCamelCase					,      exist_ok=__lowerCamelCase						)

						_lowerCAmelCase    		=					read_json(os.path.join(__lowerCamelCase					,      """params.json"""						)						)
						_lowerCAmelCase    		=					NUM_SHARDS[model_size]
						_lowerCAmelCase    		=					params["""n_layers"""]
						_lowerCAmelCase    		=					params["""n_heads"""]
						_lowerCAmelCase    		=					n_heads // num_shards
						_lowerCAmelCase    		=					params["""dim"""]
						_lowerCAmelCase    		=					dim // n_heads
						_lowerCAmelCase    		=					10_000.0
						_lowerCAmelCase    		=					1.0 / (base ** (torch.arange(0					,      __lowerCamelCase					,      2						).float() / dims_per_head))

						if "n_kv_heads" in params:
												_lowerCAmelCase    		=					params["""n_kv_heads"""]  # for GQA / MQA
												_lowerCAmelCase    		=					n_heads_per_shard // num_key_value_heads
												_lowerCAmelCase    		=					dim // num_key_value_heads
						else:  # compatibility with other checkpoints
												_lowerCAmelCase    		=					n_heads
												_lowerCAmelCase    		=					n_heads_per_shard
												_lowerCAmelCase    		=					dim

						# permute for sliced rotary
						def permute(__lowerCamelCase :Optional[int]					,      __lowerCamelCase :str=n_heads					,      __lowerCamelCase :str=dim					,      __lowerCamelCase :List[Any]=dim						):
												return w.view(__lowerCamelCase					,      dima // n_heads // 2					,      2					,      __lowerCamelCase						).transpose(1					,      2						).reshape(__lowerCamelCase					,      __lowerCamelCase						)

						print(f'Fetching all parameters from the checkpoint at {input_base_path}.'						)
						# Load weights
						if model_size == "7B":
												# Not sharded
												# (The sharded implementation would also work, but this is simpler.)
												_lowerCAmelCase    		=					torch.load(os.path.join(__lowerCamelCase					,      """consolidated.00.pth"""						)					,      map_location="""cpu"""						)
						else:
												# Sharded
												_lowerCAmelCase    		=					[
												    torch.load(os.path.join(__lowerCamelCase					,      f'consolidated.{i:02d}.pth'						)					,      map_location="""cpu"""						)
												    for i in range(__lowerCamelCase						)
												]
						_lowerCAmelCase    		=					0
						_lowerCAmelCase    		=					{"""weight_map""": {}}
						for layer_i in range(__lowerCamelCase						):
												_lowerCAmelCase    		=					f'pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'
												if model_size == "7B":
																		# Unsharded
																		_lowerCAmelCase    		=					{
																		    f'model.layers.{layer_i}.self_attn.q_proj.weight': permute(
																		        loaded[f'layers.{layer_i}.attention.wq.weight']						),
																		    f'model.layers.{layer_i}.self_attn.k_proj.weight': permute(
																		        loaded[f'layers.{layer_i}.attention.wk.weight']						),
																		    f'model.layers.{layer_i}.self_attn.v_proj.weight': loaded[f'layers.{layer_i}.attention.wv.weight'],
																		    f'model.layers.{layer_i}.self_attn.o_proj.weight': loaded[f'layers.{layer_i}.attention.wo.weight'],
																		    f'model.layers.{layer_i}.mlp.gate_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w1.weight'],
																		    f'model.layers.{layer_i}.mlp.down_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w2.weight'],
																		    f'model.layers.{layer_i}.mlp.up_proj.weight': loaded[f'layers.{layer_i}.feed_forward.w3.weight'],
																		    f'model.layers.{layer_i}.input_layernorm.weight': loaded[f'layers.{layer_i}.attention_norm.weight'],
																		    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[f'layers.{layer_i}.ffn_norm.weight'],
																		}
												else:
																		# Sharded
																		# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
																		# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
																		# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

																		_lowerCAmelCase    		=					{
																		    f'model.layers.{layer_i}.input_layernorm.weight': loaded[0][
																		        f'layers.{layer_i}.attention_norm.weight'
																		    ].clone(),
																		    f'model.layers.{layer_i}.post_attention_layernorm.weight': loaded[0][
																		        f'layers.{layer_i}.ffn_norm.weight'
																		    ].clone(),
																		}
																		_lowerCAmelCase    		=					permute(
																		    torch.cat(
																		        [
																		            loaded[i][f'layers.{layer_i}.attention.wq.weight'].view(__lowerCamelCase					,      __lowerCamelCase					,      __lowerCamelCase						)
																		            for i in range(__lowerCamelCase						)
																		        ]					,      dim=0					,      ).reshape(__lowerCamelCase					,      __lowerCamelCase						)						)
																		_lowerCAmelCase    		=					permute(
																		    torch.cat(
																		        [
																		            loaded[i][f'layers.{layer_i}.attention.wk.weight'].view(
																		                __lowerCamelCase					,      __lowerCamelCase					,      __lowerCamelCase						)
																		            for i in range(__lowerCamelCase						)
																		        ]					,      dim=0					,      ).reshape(__lowerCamelCase					,      __lowerCamelCase						)					,      __lowerCamelCase					,      __lowerCamelCase					,      __lowerCamelCase					,      )
																		_lowerCAmelCase    		=					torch.cat(
																		    [
																		        loaded[i][f'layers.{layer_i}.attention.wv.weight'].view(
																		            __lowerCamelCase					,      __lowerCamelCase					,      __lowerCamelCase						)
																		        for i in range(__lowerCamelCase						)
																		    ]					,      dim=0					,      ).reshape(__lowerCamelCase					,      __lowerCamelCase						)

																		_lowerCAmelCase    		=					torch.cat(
																		    [loaded[i][f'layers.{layer_i}.attention.wo.weight'] for i in range(__lowerCamelCase						)]					,      dim=1						)
																		_lowerCAmelCase    		=					torch.cat(
																		    [loaded[i][f'layers.{layer_i}.feed_forward.w1.weight'] for i in range(__lowerCamelCase						)]					,      dim=0						)
																		_lowerCAmelCase    		=					torch.cat(
																		    [loaded[i][f'layers.{layer_i}.feed_forward.w2.weight'] for i in range(__lowerCamelCase						)]					,      dim=1						)
																		_lowerCAmelCase    		=					torch.cat(
																		    [loaded[i][f'layers.{layer_i}.feed_forward.w3.weight'] for i in range(__lowerCamelCase						)]					,      dim=0						)

												_lowerCAmelCase    		=					inv_freq
												for k, v in state_dict.items():
																		_lowerCAmelCase    		=					filename
																		param_count += v.numel()
												torch.save(__lowerCamelCase					,      os.path.join(__lowerCamelCase					,      __lowerCamelCase						)						)

						_lowerCAmelCase    		=					f'pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'
						if model_size == "7B":
												# Unsharded
												_lowerCAmelCase    		=					{
												    """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
												    """model.norm.weight""": loaded["""norm.weight"""],
												    """lm_head.weight""": loaded["""output.weight"""],
												}
						else:
												_lowerCAmelCase    		=					{
												    """model.norm.weight""": loaded[0]["""norm.weight"""],
												    """model.embed_tokens.weight""": torch.cat(
												        [loaded[i]["""tok_embeddings.weight"""] for i in range(__lowerCamelCase						)]					,      dim=1						),
												    """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(__lowerCamelCase						)]					,      dim=0						),
												}

						for k, v in state_dict.items():
												_lowerCAmelCase    		=					filename
												param_count += v.numel()
						torch.save(__lowerCamelCase					,      os.path.join(__lowerCamelCase					,      __lowerCamelCase						)						)

						# Write configs
						_lowerCAmelCase    		=					{"""total_size""": param_count * 2}
						write_json(__lowerCamelCase					,      os.path.join(__lowerCamelCase					,      """pytorch_model.bin.index.json"""						)						)
						_lowerCAmelCase    		=					params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
						_lowerCAmelCase    		=					params["""multiple_of"""] if """multiple_of""" in params else 256
						_lowerCAmelCase    		=					LlamaConfig(
						    hidden_size=__lowerCamelCase					,      intermediate_size=compute_intermediate_size(__lowerCamelCase					,      __lowerCamelCase					,      __lowerCamelCase						)					,      num_attention_heads=params["""n_heads"""]					,      num_hidden_layers=params["""n_layers"""]					,      rms_norm_eps=params["""norm_eps"""]					,      num_key_value_heads=__lowerCamelCase					,      )
						config.save_pretrained(__lowerCamelCase						)

						# Make space so we can load the model properly now.
						del state_dict
						del loaded
						gc.collect()

						print("""Loading the checkpoint in a Llama model."""						)
						_lowerCAmelCase    		=					LlamaForCausalLM.from_pretrained(__lowerCamelCase					,      torch_dtype=torch.floataa					,      low_cpu_mem_usage=__lowerCamelCase						)
						# Avoid saving this as part of the config.
						del model.config._name_or_path

						print("""Saving in the Transformers format."""						)
						model.save_pretrained(__lowerCamelCase					,      safe_serialization=__lowerCamelCase						)
						shutil.rmtree(__lowerCamelCase						)


def A (__lowerCamelCase :Union[str, Any]					,      __lowerCamelCase :Union[str, Any]						):
						# Initialize the tokenizer based on the `spm` model
						_lowerCAmelCase    		=					LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
						print(f'Saving a {tokenizer_class.__name__} to {tokenizer_path}.'						)
						_lowerCAmelCase    		=					tokenizer_class(__lowerCamelCase						)
						tokenizer.save_pretrained(__lowerCamelCase						)


def A ():
						_lowerCAmelCase    		=					argparse.ArgumentParser()
						parser.add_argument(
						    """--input_dir"""					,      help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""					,      )
						parser.add_argument(
						    """--model_size"""					,      choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]					,      )
						parser.add_argument(
						    """--output_dir"""					,      help="""Location to write HF model and tokenizer"""					,      )
						parser.add_argument("""--safe_serialization"""					,      type=__lowerCamelCase					,      help="""Whether or not to save using `safetensors`."""						)
						_lowerCAmelCase    		=					parser.parse_args()
						if args.model_size != "tokenizer_only":
												write_model(
												    model_path=args.output_dir					,      input_base_path=os.path.join(args.input_dir					,      args.model_size						)					,      model_size=args.model_size					,      safe_serialization=args.safe_serialization					,      )
						_lowerCAmelCase    		=					os.path.join(args.input_dir					,      """tokenizer.model"""						)
						write_tokenizer(args.output_dir					,      __lowerCamelCase						)


if __name__ == "__main__":
						main()
