





'''simple docstring'''


import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

_SCREAMING_SNAKE_CASE									=				[
    "word_embeddings_layernorm.weight",
    "word_embeddings_layernorm.bias",
    "input_layernorm.weight",
    "input_layernorm.bias",
    "post_attention_layernorm.weight",
    "post_attention_layernorm.bias",
    "self_attention.dense.bias",
    "mlp.dense_4h_to_h.bias",
    "ln_f.weight",
    "ln_f.bias",
]

_SCREAMING_SNAKE_CASE									=				[
    "mlp.dense_4h_to_h.weight",
    "self_attention.dense.weight",
]


def 						__a(SCREAMING_SNAKE_CASE_	:       List[Any]		,							SCREAMING_SNAKE_CASE_	:       Any     ):



    '''simple docstring'''





    _lowerCAmelCase						      =   {
        "word_embeddings.weight": "word_embeddings.weight",
        "word_embeddings.norm.weight": "word_embeddings_layernorm.weight",
        "word_embeddings.norm.bias": "word_embeddings_layernorm.bias",
        "weight": "ln_f.weight",
        "bias": "ln_f.bias",
    }

    if key in layer_rename_map:
        return layer_rename_map[key]

    # Handle transformer blocks
    _lowerCAmelCase						      =   int(re.match(R".*layer_(\d*).*"		,							SCREAMING_SNAKE_CASE_     )[1]     )
    layer_number -= 3
    return F'''h.{layer_number}.''' + key


def 						__a(SCREAMING_SNAKE_CASE_	:       Dict     ):



    '''simple docstring'''





    if dtype == torch.bool:
        return 1 / 8
    _lowerCAmelCase						      =   re.search(R"[^\d](\d+)$"		,							str(SCREAMING_SNAKE_CASE_     )     )
    if bit_search is None:
        raise ValueError(F'''`dtype` is not a valid dtype: {dtype}.'''     )
    _lowerCAmelCase						      =   int(bit_search.groups()[0]     )
    return bit_size // 8


def 						__a(SCREAMING_SNAKE_CASE_	:       Optional[Any]		,							SCREAMING_SNAKE_CASE_	:       Dict		,							SCREAMING_SNAKE_CASE_	:       Union[str, Any]		,							SCREAMING_SNAKE_CASE_	:       Optional[int]		,							SCREAMING_SNAKE_CASE_	:       List[Any]     ):



    '''simple docstring'''





    if bloom_config_file == "":
        _lowerCAmelCase						      =   BloomConfig()
    else:
        _lowerCAmelCase						      =   BloomConfig.from_json_file(SCREAMING_SNAKE_CASE_     )

    if shard_model:
        _lowerCAmelCase						      =   os.listdir(SCREAMING_SNAKE_CASE_     )
        _lowerCAmelCase						      =   sorted(filter(lambda SCREAMING_SNAKE_CASE_     : s.startswith("layer"     ) and "model_00" in s		,							SCREAMING_SNAKE_CASE_     )     )

        _lowerCAmelCase						      =   {"weight_map": {}, "metadata": {}}
        _lowerCAmelCase						      =   0

        _lowerCAmelCase						      =   None

        _lowerCAmelCase						      =   BloomConfig()

        for j, file in enumerate(SCREAMING_SNAKE_CASE_     ):
            print("Processing file: {}".format(SCREAMING_SNAKE_CASE_     )     )
            _lowerCAmelCase						      =   None

            for i in range(SCREAMING_SNAKE_CASE_     ):
                # load all TP files
                _lowerCAmelCase						      =   file.replace("model_00"		,							F'''model_0{i}'''     )
                _lowerCAmelCase						      =   torch.load(os.path.join(SCREAMING_SNAKE_CASE_		,							SCREAMING_SNAKE_CASE_     )		,							map_location="cpu"     )

                # Rename keys in the transformers names
                _lowerCAmelCase						      =   list(temp.keys()     )
                for key in keys:
                    _lowerCAmelCase						      =   temp.pop(SCREAMING_SNAKE_CASE_     )

                if tensors is None:
                    _lowerCAmelCase						      =   temp
                else:
                    for key in tensors.keys():
                        if any(key.endswith(SCREAMING_SNAKE_CASE_     ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH     ):
                            # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                            tensors[key] += temp[key]
                        else:
                            # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                            _lowerCAmelCase						      =   1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN     ) else 0
                            # We concatenate these weights accross TP ranks
                            _lowerCAmelCase						      =   torch.cat([tensors[key], temp[key]]		,							dim=SCREAMING_SNAKE_CASE_     )

            # Divide by the number of TP the weights we want to average
            for key in tensors.keys():
                if any(key.endswith(SCREAMING_SNAKE_CASE_     ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH     ):
                    _lowerCAmelCase						      =   tensors[key] / pretraining_tp
            torch.save(
                SCREAMING_SNAKE_CASE_		,							os.path.join(
                    SCREAMING_SNAKE_CASE_		,							"pytorch_model_{}-of-{}.bin".format(str(j + 1     ).zfill(5     )		,							str(len(SCREAMING_SNAKE_CASE_     )     ).zfill(5     )     )		,							)		,							)

            for key in tensors.keys():
                _lowerCAmelCase						      =   tensors[key]
                total_size += value.numel() * get_dtype_size(value.dtype     )
                if key not in index_dict["weight_map"]:
                    _lowerCAmelCase						      =   "pytorch_model_{}-of-{}.bin".format(
                        str(j + 1     ).zfill(5     )		,							str(len(SCREAMING_SNAKE_CASE_     )     ).zfill(5     )     )

        _lowerCAmelCase						      =   BloomConfig()
        _lowerCAmelCase						      =   pytorch_dump_folder_path + "/" + CONFIG_NAME
        _lowerCAmelCase						      =   total_size
        with open(SCREAMING_SNAKE_CASE_		,							"w"		,							encoding="utf-8"     ) as f:
            f.write(config.to_json_string()     )
        with open(os.path.join(SCREAMING_SNAKE_CASE_		,							WEIGHTS_NAME + ".index.json"     )		,							"w"		,							encoding="utf-8"     ) as f:
            _lowerCAmelCase						      =   json.dumps(SCREAMING_SNAKE_CASE_		,							indent=2		,							sort_keys=SCREAMING_SNAKE_CASE_     ) + "\n"
            f.write(SCREAMING_SNAKE_CASE_     )
    else:
        _lowerCAmelCase						      =   BloomModel(SCREAMING_SNAKE_CASE_     )

        _lowerCAmelCase						      =   os.listdir(SCREAMING_SNAKE_CASE_     )
        _lowerCAmelCase						      =   sorted(filter(lambda SCREAMING_SNAKE_CASE_     : s.startswith("layer"     ) and "model_00" in s		,							SCREAMING_SNAKE_CASE_     )     )

        _lowerCAmelCase						      =   None
        for i, file in enumerate(SCREAMING_SNAKE_CASE_     ):
            _lowerCAmelCase						      =   None
            for i in range(SCREAMING_SNAKE_CASE_     ):
                # load all TP files
                _lowerCAmelCase						      =   file.replace("model_00"		,							F'''model_0{i}'''     )
                _lowerCAmelCase						      =   torch.load(os.path.join(SCREAMING_SNAKE_CASE_		,							SCREAMING_SNAKE_CASE_     )		,							map_location="cpu"     )

                # Rename keys in the transformers names
                _lowerCAmelCase						      =   list(temp.keys()     )
                for key in keys:
                    _lowerCAmelCase						      =   temp.pop(SCREAMING_SNAKE_CASE_     )

                if tensors is None:
                    _lowerCAmelCase						      =   temp
                else:
                    for key in tensors.keys():
                        # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                        if any(key.endswith(SCREAMING_SNAKE_CASE_     ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH     ):
                            tensors[key] += temp[key]
                        else:
                            # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                            _lowerCAmelCase						      =   1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN     ) else 0
                            # We concatenate these weights accross TP ranks
                            _lowerCAmelCase						      =   torch.cat([tensors[key], temp[key]]		,							dim=SCREAMING_SNAKE_CASE_     )

            # Divide by the number of TP the weights we want to average
            for key in tensors.keys():
                if any(key.endswith(SCREAMING_SNAKE_CASE_     ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH     ):
                    _lowerCAmelCase						      =   tensors[key] / pretraining_tp

            _lowerCAmelCase						      =   model.load_state_dict(SCREAMING_SNAKE_CASE_		,							strict=SCREAMING_SNAKE_CASE_     )
            assert not other_keys.unexpected_keys, F'''The keys {other_keys.unexpected_keys} are unexpected'''
            if missing_keys is None:
                _lowerCAmelCase						      =   set(other_keys.missing_keys     )
            else:
                _lowerCAmelCase						      =   missing_keys.intersection(set(other_keys.missing_keys     )     )

        assert not missing_keys, F'''The keys {missing_keys} are missing'''

        # Save pytorch-model
        os.makedirs(SCREAMING_SNAKE_CASE_		,							exist_ok=SCREAMING_SNAKE_CASE_     )
        _lowerCAmelCase						      =   pytorch_dump_folder_path + "/" + WEIGHTS_NAME
        _lowerCAmelCase						      =   pytorch_dump_folder_path + "/" + CONFIG_NAME
        print(F'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''     )
        if config.torch_dtype is not None:
            _lowerCAmelCase						      =   model.to(config.torch_dtype     )
        torch.save(model.state_dict()		,							SCREAMING_SNAKE_CASE_     )
        print(F'''Save configuration file to {pytorch_config_dump_path}'''     )
        with open(SCREAMING_SNAKE_CASE_		,							"w"		,							encoding="utf-8"     ) as f:
            f.write(config.to_json_string()     )


if __name__ == "__main__":
       _SCREAMING_SNAKE_CASE									=				argparse.ArgumentParser()
       # Required parameters
       parser.add_argument(
           "--bloom_checkpoint_path",
           default=None,
           type=str,
           required=True,
           help="Path to the Megatron-LM checkpoint path.",
       )
       parser.add_argument(
           "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
       )
       parser.add_argument(
           "--bloom_config_file",
           default="",
           type=str,
           help=(
               "An optional config json file corresponding to the pre-trained model. \n"
               "This specifies the model architecture."
           ),
       )
       parser.add_argument(
           "--shard_model",
           action="store_true",
           help="An optional setting to shard the output model \nThis enables sharding the converted checkpoint",
       )
       parser.add_argument(
           "--pretraining_tp",
           default=4,
           type=int,
           help="Pretraining TP rank that has been used when training the model in Megatron-LM \n",
       )
       _SCREAMING_SNAKE_CASE									=				parser.parse_args()
       convert_bloom_checkpoint_to_pytorch(
           args.bloom_checkpoint_path,
           args.bloom_config_file,
           args.pytorch_dump_folder_path,
           args.shard_model,
           args.pretraining_tp,
       )






import random
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    UNetaDConditionModel,
    VideoToVideoSDPipeline,
)
from diffusers.utils import floats_tensor, is_xformers_available, skip_mps
from diffusers.utils.testing_utils import enable_full_determinism, slow, torch_device

from ..pipeline_params import (
    TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_PARAMS,
)
from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()



@skip_mps
class 	__lowerCamelCase			(_a				,      unittest.TestCase						):
       _lowercase											=     VideoToVideoSDPipeline
       _lowercase											=     TEXT_GUIDED_IMAGE_VARIATION_PARAMS.union({"""video"""}						) - {"""image""", """width""", """height"""}
       _lowercase											=     TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS.union({"""video"""}						) - {"""image"""}
       _lowercase											=     PipelineTesterMixin.required_optional_params - {"""latents"""}
       _lowercase											=     False

       # No `output_type`.
       _lowercase											=     frozenset(
           [
               """num_inference_steps""",
               """generator""",
               """latents""",
               """return_dict""",
               """callback""",
               """callback_steps""",
           ]						)







       def 		snake_case_	(			self: List[str]					):






            '''simple docstring'''




            torch.manual_seed(0					)
            __UpperCamelCase =			UNetaDConditionModel(
                block_out_channels=(32, 64, 64, 64),layers_per_block=2,sample_size=32,in_channels=4,out_channels=4,down_block_types=('CrossAttnDownBlock3D', 'CrossAttnDownBlock3D', 'CrossAttnDownBlock3D', 'DownBlock3D'),up_block_types=('UpBlock3D', 'CrossAttnUpBlock3D', 'CrossAttnUpBlock3D', 'CrossAttnUpBlock3D'),cross_attention_dim=32,attention_head_dim=4,)
            __UpperCamelCase =			DDIMScheduler(
                beta_start=0.0_0_0_8_5,beta_end=0.0_1_2,beta_schedule='scaled_linear',clip_sample=A_,set_alpha_to_one=A_,)
            torch.manual_seed(0					)
            __UpperCamelCase =			AutoencoderKL(
                block_out_channels=[32, 64],in_channels=3,out_channels=3,down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D'],up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D'],latent_channels=4,sample_size=128,)
            torch.manual_seed(0					)
            __UpperCamelCase =			CLIPTextConfig(
                bos_token_id=0,eos_token_id=2,hidden_size=32,intermediate_size=37,layer_norm_eps=1E-05,num_attention_heads=4,num_hidden_layers=5,pad_token_id=1,vocab_size=1000,hidden_act='gelu',projection_dim=512,)
            __UpperCamelCase =			CLIPTextModel(A_					)
            __UpperCamelCase =			CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'					)

            __UpperCamelCase =			{
                'unet': unet,
                'scheduler': scheduler,
                'vae': vae,
                'text_encoder': text_encoder,
                'tokenizer': tokenizer,
            }
            return components







       def 		snake_case_	(			self: Union[str, Any],A_: Any,A_: Any=0					):






            '''simple docstring'''




            __UpperCamelCase =			floats_tensor((1, 3, 3, 32, 32),rng=random.Random(A_					)					).to(A_					)

            if str(A_					).startswith('mps'					):
                 __UpperCamelCase =			torch.manual_seed(A_					)
            else:
                 __UpperCamelCase =			torch.Generator(device=A_					).manual_seed(A_					)
            __UpperCamelCase =			{
                'prompt': 'A painting of a squirrel eating a burger',
                'video': video,
                'generator': generator,
                'num_inference_steps': 2,
                'guidance_scale': 6.0,
                'output_type': 'pt',
            }
            return inputs







       def 		snake_case_	(			self: Union[str, Any]					):






            '''simple docstring'''




            __UpperCamelCase =			'cpu'  # ensure determinism for the device-dependent torch.Generator
            __UpperCamelCase =			self.get_dummy_components()
            __UpperCamelCase =			VideoToVideoSDPipeline(**A_					)
            __UpperCamelCase =			sd_pipe.to(A_					)
            sd_pipe.set_progress_bar_config(disable=A_					)

            __UpperCamelCase =			self.get_dummy_inputs(A_					)
            __UpperCamelCase =			'np'
            __UpperCamelCase =			sd_pipe(**A_					).frames
            __UpperCamelCase =			frames[0][-3:, -3:, -1]

            assert frames[0].shape == (32, 32, 3)
            __UpperCamelCase =			np.array([106, 117, 113, 174, 137, 112, 148, 151, 131]					)

            assert np.abs(image_slice.flatten() - expected_slice					).max() < 1E-2







       @unittest.skipIf(
           torch_device != 'cuda' or not is_xformers_available(),reason='XFormers attention is only available with CUDA and `xformers` installed',)
       def 		snake_case_	(			self: Any					):






            '''simple docstring'''




            self._test_xformers_attention_forwardGenerator_pass(test_mean_pixel_difference=A_,expected_max_diff=5E-3					)







       @unittest.skip(reason='Batching needs to be properly figured out first for this pipeline.'					)
       def 		snake_case_	(			self: str					):






            '''simple docstring'''




            pass







       @unittest.skip(reason='Batching needs to be properly figured out first for this pipeline.'					)
       def 		snake_case_	(			self: Optional[Any]					):






            '''simple docstring'''




            pass







       @unittest.skip(reason='`num_images_per_prompt` argument is not supported for this pipeline.'					)
       def 		snake_case_	(			self: int					):






            '''simple docstring'''




            pass







       def 		snake_case_	(			self: Any					):






            '''simple docstring'''




            return super().test_progress_bar()



@slow
@skip_mps
class 	__lowerCamelCase			(unittest.TestCase						):







       def 		snake_case_	(			self: Tuple					):






            '''simple docstring'''




            __UpperCamelCase =			VideoToVideoSDPipeline.from_pretrained('cerspense/zeroscope_v2_XL',torch_dtype=torch.floataa					)
            pipe.enable_model_cpu_offload()

            # 10 frames
            __UpperCamelCase =			torch.Generator(device='cpu'					).manual_seed(0					)
            __UpperCamelCase =			torch.randn((1, 10, 3, 1024, 576),generator=A_					)
            __UpperCamelCase =			video.to('cuda'					)

            __UpperCamelCase =			'Spiderman is surfing'

            __UpperCamelCase =			pipe(A_,video=A_,generator=A_,num_inference_steps=3,output_type='pt'					).frames

            __UpperCamelCase =			np.array([-1.0_4_5_8_9_8_4, -1.1_2_7_9_2_9_7, -0.9_6_6_3_0_8_6, -0.9_1_5_0_3_9_0_6, -0.7_5_0_9_7_6_5_6]					)
            assert np.abs(video_frames.cpu().numpy()[0, 0, 0, 0, -5:] - expected_array					).sum() < 1E-2


