


import argparse
import json
import os

import torch
from torch import nn

from transformers import NllbMoeConfig, NllbMoeModel
from transformers.modeling_utils import dtype_byte_size
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME
def      _a					( SCREAMING_SNAKE_CASE	):


		"""simple docstring"""




		lowercase__				= [
		    '''encoder.version''',
		    '''decoder.version''',
		    '''model.encoder.version''',
		    '''model.decoder.version''',
		    '''decoder.output_projection.weight''',
		    '''_float_tensor''',
		    '''encoder.embed_positions._float_tensor''',
		    '''decoder.embed_positions._float_tensor''',
		]
		for k in ignore_keys:
				state_dict.pop(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)
def      _a					( SCREAMING_SNAKE_CASE	):


		"""simple docstring"""




		lowercase__	,						lowercase__				= emb.weight.shape
		lowercase__				= nn.Linear(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE		,							bias=SCREAMING_SNAKE_CASE	)
		lowercase__				= emb.weight.data
		return lin_layer
def      _a					( SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE=None	):


		"""simple docstring"""




		lowercase__				= {}
		for old_key in state_dict.keys():
				lowercase__				= old_key
				if "moe_layer.experts." in key:
						if expert_idx is not None:
								lowercase__				= key.replace('''moe_layer.experts.0'''		,							f'ffn.experts.expert_{expert_idx}'	)
						else:
								lowercase__				= key.replace('''moe_layer.experts.'''		,							'''ffn.experts.expert_'''	)
				if "gate" in key:
						lowercase__				= key.replace('''.moe_layer.gate.wg'''		,							'''.ffn.router.classifier'''	)
				if "fc2" and "experts" not in key:
						lowercase__				= key.replace('''.fc2.'''		,							'''.ffn.fc2.'''	)
				if "fc1" and "experts" not in key:
						lowercase__				= key.replace('''.fc1.'''		,							'''.ffn.fc1.'''	)
				if ".encoder_attn." in key:
						lowercase__				= key.replace('''.encoder_attn.'''		,							'''.cross_attention.'''	)
				if "encoder_attn_layer_norm" in key:
						lowercase__				= key.replace('''encoder_attn_layer_norm'''		,							'''cross_attention_layer_norm'''	)
				if "final_layer_norm" in key:
						lowercase__				= key.replace('''final_layer_norm'''		,							'''ff_layer_norm'''	)
				lowercase__				= state_dict[old_key]
		return new_dict
def      _a					( SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE = WEIGHTS_NAME	):


		"""simple docstring"""




		lowercase__				= []
		lowercase__				= 0
		os.makedirs(SCREAMING_SNAKE_CASE		,							exist_ok=SCREAMING_SNAKE_CASE	)

		for expert in range(SCREAMING_SNAKE_CASE	):
				lowercase__				= switch_checkpoint_path + f'-rank-{expert}.pt'
				if os.path.isfile(SCREAMING_SNAKE_CASE	):
						lowercase__				= torch.load(SCREAMING_SNAKE_CASE	)['''model''']
						remove_ignore_keys_(SCREAMING_SNAKE_CASE	)
						lowercase__				= rename_fairseq_keys(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)
						lowercase__				= os.path.join(
						    SCREAMING_SNAKE_CASE		,							weights_name.replace('''.bin'''		,							f'-{len(SCREAMING_SNAKE_CASE	)+1:05d}-of-???.bin'	)	)
						torch.save(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)
						sharded_state_dicts.append(expert_state.keys()	)
						total_size += sum([value.numel() for key, value in expert_state.items()]	) * dtype_byte_size(
						    expert_state[list(SCREAMING_SNAKE_CASE	)[0]].dtype	)

    # Add the last block
		lowercase__				= os.path.join(SCREAMING_SNAKE_CASE		,							weights_name.replace('''.bin'''		,							f'-{len(SCREAMING_SNAKE_CASE	)+1:05d}-of-???.bin'	)	)
		lowercase__				= torch.load(switch_checkpoint_path + '''-shared.pt'''	)['''model''']
		remove_ignore_keys_(SCREAMING_SNAKE_CASE	)
		lowercase__				= rename_fairseq_keys(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)
		lowercase__				= shared_weights['''decoder.embed_tokens.weight''']
		sharded_state_dicts.append(shared_weights.keys()	)

		# If we only have the shared weights (dummy model/experts saved on the same file)
		if len(SCREAMING_SNAKE_CASE	) == 1:
				lowercase__				= os.path.join(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)
				torch.save(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)
				return {weights_name: sharded_state_dicts[0]}, None
		else:
				torch.save(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)
		# Otherwise, let's build the index
		lowercase__				= {}
		for idx, shard in enumerate(SCREAMING_SNAKE_CASE	):
				lowercase__				= weights_name.replace('''.bin'''		,							f'-{idx+1:05d}-of-{len(SCREAMING_SNAKE_CASE	):05d}.bin'	)
				lowercase__				= os.path.join(SCREAMING_SNAKE_CASE		,							weights_name.replace('''.bin'''		,							f'-{idx+1:05d}-of-???.bin'	)	)
				os.rename(SCREAMING_SNAKE_CASE		,							os.path.join(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)	)
				for key in shard:
						lowercase__				= shard_file

    # Add the metadata
		lowercase__				= {'''total_size''': total_size}
		lowercase__				= {'''metadata''': metadata, '''weight_map''': weight_map}

		with open(os.path.join(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)		,							'''w'''		,							encoding='''utf-8'''	) as f:
				lowercase__				= json.dumps(SCREAMING_SNAKE_CASE		,							indent=2		,							sort_keys=SCREAMING_SNAKE_CASE	) + '''\n'''
				f.write(SCREAMING_SNAKE_CASE	)

		return metadata, index


if __name__ == "__main__":
			lowerCAmelCase			=				argparse.ArgumentParser()
			# Required parameters
			parser.add_argument(
			    '--nllb_moe_checkpoint_path',
			    default='/home/arthur_huggingface_co/fairseq/weights/checkpoints/model_moe_54b/checkpoint_2_300000',
			    type=str,
			    required=False,
			    help='Path to a directory containing a folder per layer. Follows the original Google format.',
			)
			parser.add_argument('--dtype', default='float32', type=str, required=False, help='dtype of the saved model')
			parser.add_argument(
			    '--pytorch_dump_folder_path',
			    default='/home/arthur_huggingface_co/fairseq/weights/checkpoints/hf-converted-moe-54b',
			    type=str,
			    required=False,
			    help='Path to the output pytorch model.',
			)
			lowerCAmelCase			=				parser.parse_args()
			lowerCAmelCase,		lowerCAmelCase			=				shard_on_the_fly(
			    args.nllb_moe_checkpoint_path,
			    args.pytorch_dump_folder_path,
			    128,
			    args.dtype,
			)

			lowerCAmelCase			=				NllbMoeConfig.from_pretrained(
			    'facebook/nllb-200-3.3B', encoder_sparse_step=4, decoder_sparse_step=4, num_experts=128
			)
			config.save_pretrained(args.pytorch_dump_folder_path)
			lowerCAmelCase			=				NllbMoeModel.from_pretrained(args.pytorch_dump_folder_path)
			print('Done')
			model.save_pretrained(args.pytorch_dump_folder_path)



'''simple docstring'''






import unittest

from diffusers.pipelines.pipeline_utils import is_safetensors_compatible







class lowercase_       (		unittest.TestCase			):







   """simple docstring"""

   def 	__UpperCAmelCase   (      self      :			Optional[Any]     )  ->			Dict:
        _A    		=  [
            'safety_checker/pytorch_model.bin',
            'safety_checker/model.safetensors',
            'vae/diffusion_pytorch_model.bin',
            'vae/diffusion_pytorch_model.safetensors',
            'text_encoder/pytorch_model.bin',
            'text_encoder/model.safetensors',
            'unet/diffusion_pytorch_model.bin',
            'unet/diffusion_pytorch_model.safetensors',
        ]
        self.assertTrue(is_safetensors_compatible(UpperCamelCase__     )     )
   def 	__UpperCAmelCase   (      self      :			Union[str, Any]     )  ->			Optional[Any]:
        _A    		=  [
            'unet/diffusion_pytorch_model.bin',
            'unet/diffusion_pytorch_model.safetensors',
        ]
        self.assertTrue(is_safetensors_compatible(UpperCamelCase__     )     )
   def 	__UpperCAmelCase   (      self      :			str     )  ->			Union[str, Any]:
        _A    		=  [
            'safety_checker/pytorch_model.bin',
            'safety_checker/model.safetensors',
            'vae/diffusion_pytorch_model.bin',
            'vae/diffusion_pytorch_model.safetensors',
            'text_encoder/pytorch_model.bin',
            'text_encoder/model.safetensors',
            'unet/diffusion_pytorch_model.bin',
            # Removed: 'unet/diffusion_pytorch_model.safetensors',
        ]
        self.assertFalse(is_safetensors_compatible(UpperCamelCase__     )     )
   def 	__UpperCAmelCase   (      self      :			int     )  ->			Dict:
        _A    		=  [
            'text_encoder/pytorch_model.bin',
            'text_encoder/model.safetensors',
        ]
        self.assertTrue(is_safetensors_compatible(UpperCamelCase__     )     )
   def 	__UpperCAmelCase   (      self      :			Union[str, Any]     )  ->			str:
        _A    		=  [
            'safety_checker/pytorch_model.bin',
            'safety_checker/model.safetensors',
            'vae/diffusion_pytorch_model.bin',
            'vae/diffusion_pytorch_model.safetensors',
            'text_encoder/pytorch_model.bin',
            # Removed: 'text_encoder/model.safetensors',
            'unet/diffusion_pytorch_model.bin',
            'unet/diffusion_pytorch_model.safetensors',
        ]
        self.assertFalse(is_safetensors_compatible(UpperCamelCase__     )     )
   def 	__UpperCAmelCase   (      self      :			str     )  ->			Dict:
        _A    		=  [
            'safety_checker/pytorch_model.fp16.bin',
            'safety_checker/model.fp16.safetensors',
            'vae/diffusion_pytorch_model.fp16.bin',
            'vae/diffusion_pytorch_model.fp16.safetensors',
            'text_encoder/pytorch_model.fp16.bin',
            'text_encoder/model.fp16.safetensors',
            'unet/diffusion_pytorch_model.fp16.bin',
            'unet/diffusion_pytorch_model.fp16.safetensors',
        ]
        _A    		=  'fp16'
        self.assertTrue(is_safetensors_compatible(UpperCamelCase__, variant=UpperCamelCase__     )     )
   def 	__UpperCAmelCase   (      self      :			Optional[int]     )  ->			List[str]:
        _A    		=  [
            'unet/diffusion_pytorch_model.fp16.bin',
            'unet/diffusion_pytorch_model.fp16.safetensors',
        ]
        _A    		=  'fp16'
        self.assertTrue(is_safetensors_compatible(UpperCamelCase__, variant=UpperCamelCase__     )     )
   def 	__UpperCAmelCase   (      self      :			List[str]     )  ->			List[str]:
        # pass variant but use the non-variant filenames
        _A    		=  [
            'unet/diffusion_pytorch_model.bin',
            'unet/diffusion_pytorch_model.safetensors',
        ]
        _A    		=  'fp16'
        self.assertTrue(is_safetensors_compatible(UpperCamelCase__, variant=UpperCamelCase__     )     )
   def 	__UpperCAmelCase   (      self      :			List[Any]     )  ->			Optional[Any]:
        _A    		=  [
            'safety_checker/pytorch_model.fp16.bin',
            'safety_checker/model.fp16.safetensors',
            'vae/diffusion_pytorch_model.fp16.bin',
            'vae/diffusion_pytorch_model.fp16.safetensors',
            'text_encoder/pytorch_model.fp16.bin',
            'text_encoder/model.fp16.safetensors',
            'unet/diffusion_pytorch_model.fp16.bin',
            # Removed: 'unet/diffusion_pytorch_model.fp16.safetensors',
        ]
        _A    		=  'fp16'
        self.assertFalse(is_safetensors_compatible(UpperCamelCase__, variant=UpperCamelCase__     )     )
   def 	__UpperCAmelCase   (      self      :			Dict     )  ->			Union[str, Any]:
        _A    		=  [
            'text_encoder/pytorch_model.fp16.bin',
            'text_encoder/model.fp16.safetensors',
        ]
        _A    		=  'fp16'
        self.assertTrue(is_safetensors_compatible(UpperCamelCase__, variant=UpperCamelCase__     )     )
   def 	__UpperCAmelCase   (      self      :			Tuple     )  ->			str:
        # pass variant but use the non-variant filenames
        _A    		=  [
            'text_encoder/pytorch_model.bin',
            'text_encoder/model.safetensors',
        ]
        _A    		=  'fp16'
        self.assertTrue(is_safetensors_compatible(UpperCamelCase__, variant=UpperCamelCase__     )     )

   def 	__UpperCAmelCase   (      self      :			List[Any]     )  ->			int:
        _A    		=  [
            'safety_checker/pytorch_model.fp16.bin',
            'safety_checker/model.fp16.safetensors',
            'vae/diffusion_pytorch_model.fp16.bin',
            'vae/diffusion_pytorch_model.fp16.safetensors',
            'text_encoder/pytorch_model.fp16.bin',
            # 'text_encoder/model.fp16.safetensors',
            'unet/diffusion_pytorch_model.fp16.bin',
            'unet/diffusion_pytorch_model.fp16.safetensors',
        ]
        _A    		=  'fp16'
        self.assertFalse(is_safetensors_compatible(UpperCamelCase__, variant=UpperCamelCase__     )     )



