


import argparse
import json
import os

import torch
from torch import nn

from transformers import NllbMoeConfig, NllbMoeModel
from transformers.modeling_utils import dtype_byte_size
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME





def      UpperCAmelCase      (		lowerCamelCase_  :List[Any] ):







		'''simple docstring'''
		snake_case_		:							Tuple			       =						[
		    """encoder.version""",
		    """decoder.version""",
		    """model.encoder.version""",
		    """model.decoder.version""",
		    """decoder.output_projection.weight""",
		    """_float_tensor""",
		    """encoder.embed_positions._float_tensor""",
		    """decoder.embed_positions._float_tensor""",
		]
		for k in ignore_keys:
				state_dict.pop(lowerCamelCase_				,      lowerCamelCase_ )





def      UpperCAmelCase      (		lowerCamelCase_  :List[Any] ):







		'''simple docstring'''
		snake_case_		:							Dict			       =						emb.weight.shape
		snake_case_		:							str			       =						nn.Linear(lowerCamelCase_				,      lowerCamelCase_				,      bias=lowerCamelCase_ )
		snake_case_		:							str			       =						emb.weight.data
		return lin_layer





def      UpperCAmelCase      (		lowerCamelCase_  :List[Any]				,      lowerCamelCase_  :List[str]=None ):







		'''simple docstring'''
		snake_case_		:							Union[str, Any]			       =						{}
		for old_key in state_dict.keys():
				snake_case_		:							Optional[int]			       =						old_key
				if "moe_layer.experts." in key:
						if expert_idx is not None:
								snake_case_		:							Dict			       =						key.replace("""moe_layer.experts.0"""				,      F'''ffn.experts.expert_{expert_idx}''' )
						else:
								snake_case_		:							str			       =						key.replace("""moe_layer.experts."""				,      """ffn.experts.expert_""" )
				if "gate" in key:
						snake_case_		:							List[str]			       =						key.replace(""".moe_layer.gate.wg"""				,      """.ffn.router.classifier""" )
				if "fc2" and "experts" not in key:
						snake_case_		:							Any			       =						key.replace(""".fc2."""				,      """.ffn.fc2.""" )
				if "fc1" and "experts" not in key:
						snake_case_		:							str			       =						key.replace(""".fc1."""				,      """.ffn.fc1.""" )
				if ".encoder_attn." in key:
						snake_case_		:							str			       =						key.replace(""".encoder_attn."""				,      """.cross_attention.""" )
				if "encoder_attn_layer_norm" in key:
						snake_case_		:							List[str]			       =						key.replace("""encoder_attn_layer_norm"""				,      """cross_attention_layer_norm""" )
				if "final_layer_norm" in key:
						snake_case_		:							Dict			       =						key.replace("""final_layer_norm"""				,      """ff_layer_norm""" )
				snake_case_		:							Dict			       =						state_dict[old_key]
		return new_dict





def      UpperCAmelCase      (		lowerCamelCase_  :Tuple				,      lowerCamelCase_  :Optional[Any]				,      lowerCamelCase_  :Union[str, Any]				,      lowerCamelCase_  :Any				,      lowerCamelCase_  :str = WEIGHTS_NAME ):







		'''simple docstring'''
		snake_case_		:							Tuple			       =						[]
		snake_case_		:							Dict			       =						0
		os.makedirs(lowerCamelCase_				,      exist_ok=lowerCamelCase_ )

		for expert in range(lowerCamelCase_ ):
				snake_case_		:							Optional[Any]			       =						switch_checkpoint_path + F'''-rank-{expert}.pt'''
				if os.path.isfile(lowerCamelCase_ ):
						snake_case_		:							List[Any]			       =						torch.load(lowerCamelCase_ )["""model"""]
						remove_ignore_keys_(lowerCamelCase_ )
						snake_case_		:							List[str]			       =						rename_fairseq_keys(lowerCamelCase_				,      lowerCamelCase_ )
						snake_case_		:							List[str]			       =						os.path.join(
						    lowerCamelCase_				,      weights_name.replace(""".bin"""				,      F'''-{len(lowerCamelCase_ )+1:05d}-of-???.bin''' ) )
						torch.save(lowerCamelCase_				,      lowerCamelCase_ )
						sharded_state_dicts.append(expert_state.keys() )
						total_size += sum([value.numel() for key, value in expert_state.items()] ) * dtype_byte_size(
						    expert_state[list(lowerCamelCase_ )[0]].dtype )

    # Add the last block
		snake_case_		:							Tuple			       =						os.path.join(lowerCamelCase_				,      weights_name.replace(""".bin"""				,      F'''-{len(lowerCamelCase_ )+1:05d}-of-???.bin''' ) )
		snake_case_		:							Tuple			       =						torch.load(switch_checkpoint_path + """-shared.pt""" )["""model"""]
		remove_ignore_keys_(lowerCamelCase_ )
		snake_case_		:							Tuple			       =						rename_fairseq_keys(lowerCamelCase_				,      lowerCamelCase_ )
		snake_case_		:							List[str]			       =						shared_weights["""decoder.embed_tokens.weight"""]
		sharded_state_dicts.append(shared_weights.keys() )

		# If we only have the shared weights (dummy model/experts saved on the same file)
		if len(lowerCamelCase_ ) == 1:
				snake_case_		:							Union[str, Any]			       =						os.path.join(lowerCamelCase_				,      lowerCamelCase_ )
				torch.save(lowerCamelCase_				,      lowerCamelCase_ )
				return {weights_name: sharded_state_dicts[0]}, None
		else:
				torch.save(lowerCamelCase_				,      lowerCamelCase_ )
		# Otherwise, let's build the index
		snake_case_		:							str			       =						{}
		for idx, shard in enumerate(lowerCamelCase_ ):
				snake_case_		:							List[str]			       =						weights_name.replace(""".bin"""				,      F'''-{idx+1:05d}-of-{len(lowerCamelCase_ ):05d}.bin''' )
				snake_case_		:							Optional[int]			       =						os.path.join(lowerCamelCase_				,      weights_name.replace(""".bin"""				,      F'''-{idx+1:05d}-of-???.bin''' ) )
				os.rename(lowerCamelCase_				,      os.path.join(lowerCamelCase_				,      lowerCamelCase_ ) )
				for key in shard:
						snake_case_		:							Optional[int]			       =						shard_file

    # Add the metadata
		snake_case_		:							Any			       =						{"""total_size""": total_size}
		snake_case_		:							int			       =						{"""metadata""": metadata, """weight_map""": weight_map}

		with open(os.path.join(lowerCamelCase_				,      lowerCamelCase_ )				,      """w"""				,      encoding="""utf-8""" ) as f:
				snake_case_		:							List[str]			       =						json.dumps(lowerCamelCase_				,      indent=2				,      sort_keys=lowerCamelCase_ ) + """\n"""
				f.write(lowerCamelCase_ )

		return metadata, index


if __name__ == "__main__":
							__A	:	int     =     argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '--nllb_moe_checkpoint_path',
							    default='/home/arthur_huggingface_co/fairseq/weights/checkpoints/model_moe_54b/checkpoint_2_300000',
							    type=str,
							    required=False,
							    help='Path to a directory containing a folder per layer. Follows the original Google format.',
							)
							parser.add_argument('--dtype', default='float32', type=str, required=False, help='dtype of the saved model')
							parser.add_argument(
							    '--pytorch_dump_folder_path',
							    default='/home/arthur_huggingface_co/fairseq/weights/checkpoints/hf-converted-moe-54b',
							    type=str,
							    required=False,
							    help='Path to the output pytorch model.',
							)
							__A	:	List[str]     =     parser.parse_args()
							__A	:	List[Any]     =     shard_on_the_fly(
							    args.nllb_moe_checkpoint_path,
							    args.pytorch_dump_folder_path,
							    128,
							    args.dtype,
							)

							__A	:	Tuple     =     NllbMoeConfig.from_pretrained(
							    'facebook/nllb-200-3.3B', encoder_sparse_step=4, decoder_sparse_step=4, num_experts=128
							)
							config.save_pretrained(args.pytorch_dump_folder_path)
							__A	:	List[str]     =     NllbMoeModel.from_pretrained(args.pytorch_dump_folder_path)
							print('Done')
							model.save_pretrained(args.pytorch_dump_folder_path)



'''simple docstring'''






import unittest

from transformers import EsmConfig, is_torch_available
from transformers.testing_utils import TestCasePlus, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import EsmForMaskedLM, EsmForSequenceClassification, EsmForTokenClassification, EsmModel
							from transformers.models.esm.modeling_esm import (
							    ESM_PRETRAINED_MODEL_ARCHIVE_LIST,
							    EsmEmbeddings,
							    create_position_ids_from_input_ids,
							)
class 						__UpperCamelCase	:





							def __init__(  self		:Union[str, Any]  ,_UpperCamelCase		:int  ,_UpperCamelCase		:str=1_3  ,_UpperCamelCase		:Tuple=7  ,_UpperCamelCase		:Tuple=False  ,_UpperCamelCase		:Tuple=True  ,_UpperCamelCase		:Union[str, Any]=False  ,_UpperCamelCase		:int=True  ,_UpperCamelCase		:List[str]=3_3  ,_UpperCamelCase		:Any=3_2  ,_UpperCamelCase		:Any=5  ,_UpperCamelCase		:List[str]=4  ,_UpperCamelCase		:Tuple=3_7  ,_UpperCamelCase		:Optional[Any]="gelu"  ,_UpperCamelCase		:Any=0.1  ,_UpperCamelCase		:List[Any]=0.1  ,_UpperCamelCase		:Any=5_1_2  ,_UpperCamelCase		:Tuple=1_6  ,_UpperCamelCase		:Any=2  ,_UpperCamelCase		:Optional[Any]=0.02  ,_UpperCamelCase		:List[str]=3  ,_UpperCamelCase		:Union[str, Any]=4  ,_UpperCamelCase		:Dict=None  ,):
									snake_case_		:							Tuple			       =						parent
									snake_case_		:							List[str]			       =						batch_size
									snake_case_		:							List[str]			       =						seq_length
									snake_case_		:							Any			       =						is_training
									snake_case_		:							List[Any]			       =						use_input_mask
									snake_case_		:							int			       =						use_token_type_ids
									snake_case_		:							Optional[int]			       =						use_labels
									snake_case_		:							List[str]			       =						vocab_size
									snake_case_		:							Dict			       =						hidden_size
									snake_case_		:							Optional[int]			       =						num_hidden_layers
									snake_case_		:							Optional[Any]			       =						num_attention_heads
									snake_case_		:							int			       =						intermediate_size
									snake_case_		:							Optional[Any]			       =						hidden_act
									snake_case_		:							int			       =						hidden_dropout_prob
									snake_case_		:							Optional[int]			       =						attention_probs_dropout_prob
									snake_case_		:							Optional[int]			       =						max_position_embeddings
									snake_case_		:							List[str]			       =						type_vocab_size
									snake_case_		:							int			       =						type_sequence_label_size
									snake_case_		:							Dict			       =						initializer_range
									snake_case_		:							Any			       =						num_labels
									snake_case_		:							Any			       =						num_choices
									snake_case_		:							Tuple			       =						scope





							def 		a__   (  self		:str			):
									snake_case_		:							Dict			       =						ids_tensor([self.batch_size, self.seq_length]  ,self.vocab_size			)

									snake_case_		:							str			       =						None
									if self.use_input_mask:
											snake_case_		:							Optional[int]			       =						random_attention_mask([self.batch_size, self.seq_length]			)

									snake_case_		:							Dict			       =						None
									snake_case_		:							List[str]			       =						None
									snake_case_		:							Tuple			       =						None
									if self.use_labels:
											snake_case_		:							Dict			       =						ids_tensor([self.batch_size]  ,self.type_sequence_label_size			)
											snake_case_		:							Any			       =						ids_tensor([self.batch_size, self.seq_length]  ,self.num_labels			)
											snake_case_		:							Union[str, Any]			       =						ids_tensor([self.batch_size]  ,self.num_choices			)

									snake_case_		:							Dict			       =						self.get_config()

									return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels





							def 		a__   (  self		:Optional[Any]			):
									return EsmConfig(
									    vocab_size=self.vocab_size  ,hidden_size=self.hidden_size  ,pad_token_id=1  ,num_hidden_layers=self.num_hidden_layers  ,num_attention_heads=self.num_attention_heads  ,intermediate_size=self.intermediate_size  ,hidden_act=self.hidden_act  ,hidden_dropout_prob=self.hidden_dropout_prob  ,attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,max_position_embeddings=self.max_position_embeddings  ,type_vocab_size=self.type_vocab_size  ,initializer_range=self.initializer_range  ,)





							def 		a__   (  self		:str  ,_UpperCamelCase		:List[Any]  ,_UpperCamelCase		:List[str]  ,_UpperCamelCase		:Any  ,_UpperCamelCase		:Optional[int]  ,_UpperCamelCase		:List[Any]  ,_UpperCamelCase		:Optional[int]			):
									snake_case_		:							Union[str, Any]			       =						EsmModel(config=_UpperCamelCase			)
									model.to(_UpperCamelCase			)
									model.eval()
									snake_case_		:							int			       =						model(_UpperCamelCase  ,attention_mask=_UpperCamelCase			)
									snake_case_		:							str			       =						model(_UpperCamelCase			)
									snake_case_		:							Optional[int]			       =						model(_UpperCamelCase			)

									self.parent.assertEqual(result.last_hidden_state.shape  ,(self.batch_size, self.seq_length, self.hidden_size)			)
									self.parent.assertEqual(result.pooler_output.shape  ,(self.batch_size, self.hidden_size)			)





							def 		a__   (  self		:Any  ,_UpperCamelCase		:Optional[Any]  ,_UpperCamelCase		:Optional[Any]  ,_UpperCamelCase		:Any  ,_UpperCamelCase		:Dict  ,_UpperCamelCase		:Union[str, Any]  ,_UpperCamelCase		:Optional[int]			):
									snake_case_		:							str			       =						EsmForMaskedLM(config=_UpperCamelCase			)
									model.to(_UpperCamelCase			)
									model.eval()
									snake_case_		:							Dict			       =						model(_UpperCamelCase  ,attention_mask=_UpperCamelCase  ,labels=_UpperCamelCase			)
									self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.seq_length, self.vocab_size)			)





							def 		a__   (  self		:Optional[int]  ,_UpperCamelCase		:Optional[Any]  ,_UpperCamelCase		:List[str]  ,_UpperCamelCase		:Union[str, Any]  ,_UpperCamelCase		:Union[str, Any]  ,_UpperCamelCase		:Union[str, Any]  ,_UpperCamelCase		:Any			):
									snake_case_		:							List[Any]			       =						self.num_labels
									snake_case_		:							int			       =						EsmForTokenClassification(config=_UpperCamelCase			)
									model.to(_UpperCamelCase			)
									model.eval()
									snake_case_		:							int			       =						model(_UpperCamelCase  ,attention_mask=_UpperCamelCase  ,labels=_UpperCamelCase			)
									self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.seq_length, self.num_labels)			)





							def 		a__   (  self		:Union[str, Any]			):
									snake_case_		:							Dict			       =						self.prepare_config_and_inputs()
									(
									    (
									    snake_case_
									)       ,      (
									    snake_case_
									)       ,      (
									    snake_case_
									)       ,      (
									    snake_case_
									)       ,      (
									    snake_case_
									)       ,      (
									    snake_case_
									)       ,      
									)		:							Optional[int]			       =						config_and_inputs
									snake_case_		:							List[str]			       =						{"""input_ids""": input_ids, """attention_mask""": input_mask}
									return config, inputs_dict





@require_torch
class 						__UpperCamelCase	(						lowercase__						,  lowercase__						,  unittest.TestCase							):
							lowercase			:		Optional[int]   		=  False

							lowercase			:		List[str]   		=  (
							    (
							        EsmForMaskedLM,
							        EsmModel,
							        EsmForSequenceClassification,
							        EsmForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							lowercase			:		int   		=  ()
							lowercase			:		List[str]   		=  (
							    {
							        'feature-extraction': EsmModel,
							        'fill-mask': EsmForMaskedLM,
							        'text-classification': EsmForSequenceClassification,
							        'token-classification': EsmForTokenClassification,
							        'zero-shot': EsmForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)
							lowercase			:		str   		=  True





							def 		a__   (  self		:Any			):
									snake_case_		:							Any			       =						EsmModelTester(self			)
									snake_case_		:							str			       =						ConfigTester(self  ,config_class=_UpperCamelCase  ,hidden_size=3_7			)





							def 		a__   (  self		:Optional[Any]			):
									self.config_tester.run_common_tests()





							def 		a__   (  self		:Optional[Any]			):
									snake_case_		:							str			       =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*_UpperCamelCase			)





							def 		a__   (  self		:Dict			):
									snake_case_		:							Dict			       =						self.model_tester.prepare_config_and_inputs()
									for type in ["absolute", "relative_key", "relative_key_query"]:
											snake_case_		:							int			       =						type
											self.model_tester.create_and_check_model(*_UpperCamelCase			)





							def 		a__   (  self		:Dict			):
									snake_case_		:							str			       =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_masked_lm(*_UpperCamelCase			)





							def 		a__   (  self		:List[str]			):
									snake_case_		:							List[str]			       =						self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_token_classification(*_UpperCamelCase			)





							@slow
							def 		a__   (  self		:Union[str, Any]			):
									for model_name in ESM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											snake_case_		:							int			       =						EsmModel.from_pretrained(_UpperCamelCase			)
											self.assertIsNotNone(_UpperCamelCase			)





							def 		a__   (  self		:Tuple			):
									snake_case_		:							Dict			       =						self.model_tester.prepare_config_and_inputs()[0]
									snake_case_		:							Optional[int]			       =						EsmEmbeddings(config=_UpperCamelCase			)

									snake_case_		:							List[Any]			       =						torch.as_tensor([[1_2, 3_1, 1_3, model.padding_idx]]			)
									snake_case_		:							Union[str, Any]			       =						torch.as_tensor(
									    [
									        [
									            0 + model.padding_idx + 1,
									            1 + model.padding_idx + 1,
									            2 + model.padding_idx + 1,
									            model.padding_idx,
									        ]
									    ]			)
									snake_case_		:							Optional[Any]			       =						create_position_ids_from_input_ids(_UpperCamelCase  ,model.padding_idx			)
									self.assertEqual(position_ids.shape  ,expected_positions.shape			)
									self.assertTrue(torch.all(torch.eq(_UpperCamelCase  ,_UpperCamelCase			)			)			)





							def 		a__   (  self		:List[Any]			):
									snake_case_		:							Union[str, Any]			       =						self.model_tester.prepare_config_and_inputs()[0]
									snake_case_		:							List[Any]			       =						EsmEmbeddings(config=_UpperCamelCase			)

									snake_case_		:							Dict			       =						torch.empty(2  ,4  ,3_0			)
									snake_case_		:							List[Any]			       =						[
									    0 + embeddings.padding_idx + 1,
									    1 + embeddings.padding_idx + 1,
									    2 + embeddings.padding_idx + 1,
									    3 + embeddings.padding_idx + 1,
									]
									snake_case_		:							Any			       =						torch.as_tensor([expected_single_positions, expected_single_positions]			)
									snake_case_		:							str			       =						embeddings.create_position_ids_from_inputs_embeds(_UpperCamelCase			)
									self.assertEqual(position_ids.shape  ,expected_positions.shape			)
									self.assertTrue(torch.all(torch.eq(_UpperCamelCase  ,_UpperCamelCase			)			)			)





							@unittest.skip("""Esm does not support embedding resizing"""			)
							def 		a__   (  self		:Optional[Any]			):
									pass





							@unittest.skip("""Esm does not support embedding resizing"""			)
							def 		a__   (  self		:Optional[int]			):
									pass





							@unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests."""			)
							def 		a__   (  self		:Optional[int]			):
									pass





@require_torch
class 						__UpperCamelCase	(						lowercase__							):





							@slow
							def 		a__   (  self		:Any			):
									with torch.no_grad():
											snake_case_		:							Optional[Any]			       =						EsmForMaskedLM.from_pretrained("""facebook/esm2_t6_8M_UR50D"""			)
											model.eval()
											snake_case_		:							List[Any]			       =						torch.tensor([[0, 1, 2, 3, 4, 5]]			)
											snake_case_		:							Dict			       =						model(_UpperCamelCase			)[0]

											snake_case_		:							Optional[int]			       =						3_3

											snake_case_		:							List[Any]			       =						torch.Size((1, 6, vocab_size)			)
											self.assertEqual(output.shape  ,_UpperCamelCase			)

											snake_case_		:							Union[str, Any]			       =						torch.tensor(
											    [[[8.92_15, -10.58_98, -6.46_71], [-6.39_67, -13.91_14, -1.12_12], [-7.78_12, -13.95_16, -3.74_06]]]			)
											self.assertTrue(torch.allclose(output[:, :3, :3]  ,_UpperCamelCase  ,atol=1E-4			)			)





							@slow
							def 		a__   (  self		:List[Any]			):
									with torch.no_grad():
											snake_case_		:							List[Any]			       =						EsmModel.from_pretrained("""facebook/esm2_t6_8M_UR50D"""			)
											model.eval()

											snake_case_		:							str			       =						torch.tensor([[0, 6, 4, 1_3, 5, 4, 1_6, 1_2, 1_1, 7, 2]]			)
											snake_case_		:							Optional[Any]			       =						model(_UpperCamelCase			)[0]
											# compare the actual values for a slice.
											snake_case_		:							List[str]			       =						torch.tensor(
											    [[[0.14_44, 0.54_13, 0.32_48], [0.30_34, 0.00_53, 0.31_08], [0.32_28, -0.24_99, 0.34_15]]]			)
											self.assertTrue(torch.allclose(output[:, :3, :3]  ,_UpperCamelCase  ,atol=1E-4			)			)