


import argparse
import json
import os

import torch
from torch import nn

from transformers import NllbMoeConfig, NllbMoeModel
from transformers.modeling_utils import dtype_byte_size
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME
def      _a					( SCREAMING_SNAKE_CASE	):


		"""simple docstring"""




		lowercase__				= [
		    '''encoder.version''',
		    '''decoder.version''',
		    '''model.encoder.version''',
		    '''model.decoder.version''',
		    '''decoder.output_projection.weight''',
		    '''_float_tensor''',
		    '''encoder.embed_positions._float_tensor''',
		    '''decoder.embed_positions._float_tensor''',
		]
		for k in ignore_keys:
				state_dict.pop(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)
def      _a					( SCREAMING_SNAKE_CASE	):


		"""simple docstring"""




		lowercase__	,						lowercase__				= emb.weight.shape
		lowercase__				= nn.Linear(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE		,							bias=SCREAMING_SNAKE_CASE	)
		lowercase__				= emb.weight.data
		return lin_layer
def      _a					( SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE=None	):


		"""simple docstring"""




		lowercase__				= {}
		for old_key in state_dict.keys():
				lowercase__				= old_key
				if "moe_layer.experts." in key:
						if expert_idx is not None:
								lowercase__				= key.replace('''moe_layer.experts.0'''		,							f'ffn.experts.expert_{expert_idx}'	)
						else:
								lowercase__				= key.replace('''moe_layer.experts.'''		,							'''ffn.experts.expert_'''	)
				if "gate" in key:
						lowercase__				= key.replace('''.moe_layer.gate.wg'''		,							'''.ffn.router.classifier'''	)
				if "fc2" and "experts" not in key:
						lowercase__				= key.replace('''.fc2.'''		,							'''.ffn.fc2.'''	)
				if "fc1" and "experts" not in key:
						lowercase__				= key.replace('''.fc1.'''		,							'''.ffn.fc1.'''	)
				if ".encoder_attn." in key:
						lowercase__				= key.replace('''.encoder_attn.'''		,							'''.cross_attention.'''	)
				if "encoder_attn_layer_norm" in key:
						lowercase__				= key.replace('''encoder_attn_layer_norm'''		,							'''cross_attention_layer_norm'''	)
				if "final_layer_norm" in key:
						lowercase__				= key.replace('''final_layer_norm'''		,							'''ff_layer_norm'''	)
				lowercase__				= state_dict[old_key]
		return new_dict
def      _a					( SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE = WEIGHTS_NAME	):


		"""simple docstring"""




		lowercase__				= []
		lowercase__				= 0
		os.makedirs(SCREAMING_SNAKE_CASE		,							exist_ok=SCREAMING_SNAKE_CASE	)

		for expert in range(SCREAMING_SNAKE_CASE	):
				lowercase__				= switch_checkpoint_path + f'-rank-{expert}.pt'
				if os.path.isfile(SCREAMING_SNAKE_CASE	):
						lowercase__				= torch.load(SCREAMING_SNAKE_CASE	)['''model''']
						remove_ignore_keys_(SCREAMING_SNAKE_CASE	)
						lowercase__				= rename_fairseq_keys(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)
						lowercase__				= os.path.join(
						    SCREAMING_SNAKE_CASE		,							weights_name.replace('''.bin'''		,							f'-{len(SCREAMING_SNAKE_CASE	)+1:05d}-of-???.bin'	)	)
						torch.save(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)
						sharded_state_dicts.append(expert_state.keys()	)
						total_size += sum([value.numel() for key, value in expert_state.items()]	) * dtype_byte_size(
						    expert_state[list(SCREAMING_SNAKE_CASE	)[0]].dtype	)

    # Add the last block
		lowercase__				= os.path.join(SCREAMING_SNAKE_CASE		,							weights_name.replace('''.bin'''		,							f'-{len(SCREAMING_SNAKE_CASE	)+1:05d}-of-???.bin'	)	)
		lowercase__				= torch.load(switch_checkpoint_path + '''-shared.pt'''	)['''model''']
		remove_ignore_keys_(SCREAMING_SNAKE_CASE	)
		lowercase__				= rename_fairseq_keys(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)
		lowercase__				= shared_weights['''decoder.embed_tokens.weight''']
		sharded_state_dicts.append(shared_weights.keys()	)

		# If we only have the shared weights (dummy model/experts saved on the same file)
		if len(SCREAMING_SNAKE_CASE	) == 1:
				lowercase__				= os.path.join(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)
				torch.save(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)
				return {weights_name: sharded_state_dicts[0]}, None
		else:
				torch.save(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)
		# Otherwise, let's build the index
		lowercase__				= {}
		for idx, shard in enumerate(SCREAMING_SNAKE_CASE	):
				lowercase__				= weights_name.replace('''.bin'''		,							f'-{idx+1:05d}-of-{len(SCREAMING_SNAKE_CASE	):05d}.bin'	)
				lowercase__				= os.path.join(SCREAMING_SNAKE_CASE		,							weights_name.replace('''.bin'''		,							f'-{idx+1:05d}-of-???.bin'	)	)
				os.rename(SCREAMING_SNAKE_CASE		,							os.path.join(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)	)
				for key in shard:
						lowercase__				= shard_file

    # Add the metadata
		lowercase__				= {'''total_size''': total_size}
		lowercase__				= {'''metadata''': metadata, '''weight_map''': weight_map}

		with open(os.path.join(SCREAMING_SNAKE_CASE		,							SCREAMING_SNAKE_CASE	)		,							'''w'''		,							encoding='''utf-8'''	) as f:
				lowercase__				= json.dumps(SCREAMING_SNAKE_CASE		,							indent=2		,							sort_keys=SCREAMING_SNAKE_CASE	) + '''\n'''
				f.write(SCREAMING_SNAKE_CASE	)

		return metadata, index


if __name__ == "__main__":
			lowerCAmelCase			=				argparse.ArgumentParser()
			# Required parameters
			parser.add_argument(
			    '--nllb_moe_checkpoint_path',
			    default='/home/arthur_huggingface_co/fairseq/weights/checkpoints/model_moe_54b/checkpoint_2_300000',
			    type=str,
			    required=False,
			    help='Path to a directory containing a folder per layer. Follows the original Google format.',
			)
			parser.add_argument('--dtype', default='float32', type=str, required=False, help='dtype of the saved model')
			parser.add_argument(
			    '--pytorch_dump_folder_path',
			    default='/home/arthur_huggingface_co/fairseq/weights/checkpoints/hf-converted-moe-54b',
			    type=str,
			    required=False,
			    help='Path to the output pytorch model.',
			)
			lowerCAmelCase			=				parser.parse_args()
			lowerCAmelCase,		lowerCAmelCase			=				shard_on_the_fly(
			    args.nllb_moe_checkpoint_path,
			    args.pytorch_dump_folder_path,
			    128,
			    args.dtype,
			)

			lowerCAmelCase			=				NllbMoeConfig.from_pretrained(
			    'facebook/nllb-200-3.3B', encoder_sparse_step=4, decoder_sparse_step=4, num_experts=128
			)
			config.save_pretrained(args.pytorch_dump_folder_path)
			lowerCAmelCase			=				NllbMoeModel.from_pretrained(args.pytorch_dump_folder_path)
			print('Done')
			model.save_pretrained(args.pytorch_dump_folder_path)








from typing import Dict, List, Optional, Tuple, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_torch_available, is_torch_tensor, logging


if is_torch_available():
					import torch


__A								=       logging.get_logger(__name__)







class        __lowerCAmelCase					(			__magic_name__					):

			"""simple docstring"""







			snake_case_  						=			['''pixel_values''']


			def __init__( self     ,		lowerCamelCase__ = True     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = PILImageResampling.BILINEAR     ,		lowerCamelCase__ = True     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = True     ,		lowerCamelCase__ = 1 / 255     ,		lowerCamelCase__ = True     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		**lowerCamelCase__     ,		)       -> None:



							'''simple docstring'''
							super().__init__(**lowerCamelCase__				)
							__lowerCamelCase									=       size if size is not None else {'shortest_edge': 256}
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__     ,		default_to_square=lowerCamelCase__				)
							__lowerCamelCase									=       crop_size if crop_size is not None else {'height': 224, 'width': 224}
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__     ,		param_name='crop_size'				)
							__lowerCamelCase									=       do_resize
							__lowerCamelCase									=       size
							__lowerCamelCase									=       resample
							__lowerCamelCase									=       do_center_crop
							__lowerCamelCase									=       crop_size
							__lowerCamelCase									=       do_rescale
							__lowerCamelCase									=       rescale_factor
							__lowerCamelCase									=       do_normalize
							__lowerCamelCase									=       image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
							__lowerCamelCase									=       image_std if image_std is not None else IMAGENET_STANDARD_STD


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__ = PILImageResampling.BICUBIC     ,		lowerCamelCase__ = None     ,		**lowerCamelCase__     ,		)       -> np.ndarray:



							'''simple docstring'''
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__     ,		default_to_square=lowerCamelCase__				)
							if "shortest_edge" not in size:
											raise ValueError(f"""The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}"""				)
							__lowerCamelCase									=       get_resize_output_image_size(lowerCamelCase__     ,		size=size['shortest_edge']     ,		default_to_square=lowerCamelCase__				)
							return resize(lowerCamelCase__     ,		size=lowerCamelCase__     ,		resample=lowerCamelCase__     ,		data_format=lowerCamelCase__     ,		**lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__ = None     ,		**lowerCamelCase__     ,		)       -> np.ndarray:



							'''simple docstring'''
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__				)
							if "height" not in size or "width" not in size:
											raise ValueError(f"""The `size` parameter must contain the keys `height` and `width`. Got {size.keys()}"""				)
							return center_crop(lowerCamelCase__     ,		size=(size['height'], size['width'])     ,		data_format=lowerCamelCase__     ,		**lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__ = None     ,		**lowerCamelCase__				)       -> np.ndarray:



							'''simple docstring'''
							return rescale(lowerCamelCase__     ,		scale=lowerCamelCase__     ,		data_format=lowerCamelCase__     ,		**lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__ = None     ,		**lowerCamelCase__     ,		)       -> np.ndarray:



							'''simple docstring'''
							return normalize(lowerCamelCase__     ,		mean=lowerCamelCase__     ,		std=lowerCamelCase__     ,		data_format=lowerCamelCase__     ,		**lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = ChannelDimension.FIRST     ,		**lowerCamelCase__     ,		)       -> str:



							'''simple docstring'''
							__lowerCamelCase									=       do_resize if do_resize is not None else self.do_resize
							__lowerCamelCase									=       size if size is not None else self.size
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__     ,		default_to_square=lowerCamelCase__				)
							__lowerCamelCase									=       resample if resample is not None else self.resample
							__lowerCamelCase									=       do_center_crop if do_center_crop is not None else self.do_center_crop
							__lowerCamelCase									=       crop_size if crop_size is not None else self.crop_size
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__     ,		param_name='crop_size'				)
							__lowerCamelCase									=       do_rescale if do_rescale is not None else self.do_rescale
							__lowerCamelCase									=       rescale_factor if rescale_factor is not None else self.rescale_factor
							__lowerCamelCase									=       do_normalize if do_normalize is not None else self.do_normalize
							__lowerCamelCase									=       image_mean if image_mean is not None else self.image_mean
							__lowerCamelCase									=       image_std if image_std is not None else self.image_std

							__lowerCamelCase									=       make_list_of_images(lowerCamelCase__				)

							if not valid_images(lowerCamelCase__				):
											raise ValueError(
											    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
											    'torch.Tensor, tf.Tensor or jax.ndarray.'				)

							if do_resize and size is None:
											raise ValueError('Size must be specified if do_resize is True.'				)

							if do_center_crop and crop_size is None:
											raise ValueError('Crop size must be specified if do_center_crop is True.'				)

							if do_rescale and rescale_factor is None:
											raise ValueError('Rescale factor must be specified if do_rescale is True.'				)

							if do_normalize and (image_mean is None or image_std is None):
											raise ValueError('Image mean and std must be specified if do_normalize is True.'				)

							# All transformations expect numpy arrays.
							__lowerCamelCase									=       [to_numpy_array(lowerCamelCase__				) for image in images]

							if do_resize:
											__lowerCamelCase									=       [self.resize(image=lowerCamelCase__     ,		size=lowerCamelCase__     ,		resample=lowerCamelCase__				) for image in images]

							if do_center_crop:
											__lowerCamelCase									=       [self.center_crop(image=lowerCamelCase__     ,		size=lowerCamelCase__				) for image in images]

							if do_rescale:
											__lowerCamelCase									=       [self.rescale(image=lowerCamelCase__     ,		scale=lowerCamelCase__				) for image in images]

							if do_normalize:
											__lowerCamelCase									=       [self.normalize(image=lowerCamelCase__     ,		mean=lowerCamelCase__     ,		std=lowerCamelCase__				) for image in images]

							__lowerCamelCase									=       [to_channel_dimension_format(lowerCamelCase__     ,		lowerCamelCase__				) for image in images]

							__lowerCamelCase									=       {'pixel_values': images}
							return BatchFeature(data=lowerCamelCase__     ,		tensor_type=lowerCamelCase__				)






			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__ = None				)       -> Optional[Any]:



							'''simple docstring'''
							__lowerCamelCase									=       outputs.logits

							# Resize logits and compute semantic segmentation maps
							if target_sizes is not None:
											if len(lowerCamelCase__				) != len(lowerCamelCase__				):
															raise ValueError(
															    'Make sure that you pass in as many target sizes as the batch dimension of the logits'				)

											if is_torch_tensor(lowerCamelCase__				):
															__lowerCamelCase									=       target_sizes.numpy()

											__lowerCamelCase									=       []

											for idx in range(len(lowerCamelCase__				)				):
															__lowerCamelCase									=       torch.nn.functional.interpolate(
															    logits[idx].unsqueeze(dim=0				)     ,		size=target_sizes[idx]     ,		mode='bilinear'     ,		align_corners=lowerCamelCase__				)
															__lowerCamelCase									=       resized_logits[0].argmax(dim=0				)
															semantic_segmentation.append(lowerCamelCase__				)
							else:
											__lowerCamelCase									=       logits.argmax(dim=1				)
											__lowerCamelCase									=       [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0]				)]

							return semantic_segmentation


