


"""simple docstring"""
import argparse
import json
import os

import torch
from torch import nn

from transformers import NllbMoeConfig, NllbMoeModel
from transformers.modeling_utils import dtype_byte_size
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME

def 				_snake_case      (   __snake_case							:      Any   ):




   """simple docstring"""







   _lowerCamelCase	:					List[Any]   =		[
       """encoder.version""",
       """decoder.version""",
       """model.encoder.version""",
       """model.decoder.version""",
       """decoder.output_projection.weight""",
       """_float_tensor""",
       """encoder.embed_positions._float_tensor""",
       """decoder.embed_positions._float_tensor""",
   ]
   for k in ignore_keys:
      state_dict.pop(__snake_case						, __snake_case   )

def 				_snake_case      (   __snake_case							:      Dict   ):




   """simple docstring"""







   _lowerCamelCase   ,    _lowerCamelCase	:					Optional[int]   =		emb.weight.shape
   _lowerCamelCase	:					Dict   =		nn.Linear(__snake_case						, __snake_case						, bias=__snake_case   )
   _lowerCamelCase	:					Dict   =		emb.weight.data
   return lin_layer

def 				_snake_case      (   __snake_case							:      Tuple						, __snake_case							:      List[str]=None   ):




   """simple docstring"""







   _lowerCamelCase	:					Any   =		{}
   for old_key in state_dict.keys():
      _lowerCamelCase	:					Optional[int]   =		old_key
      if "moe_layer.experts." in key:
         if expert_idx is not None:
            _lowerCamelCase	:					Optional[Any]   =		key.replace("""moe_layer.experts.0"""						, F'ffn.experts.expert_{expert_idx}'   )
         else:
            _lowerCamelCase	:					str   =		key.replace("""moe_layer.experts."""						, """ffn.experts.expert_"""   )
      if "gate" in key:
         _lowerCamelCase	:					Tuple   =		key.replace(""".moe_layer.gate.wg"""						, """.ffn.router.classifier"""   )
      if "fc2" and "experts" not in key:
         _lowerCamelCase	:					Any   =		key.replace(""".fc2."""						, """.ffn.fc2."""   )
      if "fc1" and "experts" not in key:
         _lowerCamelCase	:					Tuple   =		key.replace(""".fc1."""						, """.ffn.fc1."""   )
      if ".encoder_attn." in key:
         _lowerCamelCase	:					str   =		key.replace(""".encoder_attn."""						, """.cross_attention."""   )
      if "encoder_attn_layer_norm" in key:
         _lowerCamelCase	:					Optional[int]   =		key.replace("""encoder_attn_layer_norm"""						, """cross_attention_layer_norm"""   )
      if "final_layer_norm" in key:
         _lowerCamelCase	:					Any   =		key.replace("""final_layer_norm"""						, """ff_layer_norm"""   )
      _lowerCamelCase	:					Optional[Any]   =		state_dict[old_key]
   return new_dict

def 				_snake_case      (   __snake_case							:      Union[str, Any]						, __snake_case							:      str						, __snake_case							:      List[str]						, __snake_case							:      List[str]						, __snake_case							:      str = WEIGHTS_NAME   ):




   """simple docstring"""







   _lowerCamelCase	:					str   =		[]
   _lowerCamelCase	:					Dict   =		0
   os.makedirs(__snake_case						, exist_ok=__snake_case   )

   for expert in range(__snake_case   ):
      _lowerCamelCase	:					Tuple   =		switch_checkpoint_path + F'-rank-{expert}.pt'
      if os.path.isfile(__snake_case   ):
         _lowerCamelCase	:					int   =		torch.load(__snake_case   )["""model"""]
         remove_ignore_keys_(__snake_case   )
         _lowerCamelCase	:					int   =		rename_fairseq_keys(__snake_case						, __snake_case   )
         _lowerCamelCase	:					Union[str, Any]   =		os.path.join(
             __snake_case						, weights_name.replace(""".bin"""						, F'-{len(__snake_case   )+1:05d}-of-???.bin'   )   )
         torch.save(__snake_case						, __snake_case   )
         sharded_state_dicts.append(expert_state.keys()   )
         total_size += sum([value.numel() for key, value in expert_state.items()]   ) * dtype_byte_size(
             expert_state[list(__snake_case   )[0]].dtype   )

    # Add the last block
   _lowerCamelCase	:					List[Any]   =		os.path.join(__snake_case						, weights_name.replace(""".bin"""						, F'-{len(__snake_case   )+1:05d}-of-???.bin'   )   )
   _lowerCamelCase	:					Dict   =		torch.load(switch_checkpoint_path + """-shared.pt"""   )["""model"""]
   remove_ignore_keys_(__snake_case   )
   _lowerCamelCase	:					Optional[int]   =		rename_fairseq_keys(__snake_case						, __snake_case   )
   _lowerCamelCase	:					List[Any]   =		shared_weights["""decoder.embed_tokens.weight"""]
   sharded_state_dicts.append(shared_weights.keys()   )

   # If we only have the shared weights (dummy model/experts saved on the same file)
   if len(__snake_case   ) == 1:
      _lowerCamelCase	:					Dict   =		os.path.join(__snake_case						, __snake_case   )
      torch.save(__snake_case						, __snake_case   )
      return {weights_name: sharded_state_dicts[0]}, None
   else:
      torch.save(__snake_case						, __snake_case   )
   # Otherwise, let's build the index
   _lowerCamelCase	:					str   =		{}
   for idx, shard in enumerate(__snake_case   ):
      _lowerCamelCase	:					Optional[int]   =		weights_name.replace(""".bin"""						, F'-{idx+1:05d}-of-{len(__snake_case   ):05d}.bin'   )
      _lowerCamelCase	:					List[str]   =		os.path.join(__snake_case						, weights_name.replace(""".bin"""						, F'-{idx+1:05d}-of-???.bin'   )   )
      os.rename(__snake_case						, os.path.join(__snake_case						, __snake_case   )   )
      for key in shard:
         _lowerCamelCase	:					List[str]   =		shard_file

    # Add the metadata
   _lowerCamelCase	:					Optional[int]   =		{"""total_size""": total_size}
   _lowerCamelCase	:					Tuple   =		{"""metadata""": metadata, """weight_map""": weight_map}

   with open(os.path.join(__snake_case						, __snake_case   )						, """w"""						, encoding="""utf-8"""   ) as f:
      _lowerCamelCase	:					Union[str, Any]   =		json.dumps(__snake_case						, indent=2						, sort_keys=__snake_case   ) + """\n"""
      f.write(__snake_case   )

   return metadata, index


if __name__ == "__main__":
 UpperCAmelCase				=						argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     """--nllb_moe_checkpoint_path""",
     default="""/home/arthur_huggingface_co/fairseq/weights/checkpoints/model_moe_54b/checkpoint_2_300000""",
     type=str,
     required=False,
     help="""Path to a directory containing a folder per layer. Follows the original Google format.""",
 )
 parser.add_argument("""--dtype""", default="""float32""", type=str, required=False, help="""dtype of the saved model""")
 parser.add_argument(
     """--pytorch_dump_folder_path""",
     default="""/home/arthur_huggingface_co/fairseq/weights/checkpoints/hf-converted-moe-54b""",
     type=str,
     required=False,
     help="""Path to the output pytorch model.""",
 )
 UpperCAmelCase				=						parser.parse_args()
 UpperCAmelCase			,  UpperCAmelCase				=						shard_on_the_fly(
     args.nllb_moe_checkpoint_path,
     args.pytorch_dump_folder_path,
     128,
     args.dtype,
 )

 UpperCAmelCase				=						NllbMoeConfig.from_pretrained(
     """facebook/nllb-200-3.3B""", encoder_sparse_step=4, decoder_sparse_step=4, num_experts=128
 )
 config.save_pretrained(args.pytorch_dump_folder_path)
 UpperCAmelCase				=						NllbMoeModel.from_pretrained(args.pytorch_dump_folder_path)
 print("""Done""")
 model.save_pretrained(args.pytorch_dump_folder_path)



"""simple docstring"""






import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowercase							=				logging.get_logger(__name__)

_lowercase							=				{
    '''vocab_file''': '''vocab.txt''',
    '''merges_file''': '''bpe.codes''',
}

_lowercase							=				{
    '''vocab_file''': {
        '''vinai/phobert-base''': '''https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt''',
        '''vinai/phobert-large''': '''https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt''',
    },
    '''merges_file''': {
        '''vinai/phobert-base''': '''https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes''',
        '''vinai/phobert-large''': '''https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes''',
    },
}

_lowercase							=				{
    '''vinai/phobert-base''': 256,
    '''vinai/phobert-large''': 256,
}



def      lowerCAmelCase__			(		__magic_name__      )						->Optional[int]:
			__lowercase       =       set()
			__lowercase       =       word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)      )
						__lowercase       =       char

			__lowercase       =       set(__magic_name__      )
			return pairs







class 					__a     ( __a       ):



							'''simple docstring'''







							_lowerCamelCase					:			int          =   VOCAB_FILES_NAMES
							_lowerCamelCase					:			List[str]          =   PRETRAINED_VOCAB_FILES_MAP
							_lowerCamelCase					:			Optional[Any]          =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							def __init__(     self  ,    _lowerCamelCase  ,    _lowerCamelCase  ,    _lowerCamelCase="<s>"  ,    _lowerCamelCase="</s>"  ,    _lowerCamelCase="</s>"  ,    _lowerCamelCase="<s>"  ,    _lowerCamelCase="<unk>"  ,    _lowerCamelCase="<pad>"  ,    _lowerCamelCase="<mask>"  ,    **_lowerCamelCase  ,    )			->							Tuple:




										'''simple docstring'''
										super().__init__(
										    bos_token=_lowerCamelCase  ,    eos_token=_lowerCamelCase  ,    unk_token=_lowerCamelCase  ,    sep_token=_lowerCamelCase  ,    cls_token=_lowerCamelCase  ,    pad_token=_lowerCamelCase  ,    mask_token=_lowerCamelCase  ,    **_lowerCamelCase  ,    )

										__lowercase       =       vocab_file
										__lowercase       =       merges_file

										__lowercase       =       {}
										__lowercase       =       0
										__lowercase       =       1
										__lowercase       =       2
										__lowercase       =       3

										self.add_from_file(_lowerCamelCase       )

										__lowercase       =       {v: k for k, v in self.encoder.items()}

										with open(_lowerCamelCase  ,    encoding="utf-8"       ) as merges_handle:
													__lowercase       =       merges_handle.read().split("\n"       )[:-1]
										__lowercase       =       [tuple(merge.split()[:-1]       ) for merge in merges]
										__lowercase       =       dict(zip(_lowerCamelCase  ,    range(len(_lowerCamelCase       )       )       )       )
										__lowercase       =       {}
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase = None       )			->							List[int]:




										'''simple docstring'''

										if token_ids_a is None:
													return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										__lowercase       =       [self.cls_token_id]
										__lowercase       =       [self.sep_token_id]
										return cls + token_ids_a + sep + sep + token_ids_a + sep
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase = None  ,    _lowerCamelCase = False       )			->							List[int]:




										'''simple docstring'''

										if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=_lowerCamelCase  ,    token_ids_a=_lowerCamelCase  ,    already_has_special_tokens=_lowerCamelCase       )

										if token_ids_a is None:
													return [1] + ([0] * len(_lowerCamelCase       )) + [1]
										return [1] + ([0] * len(_lowerCamelCase       )) + [1, 1] + ([0] * len(_lowerCamelCase       )) + [1]
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase = None       )			->							List[int]:




										'''simple docstring'''
										__lowercase       =       [self.sep_token_id]
										__lowercase       =       [self.cls_token_id]

										if token_ids_a is None:
													return len(cls + token_ids_a + sep       ) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]
							@property
							def   SCREAMING_SNAKE_CASE						(     self       )			->							Optional[Any]:




										'''simple docstring'''
										return len(self.encoder       )
							def   SCREAMING_SNAKE_CASE						(     self       )			->							str:




										'''simple docstring'''
										return dict(self.encoder  ,    **self.added_tokens_encoder       )
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase       )			->							List[Any]:




										'''simple docstring'''
										if token in self.cache:
													return self.cache[token]
										__lowercase       =       tuple(_lowerCamelCase       )
										__lowercase       =       tuple(list(word[:-1]       ) + [word[-1] + "</w>"]       )
										__lowercase       =       get_pairs(_lowerCamelCase       )

										if not pairs:
													return token

										while True:
													__lowercase       =       min(_lowerCamelCase  ,    key=lambda _lowerCamelCase       : self.bpe_ranks.get(_lowerCamelCase  ,    float("inf"       )       )       )
													if bigram not in self.bpe_ranks:
																break
													__lowercase   ,					__lowercase       =       bigram
													__lowercase       =       []
													__lowercase       =       0
													while i < len(_lowerCamelCase       ):
																try:
																			__lowercase       =       word.index(_lowerCamelCase  ,    _lowerCamelCase       )
																except ValueError:
																			new_word.extend(word[i:]       )
																			break
																else:
																			new_word.extend(word[i:j]       )
																			__lowercase       =       j

																if word[i] == first and i < len(_lowerCamelCase       ) - 1 and word[i + 1] == second:
																			new_word.append(first + second       )
																			i += 2
																else:
																			new_word.append(word[i]       )
																			i += 1
													__lowercase       =       tuple(_lowerCamelCase       )
													__lowercase       =       new_word
													if len(_lowerCamelCase       ) == 1:
																break
													else:
																__lowercase       =       get_pairs(_lowerCamelCase       )
										__lowercase       =       "@@ ".join(_lowerCamelCase       )
										__lowercase       =       word[:-4]
										__lowercase       =       word
										return word
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase       )			->							str:




										'''simple docstring'''
										__lowercase       =       []

										__lowercase       =       re.findall(R"\S+\n?"  ,    _lowerCamelCase       )

										for token in words:
													split_tokens.extend(list(self.bpe(_lowerCamelCase       ).split(" "       )       )       )
										return split_tokens
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase       )			->							Optional[Any]:




										'''simple docstring'''
										return self.encoder.get(_lowerCamelCase  ,    self.encoder.get(self.unk_token       )       )
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase       )			->							Optional[int]:




										'''simple docstring'''
										return self.decoder.get(_lowerCamelCase  ,    self.unk_token       )
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase       )			->							Tuple:




										'''simple docstring'''
										__lowercase       =       " ".join(_lowerCamelCase       ).replace("@@ "  ,    ""       ).strip()
										return out_string
							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase  ,    _lowerCamelCase = None       )			->							Tuple[str]:




										'''simple docstring'''
										if not os.path.isdir(_lowerCamelCase       ):
													logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''       )
													return
										__lowercase       =       os.path.join(
										    _lowerCamelCase  ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )
										__lowercase       =       os.path.join(
										    _lowerCamelCase  ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]       )

										if os.path.abspath(self.vocab_file       ) != os.path.abspath(_lowerCamelCase       ):
													copyfile(self.vocab_file  ,    _lowerCamelCase       )

										if os.path.abspath(self.merges_file       ) != os.path.abspath(_lowerCamelCase       ):
													copyfile(self.merges_file  ,    _lowerCamelCase       )

										return out_vocab_file, out_merge_file





							def   SCREAMING_SNAKE_CASE						(     self  ,    _lowerCamelCase       )			->							Dict:




										'''simple docstring'''
										if isinstance(_lowerCamelCase  ,    _lowerCamelCase       ):
													try:
																with open(_lowerCamelCase  ,    "r"  ,    encoding="utf-8"       ) as fd:
																			self.add_from_file(_lowerCamelCase       )
													except FileNotFoundError as fnfe:
																raise fnfe
													except UnicodeError:
																raise Exception(f'''Incorrect encoding detected in {f}, please rebuild the dataset'''       )
													return

										__lowercase       =       f.readlines()
										for lineTmp in lines:
													__lowercase       =       lineTmp.strip()
													__lowercase       =       line.rfind(" "       )
													if idx == -1:
																raise ValueError("Incorrect dictionary format, expected '<token> <cnt>'"       )
													__lowercase       =       line[:idx]
													__lowercase       =       len(self.encoder       )





