






import argparse
import json
import os

import torch
from torch import nn

from transformers import NllbMoeConfig, NllbMoeModel
from transformers.modeling_utils import dtype_byte_size
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME

def     snake_case   (UpperCAmelCase__    )			->							Any:
  UpperCamelCase_:   Optional[Any]        =							[
      'encoder.version',
      'decoder.version',
      'model.encoder.version',
      'model.decoder.version',
      'decoder.output_projection.weight',
      '_float_tensor',
      'encoder.embed_positions._float_tensor',
      'decoder.embed_positions._float_tensor',
  ]
  for k in ignore_keys:
    state_dict.pop(_A      ,       _A    )

def     snake_case   (UpperCAmelCase__    )			->							List[str]:
  UpperCamelCase_ ,UpperCamelCase_:   Tuple        =							emb.weight.shape
  UpperCamelCase_:   Union[str, Any]        =							nn.Linear(_A      ,       _A      ,       bias=_A    )
  UpperCamelCase_:   Optional[int]        =							emb.weight.data
  return lin_layer

def     snake_case   (UpperCAmelCase__      ,       UpperCAmelCase__=None    )			->							List[str]:
  UpperCamelCase_:   List[Any]        =							{}
  for old_key in state_dict.keys():
    UpperCamelCase_:   int        =							old_key
    if "moe_layer.experts." in key:
      if expert_idx is not None:
        UpperCamelCase_:   Dict        =							key.replace('moe_layer.experts.0'      ,       F'''ffn.experts.expert_{expert_idx}'''    )
      else:
        UpperCamelCase_:   Optional[int]        =							key.replace('moe_layer.experts.'      ,       'ffn.experts.expert_'    )
    if "gate" in key:
      UpperCamelCase_:   str        =							key.replace('.moe_layer.gate.wg'      ,       '.ffn.router.classifier'    )
    if "fc2" and "experts" not in key:
      UpperCamelCase_:   Union[str, Any]        =							key.replace('.fc2.'      ,       '.ffn.fc2.'    )
    if "fc1" and "experts" not in key:
      UpperCamelCase_:   Optional[Any]        =							key.replace('.fc1.'      ,       '.ffn.fc1.'    )
    if ".encoder_attn." in key:
      UpperCamelCase_:   str        =							key.replace('.encoder_attn.'      ,       '.cross_attention.'    )
    if "encoder_attn_layer_norm" in key:
      UpperCamelCase_:   Optional[int]        =							key.replace('encoder_attn_layer_norm'      ,       'cross_attention_layer_norm'    )
    if "final_layer_norm" in key:
      UpperCamelCase_:   Optional[Any]        =							key.replace('final_layer_norm'      ,       'ff_layer_norm'    )
    UpperCamelCase_:   Optional[int]        =							state_dict[old_key]
  return new_dict

def     snake_case   (UpperCAmelCase__      ,       UpperCAmelCase__      ,       UpperCAmelCase__      ,       UpperCAmelCase__      ,       UpperCAmelCase__ = WEIGHTS_NAME    )			->							str:
  UpperCamelCase_:   Any        =							[]
  UpperCamelCase_:   Union[str, Any]        =							0
  os.makedirs(_A      ,       exist_ok=_A    )

  for expert in range(_A    ):
    UpperCamelCase_:   Optional[Any]        =							switch_checkpoint_path + F'''-rank-{expert}.pt'''
    if os.path.isfile(_A    ):
      UpperCamelCase_:   Any        =							torch.load(_A    )['model']
      remove_ignore_keys_(_A    )
      UpperCamelCase_:   Optional[int]        =							rename_fairseq_keys(_A      ,       _A    )
      UpperCamelCase_:   Optional[Any]        =							os.path.join(
          _A      ,       weights_name.replace('.bin'      ,       F'''-{len(_A    )+1:05d}-of-???.bin'''    )    )
      torch.save(_A      ,       _A    )
      sharded_state_dicts.append(expert_state.keys()    )
      total_size += sum([value.numel() for key, value in expert_state.items()]    ) * dtype_byte_size(
          expert_state[list(_A    )[0]].dtype    )

    # Add the last block
  UpperCamelCase_:   Tuple        =							os.path.join(_A      ,       weights_name.replace('.bin'      ,       F'''-{len(_A    )+1:05d}-of-???.bin'''    )    )
  UpperCamelCase_:   Optional[Any]        =							torch.load(switch_checkpoint_path + '-shared.pt'    )['model']
  remove_ignore_keys_(_A    )
  UpperCamelCase_:   Tuple        =							rename_fairseq_keys(_A      ,       _A    )
  UpperCamelCase_:   int        =							shared_weights['decoder.embed_tokens.weight']
  sharded_state_dicts.append(shared_weights.keys()    )

  # If we only have the shared weights (dummy model/experts saved on the same file)
  if len(_A    ) == 1:
    UpperCamelCase_:   Dict        =							os.path.join(_A      ,       _A    )
    torch.save(_A      ,       _A    )
    return {weights_name: sharded_state_dicts[0]}, None
  else:
    torch.save(_A      ,       _A    )
  # Otherwise, let's build the index
  UpperCamelCase_:   Dict        =							{}
  for idx, shard in enumerate(_A    ):
    UpperCamelCase_:   int        =							weights_name.replace('.bin'      ,       F'''-{idx+1:05d}-of-{len(_A    ):05d}.bin'''    )
    UpperCamelCase_:   Union[str, Any]        =							os.path.join(_A      ,       weights_name.replace('.bin'      ,       F'''-{idx+1:05d}-of-???.bin'''    )    )
    os.rename(_A      ,       os.path.join(_A      ,       _A    )    )
    for key in shard:
      UpperCamelCase_:   Optional[int]        =							shard_file

    # Add the metadata
  UpperCamelCase_:   Any        =							{'total_size': total_size}
  UpperCamelCase_:   Dict        =							{'metadata': metadata, 'weight_map': weight_map}

  with open(os.path.join(_A      ,       _A    )      ,       'w'      ,       encoding='utf-8'    ) as f:
    UpperCamelCase_:   Tuple        =							json.dumps(_A      ,       indent=2      ,       sort_keys=_A    ) + '\n'
    f.write(_A    )

  return metadata, index


if __name__ == "__main__":
   A_						:						int						      =					argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '--nllb_moe_checkpoint_path',
       default='/home/arthur_huggingface_co/fairseq/weights/checkpoints/model_moe_54b/checkpoint_2_300000',
       type=str,
       required=False,
       help='Path to a directory containing a folder per layer. Follows the original Google format.',
   )
   parser.add_argument('--dtype', default='float32', type=str, required=False, help='dtype of the saved model')
   parser.add_argument(
       '--pytorch_dump_folder_path',
       default='/home/arthur_huggingface_co/fairseq/weights/checkpoints/hf-converted-moe-54b',
       type=str,
       required=False,
       help='Path to the output pytorch model.',
   )
   A_						:						List[str]						      =					parser.parse_args()
   A_      ,					A_						:						Any						      =					shard_on_the_fly(
       args.nllb_moe_checkpoint_path,
       args.pytorch_dump_folder_path,
       128,
       args.dtype,
   )

   A_						:						Any						      =					NllbMoeConfig.from_pretrained(
       'facebook/nllb-200-3.3B', encoder_sparse_step=4, decoder_sparse_step=4, num_experts=128
   )
   config.save_pretrained(args.pytorch_dump_folder_path)
   A_						:						Union[str, Any]						      =					NllbMoeModel.from_pretrained(args.pytorch_dump_folder_path)
   print('Done')
   model.save_pretrained(args.pytorch_dump_folder_path)



import re
import string

import numpy as np

import datasets


__UpperCamelCase     		=							'\nReturns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list.\n'

__UpperCamelCase     		=							'\nArgs:\n    predictions: List of predicted texts.\n    references: List of reference texts.\n    regexes_to_ignore: List, defaults to None. Regex expressions of characters to\n        ignore when calculating the exact matches. Note: these regexes are removed\n        from the input data before the changes based on the options below (e.g. ignore_case,\n        ignore_punctuation, ignore_numbers) are applied.\n    ignore_case: Boolean, defaults to False. If true, turns everything\n        to lowercase so that capitalization differences are ignored.\n    ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\n    ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\nReturns:\n    exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 100.0, inclusive.\nExamples:\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    25.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    50.0\n\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    75.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True)\n    >>> print(round(results["exact_match"], 1))\n    100.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["The cat sat on the mat.", "Theaters are great.", "It\'s like comparing oranges and apples."]\n    >>> preds = ["The cat sat on the mat?", "Theaters are great.", "It\'s like comparing apples and oranges."]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    33.3\n\n'

__UpperCamelCase     		=							'\n'







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION       ,						_KWARGS_DESCRIPTION   )
class 							lowerCamelCase__						(				datasets.Metric   ):






			"""simple docstring"""






			def 					snake_case__					(				self  ):

						'''simple docstring'''


						return datasets.MetricInfo(
						    description=_DESCRIPTION   ,    citation=_CITATION   ,    inputs_description=_KWARGS_DESCRIPTION   ,    features=datasets.Features(
						        {
						            "predictions": datasets.Value("string"   ,    id="sequence"  ),
						            "references": datasets.Value("string"   ,    id="sequence"  ),
						        }  )   ,    reference_urls=[]   ,    )







			def 					snake_case__					(				self   ,    snake_case   ,    snake_case   ,    snake_case=None   ,    snake_case=False   ,    snake_case=False   ,    snake_case=False   ,    ):

						'''simple docstring'''


						if regexes_to_ignore is not None:
									for s in regexes_to_ignore:
												UpperCamelCase__       =							np.array([re.sub(snake_case   ,    ""   ,    snake_case  ) for x in predictions]  )
												UpperCamelCase__       =							np.array([re.sub(snake_case   ,    ""   ,    snake_case  ) for x in references]  )
						else:
									UpperCamelCase__       =							np.asarray(snake_case  )
									UpperCamelCase__       =							np.asarray(snake_case  )

						if ignore_case:
									UpperCamelCase__       =							np.char.lower(snake_case  )
									UpperCamelCase__       =							np.char.lower(snake_case  )

						if ignore_punctuation:
									UpperCamelCase__       =							string.punctuation.maketrans(""   ,    ""   ,    string.punctuation  )
									UpperCamelCase__       =							np.char.translate(snake_case   ,    table=snake_case  )
									UpperCamelCase__       =							np.char.translate(snake_case   ,    table=snake_case  )

						if ignore_numbers:
									UpperCamelCase__       =							string.digits.maketrans(""   ,    ""   ,    string.digits  )
									UpperCamelCase__       =							np.char.translate(snake_case   ,    table=snake_case  )
									UpperCamelCase__       =							np.char.translate(snake_case   ,    table=snake_case  )

						UpperCamelCase__       =							predictions == references

						return {"exact_match": np.mean(snake_case  ) * 100}


