



"""simple docstring"""




_lowercase		:				Optional[Any]      		=range(2, 20 + 1)
_lowercase		:				Any      		=[10**k for k in range(ks[-1] + 1)]
_lowercase		:				dict[int, dict[int, list[list[int]]]]      		={}

def      lowerCAmelCase_					(  _lowercase    :       Optional[Any]						,       _lowercase    :       int						,       _lowercase    :       List[Any]						,       _lowercase    :       int)							->	Dict:







			"""simple docstring"""
			a__       :						List[Any]							=  sum(a_i[j] for j in range(_lowercase						,       len(_lowercase)))
			a__       :						List[str]							=  sum(a_i[j] * base[j] for j in range(min(len(_lowercase)						,       _lowercase)))

			a__       :						Union[str, Any]							=  0, 0
			a__       :						Optional[int]							=  n - i

			a__       :						Any							=  memo.get(_lowercase)

			if sub_memo is not None:
						a__       :						str							=  sub_memo.get(_lowercase)

						if jumps is not None and len(_lowercase) > 0:
									# find and make the largest jump without going over
									a__       :						Tuple							=  -1
									for _k in range(len(_lowercase) - 1						,       -1						,       -1):
												if jumps[_k][2] <= k and jumps[_k][1] <= max_dn:
															a__       :						int							=  _k
															break

									if max_jump >= 0:
												a__       :						Optional[int]							=  jumps[max_jump]
												# since the difference between jumps is cached, add c
												a__       :						str							=  diff + c
												for j in range(min(_lowercase						,       len(_lowercase))):
															a__       :						str							=  divmod(_lowercase						,       10)
												if new_c > 0:
															add(_lowercase						,       _lowercase						,       _lowercase)

						else:
									a__       :						int							=  []
			else:
						a__       :						Optional[int]							=  {c: []}
						a__       :						List[Any]							=  sub_memo

			if dn >= max_dn or c + diff >= base[k]:
						return diff, dn

			if k > ks[0]:
						while True:
									# keep doing smaller jumps
									a__       :						int							=  next_term(_lowercase						,       k - 1						,       i + dn						,       _lowercase)
									diff += _diff
									dn += terms_jumped

									if dn >= max_dn or c + diff >= base[k]:
												break
			else:
						# would be too small a jump, just compute sequential terms instead
						a__       :						Optional[int]							=  compute(_lowercase						,       _lowercase						,       i + dn						,       _lowercase)
						diff += _diff
						dn += terms_jumped

			a__       :						int							=  sub_memo[c]

			# keep jumps sorted by # of terms skipped
			a__       :						Optional[int]							=  0
			while j < len(_lowercase):
						if jumps[j][1] > dn:
									break
						j += 1

			# cache the jump for this value digitsum(b) and c
			sub_memo[c].insert(_lowercase						,       (diff, dn, k))
			return (diff, dn)

def      lowerCAmelCase_					(  _lowercase    :       Tuple						,       _lowercase    :       Optional[int]						,       _lowercase    :       str						,       _lowercase    :       List[str])							->	Dict:







			"""simple docstring"""
			if i >= n:
						return 0, i
			if k > len(_lowercase):
						a_i.extend([0 for _ in range(k - len(_lowercase))])

			# note: a_i -> b * 10^k + c
			# ds_b -> digitsum(b)
			# ds_c -> digitsum(c)
			a__       :						Optional[int]							=  i
			a__       :						Optional[Any]							=  0, 0, 0
			for j in range(len(_lowercase)):
						if j >= k:
									ds_b += a_i[j]
						else:
									ds_c += a_i[j]

			while i < n:
						i += 1
						a__       :						Optional[Any]							=  ds_c + ds_b
						diff += addend
						a__       :						Union[str, Any]							=  0
						for j in range(_lowercase):
									a__       :						Tuple							=  a_i[j] + addend
									a__       :						Optional[Any]							=  divmod(_lowercase						,       10)

									ds_c += a_i[j]

						if addend > 0:
									break

			if addend > 0:
						add(_lowercase						,       _lowercase						,       _lowercase)
			return diff, i - start_i

def      lowerCAmelCase_					(  _lowercase    :       List[Any]						,       _lowercase    :       Optional[Any]						,       _lowercase    :       List[str])							->	Dict:







			"""simple docstring"""
			for j in range(_lowercase						,       len(_lowercase)):
						a__       :						List[Any]							=  digits[j] + addend
						if s >= 10:
									a__       :						Optional[Any]							=  divmod(_lowercase						,       10)
									a__       :						Dict							=  addend // 10 + quotient
						else:
									a__       :						List[Any]							=  s
									a__       :						Tuple							=  addend // 10

						if addend == 0:
									break

			while addend > 0:
						a__       :						int							=  divmod(_lowercase						,       10)
						digits.append(_lowercase)

def      lowerCAmelCase_					(  _lowercase    :       int = 10**15)							->	int:







			"""simple docstring"""
			a__       :						Optional[int]							=  [1]
			a__       :						str							=  1
			a__       :						Tuple							=  0
			while True:
						a__       :						Dict							=  next_term(_lowercase						,       20						,       i + dn						,       _lowercase)
						dn += terms_jumped
						if dn == n - i:
									break

			a__       :						Dict							=  0
			for j in range(len(_lowercase)):
						a_n += digits[j] * 10**j
			return a_n


if __name__ == "__main__":
		print(f'{solution() = }')









import argparse
import json
import os

import torch
from torch import nn

from transformers import NllbMoeConfig, NllbMoeModel
from transformers.modeling_utils import dtype_byte_size
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME

def      lowerCAmelCase_					(  _lowercase    :       List[str])							->	Union[str, Any]:







			"""simple docstring"""
			a__       :						List[str]							=  [
			    """encoder.version""",
			    """decoder.version""",
			    """model.encoder.version""",
			    """model.decoder.version""",
			    """decoder.output_projection.weight""",
			    """_float_tensor""",
			    """encoder.embed_positions._float_tensor""",
			    """decoder.embed_positions._float_tensor""",
			]
			for k in ignore_keys:
						state_dict.pop(_lowercase						,       _lowercase)

def      lowerCAmelCase_					(  _lowercase    :       List[Any])							->	Optional[Any]:







			"""simple docstring"""
			a__	,						a__       :						Union[str, Any]							=  emb.weight.shape
			a__       :						str							=  nn.Linear(_lowercase						,       _lowercase						,       bias=_lowercase)
			a__       :						Any							=  emb.weight.data
			return lin_layer

def      lowerCAmelCase_					(  _lowercase    :       int						,       _lowercase    :       int=None)							->	List[Any]:







			"""simple docstring"""
			a__       :						List[str]							=  {}
			for old_key in state_dict.keys():
						a__       :						Any							=  old_key
						if "moe_layer.experts." in key:
									if expert_idx is not None:
												a__       :						Dict							=  key.replace("""moe_layer.experts.0"""						,       F'''ffn.experts.expert_{expert_idx}''')
									else:
												a__       :						Optional[int]							=  key.replace("""moe_layer.experts."""						,       """ffn.experts.expert_""")
						if "gate" in key:
									a__       :						Tuple							=  key.replace(""".moe_layer.gate.wg"""						,       """.ffn.router.classifier""")
						if "fc2" and "experts" not in key:
									a__       :						Optional[int]							=  key.replace(""".fc2."""						,       """.ffn.fc2.""")
						if "fc1" and "experts" not in key:
									a__       :						Optional[int]							=  key.replace(""".fc1."""						,       """.ffn.fc1.""")
						if ".encoder_attn." in key:
									a__       :						Tuple							=  key.replace(""".encoder_attn."""						,       """.cross_attention.""")
						if "encoder_attn_layer_norm" in key:
									a__       :						Optional[Any]							=  key.replace("""encoder_attn_layer_norm"""						,       """cross_attention_layer_norm""")
						if "final_layer_norm" in key:
									a__       :						List[str]							=  key.replace("""final_layer_norm"""						,       """ff_layer_norm""")
						a__       :						str							=  state_dict[old_key]
			return new_dict

def      lowerCAmelCase_					(  _lowercase    :       Tuple						,       _lowercase    :       Optional[int]						,       _lowercase    :       Any						,       _lowercase    :       Dict						,       _lowercase    :       str = WEIGHTS_NAME)							->	Tuple:







			"""simple docstring"""
			a__       :						Tuple							=  []
			a__       :						Optional[Any]							=  0
			os.makedirs(_lowercase						,       exist_ok=_lowercase)

			for expert in range(_lowercase):
						a__       :						str							=  switch_checkpoint_path + F'''-rank-{expert}.pt'''
						if os.path.isfile(_lowercase):
									a__       :						List[str]							=  torch.load(_lowercase)["""model"""]
									remove_ignore_keys_(_lowercase)
									a__       :						Tuple							=  rename_fairseq_keys(_lowercase						,       _lowercase)
									a__       :						str							=  os.path.join(
									    _lowercase						,       weights_name.replace(""".bin"""						,       F'''-{len(_lowercase)+1:05d}-of-???.bin'''))
									torch.save(_lowercase						,       _lowercase)
									sharded_state_dicts.append(expert_state.keys())
									total_size += sum([value.numel() for key, value in expert_state.items()]) * dtype_byte_size(
									    expert_state[list(_lowercase)[0]].dtype)

    # Add the last block
			a__       :						int							=  os.path.join(_lowercase						,       weights_name.replace(""".bin"""						,       F'''-{len(_lowercase)+1:05d}-of-???.bin'''))
			a__       :						Union[str, Any]							=  torch.load(switch_checkpoint_path + """-shared.pt""")["""model"""]
			remove_ignore_keys_(_lowercase)
			a__       :						List[str]							=  rename_fairseq_keys(_lowercase						,       _lowercase)
			a__       :						int							=  shared_weights["""decoder.embed_tokens.weight"""]
			sharded_state_dicts.append(shared_weights.keys())

			# If we only have the shared weights (dummy model/experts saved on the same file)
			if len(_lowercase) == 1:
						a__       :						Optional[int]							=  os.path.join(_lowercase						,       _lowercase)
						torch.save(_lowercase						,       _lowercase)
						return {weights_name: sharded_state_dicts[0]}, None
			else:
						torch.save(_lowercase						,       _lowercase)
			# Otherwise, let's build the index
			a__       :						List[str]							=  {}
			for idx, shard in enumerate(_lowercase):
						a__       :						Union[str, Any]							=  weights_name.replace(""".bin"""						,       F'''-{idx+1:05d}-of-{len(_lowercase):05d}.bin''')
						a__       :						List[str]							=  os.path.join(_lowercase						,       weights_name.replace(""".bin"""						,       F'''-{idx+1:05d}-of-???.bin'''))
						os.rename(_lowercase						,       os.path.join(_lowercase						,       _lowercase))
						for key in shard:
									a__       :						Tuple							=  shard_file

    # Add the metadata
			a__       :						Tuple							=  {"""total_size""": total_size}
			a__       :						Optional[Any]							=  {"""metadata""": metadata, """weight_map""": weight_map}

			with open(os.path.join(_lowercase						,       _lowercase)						,       """w"""						,       encoding="""utf-8""") as f:
						a__       :						Dict							=  json.dumps(_lowercase						,       indent=2						,       sort_keys=_lowercase) + """\n"""
						f.write(_lowercase)

			return metadata, index


if __name__ == "__main__":
		_lowercase		:				Any      		=argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    "--nllb_moe_checkpoint_path",
		    default="/home/arthur_huggingface_co/fairseq/weights/checkpoints/model_moe_54b/checkpoint_2_300000",
		    type=str,
		    required=False,
		    help="Path to a directory containing a folder per layer. Follows the original Google format.",
		)
		parser.add_argument("--dtype", default="float32", type=str, required=False, help="dtype of the saved model")
		parser.add_argument(
		    "--pytorch_dump_folder_path",
		    default="/home/arthur_huggingface_co/fairseq/weights/checkpoints/hf-converted-moe-54b",
		    type=str,
		    required=False,
		    help="Path to the output pytorch model.",
		)
		_lowercase		:				Tuple      		=parser.parse_args()
		_lowercase		,					_lowercase		:				List[Any]      		=shard_on_the_fly(
		    args.nllb_moe_checkpoint_path,
		    args.pytorch_dump_folder_path,
		    128,
		    args.dtype,
		)

		_lowercase		:				int      		=NllbMoeConfig.from_pretrained(
		    "facebook/nllb-200-3.3B", encoder_sparse_step=4, decoder_sparse_step=4, num_experts=128
		)
		config.save_pretrained(args.pytorch_dump_folder_path)
		_lowercase		:				List[str]      		=NllbMoeModel.from_pretrained(args.pytorch_dump_folder_path)
		print("Done")
		model.save_pretrained(args.pytorch_dump_folder_path)




