




"""simple docstring"""







import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

a  =				[
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

a  =				[
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]





def 				lowercase       (snake_case__ :			Any   ,							snake_case__ :			Optional[int]			)  ->	Optional[int]:


				'''simple docstring'''






				lowerCAmelCase						=					{
				    'word_embeddings.weight': 'word_embeddings.weight',
				    'word_embeddings.norm.weight': 'word_embeddings_layernorm.weight',
				    'word_embeddings.norm.bias': 'word_embeddings_layernorm.bias',
				    'weight': 'ln_f.weight',
				    'bias': 'ln_f.bias',
				}

				if key in layer_rename_map:
								return layer_rename_map[key]

				# Handle transformer blocks
				lowerCAmelCase						=					int(re.match(R""".*layer_(\d*).*"""   ,							UpperCamelCase__			)[1]			)
				layer_number -= 3
				return f'''h.{layer_number}.''' + key





def 				lowercase       (snake_case__ :			Dict			)  ->	Dict:


				'''simple docstring'''






				if dtype == torch.bool:
								return 1 / 8
				lowerCAmelCase						=					re.search(R"""[^\d](\d+)$"""   ,							str(UpperCamelCase__			)			)
				if bit_search is None:
								raise ValueError(f'''`dtype` is not a valid dtype: {dtype}.'''			)
				lowerCAmelCase						=					int(bit_search.groups()[0]			)
				return bit_size // 8





def 				lowercase       (snake_case__ :			Any   ,							snake_case__ :			Union[str, Any]   ,							snake_case__ :			Tuple   ,							snake_case__ :			int   ,							snake_case__ :			str			)  ->	Optional[Any]:


				'''simple docstring'''






				if bloom_config_file == "":
								lowerCAmelCase						=					BloomConfig()
				else:
								lowerCAmelCase						=					BloomConfig.from_json_file(UpperCamelCase__			)

				if shard_model:
								lowerCAmelCase						=					os.listdir(UpperCamelCase__			)
								lowerCAmelCase						=					sorted(filter(lambda snake_case__			: s.startswith("""layer"""			) and "model_00" in s   ,							UpperCamelCase__			)			)

								lowerCAmelCase						=					{'weight_map': {}, 'metadata': {}}
								lowerCAmelCase						=					0

								lowerCAmelCase						=					None

								lowerCAmelCase						=					BloomConfig()

								for j, file in enumerate(UpperCamelCase__			):
												print("""Processing file: {}""".format(UpperCamelCase__			)			)
												lowerCAmelCase						=					None

												for i in range(UpperCamelCase__			):
																# load all TP files
																lowerCAmelCase						=					file.replace("""model_00"""   ,							f'''model_0{i}'''			)
																lowerCAmelCase						=					torch.load(os.path.join(UpperCamelCase__   ,							UpperCamelCase__			)   ,							map_location="""cpu"""			)

																# Rename keys in the transformers names
																lowerCAmelCase						=					list(temp.keys()			)
																for key in keys:
																				lowerCAmelCase						=					temp.pop(UpperCamelCase__			)

																if tensors is None:
																				lowerCAmelCase						=					temp
																else:
																				for key in tensors.keys():
																								if any(key.endswith(UpperCamelCase__			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
																												# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																												tensors[key] += temp[key]
																								else:
																												# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																												lowerCAmelCase						=					1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN			) else 0
																												# We concatenate these weights accross TP ranks
																												lowerCAmelCase						=					torch.cat([tensors[key], temp[key]]   ,							dim=UpperCamelCase__			)

																												     # Divide by the number of TP the weights we want to average
												for key in tensors.keys():
																if any(key.endswith(UpperCamelCase__			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
																				lowerCAmelCase						=					tensors[key] / pretraining_tp
												torch.save(
												    UpperCamelCase__   ,							os.path.join(
												        UpperCamelCase__   ,							"""pytorch_model_{}-of-{}.bin""".format(str(j + 1			).zfill(5			)   ,							str(len(UpperCamelCase__			)			).zfill(5			)			)   ,							)   ,							)

												for key in tensors.keys():
																lowerCAmelCase						=					tensors[key]
																total_size += value.numel() * get_dtype_size(value.dtype			)
																if key not in index_dict["weight_map"]:
																				lowerCAmelCase						=					'pytorch_model_{}-of-{}.bin'.format(
																				    str(j + 1			).zfill(5			)   ,							str(len(UpperCamelCase__			)			).zfill(5			)			)

								lowerCAmelCase						=					BloomConfig()
								lowerCAmelCase						=					pytorch_dump_folder_path + '/' + CONFIG_NAME
								lowerCAmelCase						=					total_size
								with open(UpperCamelCase__   ,							"""w"""   ,							encoding="""utf-8"""			) as f:
												f.write(config.to_json_string()			)
								with open(os.path.join(UpperCamelCase__   ,							WEIGHTS_NAME + """.index.json"""			)   ,							"""w"""   ,							encoding="""utf-8"""			) as f:
												lowerCAmelCase						=					json.dumps(UpperCamelCase__   ,							indent=2   ,							sort_keys=UpperCamelCase__			) + '\n'
												f.write(UpperCamelCase__			)
				else:
								lowerCAmelCase						=					BloomModel(UpperCamelCase__			)

								lowerCAmelCase						=					os.listdir(UpperCamelCase__			)
								lowerCAmelCase						=					sorted(filter(lambda snake_case__			: s.startswith("""layer"""			) and "model_00" in s   ,							UpperCamelCase__			)			)

								lowerCAmelCase						=					None
								for i, file in enumerate(UpperCamelCase__			):
												lowerCAmelCase						=					None
												for i in range(UpperCamelCase__			):
																# load all TP files
																lowerCAmelCase						=					file.replace("""model_00"""   ,							f'''model_0{i}'''			)
																lowerCAmelCase						=					torch.load(os.path.join(UpperCamelCase__   ,							UpperCamelCase__			)   ,							map_location="""cpu"""			)

																# Rename keys in the transformers names
																lowerCAmelCase						=					list(temp.keys()			)
																for key in keys:
																				lowerCAmelCase						=					temp.pop(UpperCamelCase__			)

																if tensors is None:
																				lowerCAmelCase						=					temp
																else:
																				for key in tensors.keys():
																								# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																								if any(key.endswith(UpperCamelCase__			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
																												tensors[key] += temp[key]
																								else:
																												# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																												lowerCAmelCase						=					1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN			) else 0
																												# We concatenate these weights accross TP ranks
																												lowerCAmelCase						=					torch.cat([tensors[key], temp[key]]   ,							dim=UpperCamelCase__			)

																												     # Divide by the number of TP the weights we want to average
												for key in tensors.keys():
																if any(key.endswith(UpperCamelCase__			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
																				lowerCAmelCase						=					tensors[key] / pretraining_tp

												lowerCAmelCase						=					model.load_state_dict(UpperCamelCase__   ,							strict=UpperCamelCase__			)
												assert not other_keys.unexpected_keys, f'''The keys {other_keys.unexpected_keys} are unexpected'''
												if missing_keys is None:
																lowerCAmelCase						=					set(other_keys.missing_keys			)
												else:
																lowerCAmelCase						=					missing_keys.intersection(set(other_keys.missing_keys			)			)

								assert not missing_keys, f'''The keys {missing_keys} are missing'''

								# Save pytorch-model
								os.makedirs(UpperCamelCase__   ,							exist_ok=UpperCamelCase__			)
								lowerCAmelCase						=					pytorch_dump_folder_path + '/' + WEIGHTS_NAME
								lowerCAmelCase						=					pytorch_dump_folder_path + '/' + CONFIG_NAME
								print(f'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''			)
								if config.torch_dtype is not None:
												lowerCAmelCase						=					model.to(config.torch_dtype			)
								torch.save(model.state_dict()   ,							UpperCamelCase__			)
								print(f'''Save configuration file to {pytorch_config_dump_path}'''			)
								with open(UpperCamelCase__   ,							"""w"""   ,							encoding="""utf-8"""			) as f:
												f.write(config.to_json_string()			)


if __name__ == "__main__":
		a  =				argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '--bloom_checkpoint_path',
		    default=None,
		    type=str,
		    required=True,
		    help='Path to the Megatron-LM checkpoint path.',
		)
		parser.add_argument(
		    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
		)
		parser.add_argument(
		    '--bloom_config_file',
		    default='',
		    type=str,
		    help=(
		        'An optional config json file corresponding to the pre-trained model. \n'
		        'This specifies the model architecture.'
		    ),
		)
		parser.add_argument(
		    '--shard_model',
		    action='store_true',
		    help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
		)
		parser.add_argument(
		    '--pretraining_tp',
		    default=4,
		    type=int,
		    help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
		)
		a  =				parser.parse_args()
		convert_bloom_checkpoint_to_pytorch(
		    args.bloom_checkpoint_path,
		    args.bloom_config_file,
		    args.pytorch_dump_folder_path,
		    args.shard_model,
		    args.pretraining_tp,
		)









'''simple docstring'''
from math import factorial

UpperCAmelCase_			:  List[str]										=   {str(d): factorial(d) for d in range(1_0)}


def       _lowercase							(					UpperCamelCase__							:       int      ):
 return sum(DIGIT_FACTORIAL[d] for d in str(UpperCamelCase__      )      )


def       _lowercase							(					):
 __A	:		Union[str, Any]	  =      7 * factorial(9      ) + 1
 return sum(i for i in range(3,					UpperCamelCase__      ) if sum_of_digit_factorial(UpperCamelCase__      ) == i      )


if __name__ == "__main__":
 print(f'''{solution() = }''')






