





import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

lowerCAmelCase							:       int					 =	[
    """word_embeddings_layernorm.weight""",
    """word_embeddings_layernorm.bias""",
    """input_layernorm.weight""",
    """input_layernorm.bias""",
    """post_attention_layernorm.weight""",
    """post_attention_layernorm.bias""",
    """self_attention.dense.bias""",
    """mlp.dense_4h_to_h.bias""",
    """ln_f.weight""",
    """ln_f.bias""",
]

lowerCAmelCase							:       Optional[Any]					 =	[
    """mlp.dense_4h_to_h.weight""",
    """self_attention.dense.weight""",
]






def 		lowerCAmelCase  (	UpperCamelCase__   :   Union[str, Any]       ,			UpperCamelCase__   :   List[str]					)      ->    Optional[int]:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				int           =   {
						    '''word_embeddings.weight''': '''word_embeddings.weight''',
						    '''word_embeddings.norm.weight''': '''word_embeddings_layernorm.weight''',
						    '''word_embeddings.norm.bias''': '''word_embeddings_layernorm.bias''',
						    '''weight''': '''ln_f.weight''',
						    '''bias''': '''ln_f.bias''',
						}

						if key in layer_rename_map:
												return layer_rename_map[key]

						# Handle transformer blocks
						__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   int(re.match(R'''.*layer_(\d*).*'''       ,			UpperCamelCase__					)[1]					)
						layer_number -= 3
						return F"""h.{layer_number}.""" + key






def 		lowerCAmelCase  (	UpperCamelCase__   :   Optional[int]					)      ->    Optional[Any]:

						"""simple docstring"""





						if dtype == torch.bool:
												return 1 / 8
						__SCREAMING_SNAKE_CASE:				int           =   re.search(R'''[^\d](\d+)$'''       ,			str(UpperCamelCase__					)					)
						if bit_search is None:
												raise ValueError(F"""`dtype` is not a valid dtype: {dtype}."""					)
						__SCREAMING_SNAKE_CASE:				Dict           =   int(bit_search.groups()[0]					)
						return bit_size // 8






def 		lowerCAmelCase  (	UpperCamelCase__   :   Dict       ,			UpperCamelCase__   :   Optional[int]       ,			UpperCamelCase__   :   Tuple       ,			UpperCamelCase__   :   Dict       ,			UpperCamelCase__   :   str					)      ->    List[str]:

						"""simple docstring"""





						if bloom_config_file == "":
												__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   BloomConfig()
						else:
												__SCREAMING_SNAKE_CASE:				str           =   BloomConfig.from_json_file(UpperCamelCase__					)

						if shard_model:
												__SCREAMING_SNAKE_CASE:				Tuple           =   os.listdir(UpperCamelCase__					)
												__SCREAMING_SNAKE_CASE:				List[Any]           =   sorted(filter(lambda UpperCamelCase__					: s.startswith('''layer'''					) and "model_00" in s       ,			UpperCamelCase__					)					)

												__SCREAMING_SNAKE_CASE:				List[str]           =   {'''weight_map''': {}, '''metadata''': {}}
												__SCREAMING_SNAKE_CASE:				int           =   0

												__SCREAMING_SNAKE_CASE:				Optional[Any]           =   None

												__SCREAMING_SNAKE_CASE:				Optional[int]           =   BloomConfig()

												for j, file in enumerate(UpperCamelCase__					):
																		print('''Processing file: {}'''.format(UpperCamelCase__					)					)
																		__SCREAMING_SNAKE_CASE:				str           =   None

																		for i in range(UpperCamelCase__					):
																								# load all TP files
																								__SCREAMING_SNAKE_CASE:				str           =   file.replace('''model_00'''       ,			F"""model_0{i}"""					)
																								__SCREAMING_SNAKE_CASE:				int           =   torch.load(os.path.join(UpperCamelCase__       ,			UpperCamelCase__					)       ,			map_location='''cpu'''					)

																								# Rename keys in the transformers names
																								__SCREAMING_SNAKE_CASE:				Optional[int]           =   list(temp.keys()					)
																								for key in keys:
																														__SCREAMING_SNAKE_CASE:				int           =   temp.pop(UpperCamelCase__					)

																								if tensors is None:
																														__SCREAMING_SNAKE_CASE:				Tuple           =   temp
																								else:
																														for key in tensors.keys():
																																				if any(key.endswith(UpperCamelCase__					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
																																										# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																										tensors[key] += temp[key]
																																				else:
																																										# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																										__SCREAMING_SNAKE_CASE:				Optional[Any]           =   1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN					) else 0
																																										# We concatenate these weights accross TP ranks
																																										__SCREAMING_SNAKE_CASE:				List[str]           =   torch.cat([tensors[key], temp[key]]       ,			dim=UpperCamelCase__					)

            # Divide by the number of TP the weights we want to average
																		for key in tensors.keys():
																								if any(key.endswith(UpperCamelCase__					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
																														__SCREAMING_SNAKE_CASE:				Any           =   tensors[key] / pretraining_tp
																		torch.save(
																		    UpperCamelCase__       ,			os.path.join(
																		        UpperCamelCase__       ,			'''pytorch_model_{}-of-{}.bin'''.format(str(j + 1					).zfill(5					)       ,			str(len(UpperCamelCase__					)					).zfill(5					)					)       ,			)       ,			)

																		for key in tensors.keys():
																								__SCREAMING_SNAKE_CASE:				List[str]           =   tensors[key]
																								total_size += value.numel() * get_dtype_size(value.dtype					)
																								if key not in index_dict["weight_map"]:
																														__SCREAMING_SNAKE_CASE:				Optional[int]           =   '''pytorch_model_{}-of-{}.bin'''.format(
																														    str(j + 1					).zfill(5					)       ,			str(len(UpperCamelCase__					)					).zfill(5					)					)

												__SCREAMING_SNAKE_CASE:				List[Any]           =   BloomConfig()
												__SCREAMING_SNAKE_CASE:				int           =   pytorch_dump_folder_path + '''/''' + CONFIG_NAME
												__SCREAMING_SNAKE_CASE:				int           =   total_size
												with open(UpperCamelCase__       ,			'''w'''       ,			encoding='''utf-8'''					) as f:
																		f.write(config.to_json_string()					)
												with open(os.path.join(UpperCamelCase__       ,			WEIGHTS_NAME + '''.index.json'''					)       ,			'''w'''       ,			encoding='''utf-8'''					) as f:
																		__SCREAMING_SNAKE_CASE:				Tuple           =   json.dumps(UpperCamelCase__       ,			indent=2       ,			sort_keys=UpperCamelCase__					) + '''\n'''
																		f.write(UpperCamelCase__					)
						else:
												__SCREAMING_SNAKE_CASE:				Any           =   BloomModel(UpperCamelCase__					)

												__SCREAMING_SNAKE_CASE:				Tuple           =   os.listdir(UpperCamelCase__					)
												__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   sorted(filter(lambda UpperCamelCase__					: s.startswith('''layer'''					) and "model_00" in s       ,			UpperCamelCase__					)					)

												__SCREAMING_SNAKE_CASE:				str           =   None
												for i, file in enumerate(UpperCamelCase__					):
																		__SCREAMING_SNAKE_CASE:				Any           =   None
																		for i in range(UpperCamelCase__					):
																								# load all TP files
																								__SCREAMING_SNAKE_CASE:				List[Any]           =   file.replace('''model_00'''       ,			F"""model_0{i}"""					)
																								__SCREAMING_SNAKE_CASE:				Tuple           =   torch.load(os.path.join(UpperCamelCase__       ,			UpperCamelCase__					)       ,			map_location='''cpu'''					)

																								# Rename keys in the transformers names
																								__SCREAMING_SNAKE_CASE:				int           =   list(temp.keys()					)
																								for key in keys:
																														__SCREAMING_SNAKE_CASE:				Any           =   temp.pop(UpperCamelCase__					)

																								if tensors is None:
																														__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   temp
																								else:
																														for key in tensors.keys():
																																				# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																				if any(key.endswith(UpperCamelCase__					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
																																										tensors[key] += temp[key]
																																				else:
																																										# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																										__SCREAMING_SNAKE_CASE:				Any           =   1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN					) else 0
																																										# We concatenate these weights accross TP ranks
																																										__SCREAMING_SNAKE_CASE:				Dict           =   torch.cat([tensors[key], temp[key]]       ,			dim=UpperCamelCase__					)

            # Divide by the number of TP the weights we want to average
																		for key in tensors.keys():
																								if any(key.endswith(UpperCamelCase__					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
																														__SCREAMING_SNAKE_CASE:				Dict           =   tensors[key] / pretraining_tp

																		__SCREAMING_SNAKE_CASE:				Optional[int]           =   model.load_state_dict(UpperCamelCase__       ,			strict=UpperCamelCase__					)
																		assert not other_keys.unexpected_keys, F"""The keys {other_keys.unexpected_keys} are unexpected"""
																		if missing_keys is None:
																								__SCREAMING_SNAKE_CASE:				Tuple           =   set(other_keys.missing_keys					)
																		else:
																								__SCREAMING_SNAKE_CASE:				Any           =   missing_keys.intersection(set(other_keys.missing_keys					)					)

												assert not missing_keys, F"""The keys {missing_keys} are missing"""

												# Save pytorch-model
												os.makedirs(UpperCamelCase__       ,			exist_ok=UpperCamelCase__					)
												__SCREAMING_SNAKE_CASE:				int           =   pytorch_dump_folder_path + '''/''' + WEIGHTS_NAME
												__SCREAMING_SNAKE_CASE:				Tuple           =   pytorch_dump_folder_path + '''/''' + CONFIG_NAME
												print(F"""Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}"""					)
												if config.torch_dtype is not None:
																		__SCREAMING_SNAKE_CASE:				List[str]           =   model.to(config.torch_dtype					)
												torch.save(model.state_dict()       ,			UpperCamelCase__					)
												print(F"""Save configuration file to {pytorch_config_dump_path}"""					)
												with open(UpperCamelCase__       ,			'''w'''       ,			encoding='''utf-8'''					) as f:
																		f.write(config.to_json_string()					)


if __name__ == "__main__":
							lowerCAmelCase							:       Optional[Any]					 =	argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    """--bloom_checkpoint_path""",
							    default=None,
							    type=str,
							    required=True,
							    help="""Path to the Megatron-LM checkpoint path.""",
							)
							parser.add_argument(
							    """--pytorch_dump_folder_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
							)
							parser.add_argument(
							    """--bloom_config_file""",
							    default="""""",
							    type=str,
							    help=(
							        """An optional config json file corresponding to the pre-trained model. \n"""
							        """This specifies the model architecture."""
							    ),
							)
							parser.add_argument(
							    """--shard_model""",
							    action="""store_true""",
							    help="""An optional setting to shard the output model \nThis enables sharding the converted checkpoint""",
							)
							parser.add_argument(
							    """--pretraining_tp""",
							    default=4,
							    type=int,
							    help="""Pretraining TP rank that has been used when training the model in Megatron-LM \n""",
							)
							lowerCAmelCase							:       Any					 =	parser.parse_args()
							convert_bloom_checkpoint_to_pytorch(
							    args.bloom_checkpoint_path,
							    args.bloom_config_file,
							    args.pytorch_dump_folder_path,
							    args.shard_model,
							    args.pretraining_tp,
							)










def 		lowerCAmelCase  (	UpperCamelCase__   :   int       ,			UpperCamelCase__   :   float       ,			UpperCamelCase__   :   float					)      ->    float:

						"""simple docstring"""





						return round(float(moles / volume					) * nfactor					)






def 		lowerCAmelCase  (	UpperCamelCase__   :   float       ,			UpperCamelCase__   :   float       ,			UpperCamelCase__   :   float					)      ->    float:

						"""simple docstring"""





						return round(float((moles * 0.08_21 * temperature) / (volume)					)					)






def 		lowerCAmelCase  (	UpperCamelCase__   :   float       ,			UpperCamelCase__   :   float       ,			UpperCamelCase__   :   float					)      ->    float:

						"""simple docstring"""





						return round(float((moles * 0.08_21 * temperature) / (pressure)					)					)






def 		lowerCAmelCase  (	UpperCamelCase__   :   float       ,			UpperCamelCase__   :   float       ,			UpperCamelCase__   :   float					)      ->    float:

						"""simple docstring"""





						return round(float((pressure * volume) / (0.08_21 * moles)					)					)


if __name__ == "__main__":
							import doctest

							doctest.testmod()



