
'''simple docstring'''







import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

_lowercase			:  Optional[int]   							=	[
    "word_embeddings_layernorm.weight",
    "word_embeddings_layernorm.bias",
    "input_layernorm.weight",
    "input_layernorm.bias",
    "post_attention_layernorm.weight",
    "post_attention_layernorm.bias",
    "self_attention.dense.bias",
    "mlp.dense_4h_to_h.bias",
    "ln_f.weight",
    "ln_f.bias",
]

_lowercase			:  Optional[int]   							=	[
    "mlp.dense_4h_to_h.weight",
    "self_attention.dense.weight",
]
def  snake_case_		(    __SCREAMING_SNAKE_CASE :			Any				,				__SCREAMING_SNAKE_CASE :			List[str]							):



						"""simple docstring"""





						lowercase_     :					List[str]													=	{
						    '''word_embeddings.weight''': '''word_embeddings.weight''',
						    '''word_embeddings.norm.weight''': '''word_embeddings_layernorm.weight''',
						    '''word_embeddings.norm.bias''': '''word_embeddings_layernorm.bias''',
						    '''weight''': '''ln_f.weight''',
						    '''bias''': '''ln_f.bias''',
						}

						if key in layer_rename_map:
												return layer_rename_map[key]

						# Handle transformer blocks
						lowercase_     :					Any													=	int(re.match(R'''.*layer_(\d*).*'''				,				__SCREAMING_SNAKE_CASE							)[1]							)
						layer_number -= 3
						return F'''h.{layer_number}.''' + key
def  snake_case_		(    __SCREAMING_SNAKE_CASE :			str							):



						"""simple docstring"""





						if dtype == torch.bool:
												return 1 / 8
						lowercase_     :					List[Any]													=	re.search(R'''[^\d](\d+)$'''				,				str(__SCREAMING_SNAKE_CASE							)							)
						if bit_search is None:
												raise ValueError(F'''`dtype` is not a valid dtype: {dtype}.'''							)
						lowercase_     :					Tuple													=	int(bit_search.groups()[0]							)
						return bit_size // 8
def  snake_case_		(    __SCREAMING_SNAKE_CASE :			int				,				__SCREAMING_SNAKE_CASE :			Any				,				__SCREAMING_SNAKE_CASE :			Optional[int]				,				__SCREAMING_SNAKE_CASE :			Optional[int]				,				__SCREAMING_SNAKE_CASE :			List[Any]							):



						"""simple docstring"""





						if bloom_config_file == "":
												lowercase_     :					str													=	BloomConfig()
						else:
												lowercase_     :					Optional[int]													=	BloomConfig.from_json_file(__SCREAMING_SNAKE_CASE							)

						if shard_model:
												lowercase_     :					Tuple													=	os.listdir(__SCREAMING_SNAKE_CASE							)
												lowercase_     :					int													=	sorted(filter(lambda __SCREAMING_SNAKE_CASE							: s.startswith('''layer'''							) and "model_00" in s				,				__SCREAMING_SNAKE_CASE							)							)

												lowercase_     :					str													=	{'''weight_map''': {}, '''metadata''': {}}
												lowercase_     :					List[Any]													=	0

												lowercase_     :					str													=	None

												lowercase_     :					Union[str, Any]													=	BloomConfig()

												for j, file in enumerate(__SCREAMING_SNAKE_CASE							):
																		print('''Processing file: {}'''.format(__SCREAMING_SNAKE_CASE							)							)
																		lowercase_     :					Tuple													=	None

																		for i in range(__SCREAMING_SNAKE_CASE							):
																								# load all TP files
																								lowercase_     :					Optional[Any]													=	file.replace('''model_00'''				,				F'''model_0{i}'''							)
																								lowercase_     :					int													=	torch.load(os.path.join(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE							)				,				map_location='''cpu'''							)

																								# Rename keys in the transformers names
																								lowercase_     :					int													=	list(temp.keys()							)
																								for key in keys:
																														lowercase_     :					Dict													=	temp.pop(__SCREAMING_SNAKE_CASE							)

																								if tensors is None:
																														lowercase_     :					Union[str, Any]													=	temp
																								else:
																														for key in tensors.keys():
																																				if any(key.endswith(__SCREAMING_SNAKE_CASE							) for end in WEIGHTS_TO_AVERAGE_ENDSWITH							):
																																										# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																										tensors[key] += temp[key]
																																				else:
																																										# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																										lowercase_     :					Tuple													=	1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN							) else 0
																																										# We concatenate these weights accross TP ranks
																																										lowercase_     :					Union[str, Any]													=	torch.cat([tensors[key], temp[key]]				,				dim=__SCREAMING_SNAKE_CASE							)

            # Divide by the number of TP the weights we want to average
																		for key in tensors.keys():
																								if any(key.endswith(__SCREAMING_SNAKE_CASE							) for end in WEIGHTS_TO_AVERAGE_ENDSWITH							):
																														lowercase_     :					str													=	tensors[key] / pretraining_tp
																		torch.save(
																		    __SCREAMING_SNAKE_CASE				,				os.path.join(
																		        __SCREAMING_SNAKE_CASE				,				'''pytorch_model_{}-of-{}.bin'''.format(str(j + 1							).zfill(5							)				,				str(len(__SCREAMING_SNAKE_CASE							)							).zfill(5							)							)				,				)				,				)

																		for key in tensors.keys():
																								lowercase_     :					int													=	tensors[key]
																								total_size += value.numel() * get_dtype_size(value.dtype							)
																								if key not in index_dict["weight_map"]:
																														lowercase_     :					Any													=	'''pytorch_model_{}-of-{}.bin'''.format(
																														    str(j + 1							).zfill(5							)				,				str(len(__SCREAMING_SNAKE_CASE							)							).zfill(5							)							)

												lowercase_     :					Dict													=	BloomConfig()
												lowercase_     :					str													=	pytorch_dump_folder_path + '''/''' + CONFIG_NAME
												lowercase_     :					Tuple													=	total_size
												with open(__SCREAMING_SNAKE_CASE				,				'''w'''				,				encoding='''utf-8'''							) as f:
																		f.write(config.to_json_string()							)
												with open(os.path.join(__SCREAMING_SNAKE_CASE				,				WEIGHTS_NAME + '''.index.json'''							)				,				'''w'''				,				encoding='''utf-8'''							) as f:
																		lowercase_     :					Any													=	json.dumps(__SCREAMING_SNAKE_CASE				,				indent=2				,				sort_keys=__SCREAMING_SNAKE_CASE							) + '''\n'''
																		f.write(__SCREAMING_SNAKE_CASE							)
						else:
												lowercase_     :					Any													=	BloomModel(__SCREAMING_SNAKE_CASE							)

												lowercase_     :					Dict													=	os.listdir(__SCREAMING_SNAKE_CASE							)
												lowercase_     :					Optional[int]													=	sorted(filter(lambda __SCREAMING_SNAKE_CASE							: s.startswith('''layer'''							) and "model_00" in s				,				__SCREAMING_SNAKE_CASE							)							)

												lowercase_     :					Optional[int]													=	None
												for i, file in enumerate(__SCREAMING_SNAKE_CASE							):
																		lowercase_     :					List[str]													=	None
																		for i in range(__SCREAMING_SNAKE_CASE							):
																								# load all TP files
																								lowercase_     :					Tuple													=	file.replace('''model_00'''				,				F'''model_0{i}'''							)
																								lowercase_     :					Dict													=	torch.load(os.path.join(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE							)				,				map_location='''cpu'''							)

																								# Rename keys in the transformers names
																								lowercase_     :					List[Any]													=	list(temp.keys()							)
																								for key in keys:
																														lowercase_     :					Optional[int]													=	temp.pop(__SCREAMING_SNAKE_CASE							)

																								if tensors is None:
																														lowercase_     :					int													=	temp
																								else:
																														for key in tensors.keys():
																																				# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																				if any(key.endswith(__SCREAMING_SNAKE_CASE							) for end in WEIGHTS_TO_AVERAGE_ENDSWITH							):
																																										tensors[key] += temp[key]
																																				else:
																																										# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																										lowercase_     :					List[str]													=	1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN							) else 0
																																										# We concatenate these weights accross TP ranks
																																										lowercase_     :					List[str]													=	torch.cat([tensors[key], temp[key]]				,				dim=__SCREAMING_SNAKE_CASE							)

            # Divide by the number of TP the weights we want to average
																		for key in tensors.keys():
																								if any(key.endswith(__SCREAMING_SNAKE_CASE							) for end in WEIGHTS_TO_AVERAGE_ENDSWITH							):
																														lowercase_     :					Tuple													=	tensors[key] / pretraining_tp

																		lowercase_     :					Union[str, Any]													=	model.load_state_dict(__SCREAMING_SNAKE_CASE				,				strict=__SCREAMING_SNAKE_CASE							)
																		assert not other_keys.unexpected_keys, F'''The keys {other_keys.unexpected_keys} are unexpected'''
																		if missing_keys is None:
																								lowercase_     :					Any													=	set(other_keys.missing_keys							)
																		else:
																								lowercase_     :					Optional[int]													=	missing_keys.intersection(set(other_keys.missing_keys							)							)

												assert not missing_keys, F'''The keys {missing_keys} are missing'''

												# Save pytorch-model
												os.makedirs(__SCREAMING_SNAKE_CASE				,				exist_ok=__SCREAMING_SNAKE_CASE							)
												lowercase_     :					List[Any]													=	pytorch_dump_folder_path + '''/''' + WEIGHTS_NAME
												lowercase_     :					Any													=	pytorch_dump_folder_path + '''/''' + CONFIG_NAME
												print(F'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''							)
												if config.torch_dtype is not None:
																		lowercase_     :					Union[str, Any]													=	model.to(config.torch_dtype							)
												torch.save(model.state_dict()				,				__SCREAMING_SNAKE_CASE							)
												print(F'''Save configuration file to {pytorch_config_dump_path}'''							)
												with open(__SCREAMING_SNAKE_CASE				,				'''w'''				,				encoding='''utf-8'''							) as f:
																		f.write(config.to_json_string()							)


if __name__ == "__main__":
							_lowercase			:  Union[str, Any]   							=	argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    "--bloom_checkpoint_path",
							    default=None,
							    type=str,
							    required=True,
							    help="Path to the Megatron-LM checkpoint path.",
							)
							parser.add_argument(
							    "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
							)
							parser.add_argument(
							    "--bloom_config_file",
							    default="",
							    type=str,
							    help=(
							        "An optional config json file corresponding to the pre-trained model. \n"
							        "This specifies the model architecture."
							    ),
							)
							parser.add_argument(
							    "--shard_model",
							    action="store_true",
							    help="An optional setting to shard the output model \nThis enables sharding the converted checkpoint",
							)
							parser.add_argument(
							    "--pretraining_tp",
							    default=4,
							    type=int,
							    help="Pretraining TP rank that has been used when training the model in Megatron-LM \n",
							)
							_lowercase			:  Union[str, Any]   							=	parser.parse_args()
							convert_bloom_checkpoint_to_pytorch(
							    args.bloom_checkpoint_path,
							    args.bloom_config_file,
							    args.pytorch_dump_folder_path,
							    args.shard_model,
							    args.pretraining_tp,
							)





'''simple docstring'''







import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional, Union

from .generation.configuration_utils import GenerationConfig
from .training_args import TrainingArguments
from .utils import add_start_docstrings


_lowercase			:  Tuple   							=	logging.getLogger(__name__)






@dataclass
@add_start_docstrings(TrainingArguments.__doc__					)
class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					field(default=lowerCamelCase_   ,						metadata={'''help''': '''Whether to use SortishSampler or not.'''}					)
					lowerCAmelCase_					       =					field(
					    default=lowerCamelCase_   ,						metadata={'''help''': '''Whether to use generate to calculate generative metrics (ROUGE, BLEU).'''}					)
					lowerCAmelCase_					       =					field(
					    default=lowerCamelCase_   ,						metadata={
					        '''help''': (
					            '''The `max_length` to use on each evaluation loop when `predict_with_generate=True`. Will default '''
					            '''to the `max_length` value of the model configuration.'''
					        )
					    }   ,						)
					lowerCAmelCase_					       =					field(
					    default=lowerCamelCase_   ,						metadata={
					        '''help''': (
					            '''The `num_beams` to use on each evaluation loop when `predict_with_generate=True`. Will default '''
					            '''to the `num_beams` value of the model configuration.'''
					        )
					    }   ,						)
					lowerCAmelCase_					       =					field(
					    default=lowerCamelCase_   ,						metadata={
					        '''help''': '''Model id, file path or url pointing to a GenerationConfig json file, to use during prediction.'''
					    }   ,						)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Dict													=	super().to_dict()
											for k, v in d.items():
																	if isinstance(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):
																							lowercase_     :					str													=	v.to_dict()
											return d



