



'''simple docstring'''

import json
import os
import shutil
import warnings
from argparse import ArgumentParser, Namespace
from pathlib import Path
from typing import List

from ..utils import logging
from . import BaseTransformersCLICommand


try:
	from cookiecutter.main import cookiecutter

	a							   =				True
except ImportError:
	a							   =				False

a							   =				logging.get_logger(__name__)  # pylint: disable=invalid-name







def a_     (    __UpperCAmelCase					)				->					int:





			"""simple docstring"""







			return AddNewModelCommand(args.testing				,							args.testing_file				,							path=args.path					)







class 						a_		(     snake_case    ):


							@staticmethod
							def    UpperCamelCase     (				a_   :						ArgumentParser			)			->					Any:
										snake_case:    Tuple      							=parser.add_parser('add-new-model'			)
										add_new_model_parser.add_argument('--testing'      ,						action='store_true'      ,						help='If in testing mode.'			)
										add_new_model_parser.add_argument('--testing_file'      ,						type=a_      ,						help='Configuration file on which to run.'			)
										add_new_model_parser.add_argument(
										    '--path'      ,						type=a_      ,						help='Path to cookiecutter. Should only be used for testing purposes.'			)
										add_new_model_parser.set_defaults(func=a_			)


							def __init__(				self   :						Any      ,						a_   :						bool      ,						a_   :						str      ,						a_   :						int=None      ,						*a_   :						List[str]			)			->					Dict:
										snake_case:    Dict      							=testing
										snake_case:    Union[str, Any]      							=testing_file
										snake_case:    Any      							=path




							def    UpperCamelCase     (				self   :						str			)			->					Optional[int]:
										warnings.warn(
										    'The command `transformers-cli add-new-model` is deprecated and will be removed in v5 of Transformers. '
										    'It is not actively maintained anymore, so might give a result that won\'t pass all tests and quality '
										    'checks, you should use `transformers-cli add-new-model-like` instead.'			)
										if not _has_cookiecutter:
													raise ImportError(
													    'Model creation dependencies are required to use the `add_new_model` command. Install them by running '
													    'the following at the root of your `transformers` clone:\n\n\t$ pip install -e .[modelcreation]\n'			)
										# Ensure that there is no other `cookiecutter-template-xxx` directory in the current working directory
										snake_case:    int      							=[directory for directory in os.listdir() if 'cookiecutter-template-' == directory[:2_2]]
										if len(a_			) > 0:
													raise ValueError(
													    'Several directories starting with `cookiecutter-template-` in current working directory. '
													    'Please clean your directory by removing all folders starting with `cookiecutter-template-` or '
													    'change your working directory.'			)

										snake_case:    Tuple      							=(
										    Path(a_			).parent.parent.parent.parent if self._path is None else Path(self._path			).parent.parent
										)
										snake_case:    Optional[int]      							=path_to_transformer_root / 'templates' / 'adding_a_new_model'

										# Execute cookiecutter
										if not self._testing:
													cookiecutter(str(a_			)			)
										else:
													with open(self._testing_file      ,						'r'			) as configuration_file:
																snake_case:    Dict      							=json.load(a_			)

													cookiecutter(
													    str(path_to_cookiecutter if self._path is None else self._path			)      ,						no_input=a_      ,						extra_context=a_      ,						)

										snake_case:    int      							=[directory for directory in os.listdir() if 'cookiecutter-template-' in directory[:2_2]][0]

										# Retrieve configuration
										with open(directory + '/configuration.json'      ,						'r'			) as configuration_file:
													snake_case:    Dict      							=json.load(a_			)

										snake_case:    Tuple      							=configuration['lowercase_modelname']
										snake_case:    Union[str, Any]      							=configuration['generate_tensorflow_pytorch_and_flax']
										os.remove(F'''{directory}/configuration.json'''			)

										snake_case:    Tuple      							='PyTorch' in generate_tensorflow_pytorch_and_flax
										snake_case:    Any      							='TensorFlow' in generate_tensorflow_pytorch_and_flax
										snake_case:    Union[str, Any]      							='Flax' in generate_tensorflow_pytorch_and_flax

										snake_case:    Optional[Any]      							=F'''{path_to_transformer_root}/src/transformers/models/{lowercase_model_name}'''
										os.makedirs(a_      ,						exist_ok=a_			)
										os.makedirs(F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}'''      ,						exist_ok=a_			)

										# Tests require submodules as they have parent imports
										with open(F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/__init__.py'''      ,						'w'			):
													pass

										shutil.move(
										    F'''{directory}/__init__.py'''      ,						F'''{model_dir}/__init__.py'''      ,						)
										shutil.move(
										    F'''{directory}/configuration_{lowercase_model_name}.py'''      ,						F'''{model_dir}/configuration_{lowercase_model_name}.py'''      ,						)

										def remove_copy_lines(a_   :						Union[str, Any]			):
													with open(a_      ,						'r'			) as f:
																snake_case:    Union[str, Any]      							=f.readlines()
													with open(a_      ,						'w'			) as f:
																for line in lines:
																			if "# Copied from transformers." not in line:
																						f.write(a_			)

										if output_pytorch:
													if not self._testing:
																remove_copy_lines(F'''{directory}/modeling_{lowercase_model_name}.py'''			)

													shutil.move(
													    F'''{directory}/modeling_{lowercase_model_name}.py'''      ,						F'''{model_dir}/modeling_{lowercase_model_name}.py'''      ,						)

													shutil.move(
													    F'''{directory}/test_modeling_{lowercase_model_name}.py'''      ,						F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_{lowercase_model_name}.py'''      ,						)
										else:
													os.remove(F'''{directory}/modeling_{lowercase_model_name}.py'''			)
													os.remove(F'''{directory}/test_modeling_{lowercase_model_name}.py'''			)

										if output_tensorflow:
													if not self._testing:
																remove_copy_lines(F'''{directory}/modeling_tf_{lowercase_model_name}.py'''			)

													shutil.move(
													    F'''{directory}/modeling_tf_{lowercase_model_name}.py'''      ,						F'''{model_dir}/modeling_tf_{lowercase_model_name}.py'''      ,						)

													shutil.move(
													    F'''{directory}/test_modeling_tf_{lowercase_model_name}.py'''      ,						F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_tf_{lowercase_model_name}.py'''      ,						)
										else:
													os.remove(F'''{directory}/modeling_tf_{lowercase_model_name}.py'''			)
													os.remove(F'''{directory}/test_modeling_tf_{lowercase_model_name}.py'''			)

										if output_flax:
													if not self._testing:
																remove_copy_lines(F'''{directory}/modeling_flax_{lowercase_model_name}.py'''			)

													shutil.move(
													    F'''{directory}/modeling_flax_{lowercase_model_name}.py'''      ,						F'''{model_dir}/modeling_flax_{lowercase_model_name}.py'''      ,						)

													shutil.move(
													    F'''{directory}/test_modeling_flax_{lowercase_model_name}.py'''      ,						F'''{path_to_transformer_root}/tests/models/{lowercase_model_name}/test_modeling_flax_{lowercase_model_name}.py'''      ,						)
										else:
													os.remove(F'''{directory}/modeling_flax_{lowercase_model_name}.py'''			)
													os.remove(F'''{directory}/test_modeling_flax_{lowercase_model_name}.py'''			)

										shutil.move(
										    F'''{directory}/{lowercase_model_name}.md'''      ,						F'''{path_to_transformer_root}/docs/source/en/model_doc/{lowercase_model_name}.md'''      ,						)

										shutil.move(
										    F'''{directory}/tokenization_{lowercase_model_name}.py'''      ,						F'''{model_dir}/tokenization_{lowercase_model_name}.py'''      ,						)

										shutil.move(
										    F'''{directory}/tokenization_fast_{lowercase_model_name}.py'''      ,						F'''{model_dir}/tokenization_{lowercase_model_name}_fast.py'''      ,						)

										from os import fdopen, remove
										from shutil import copymode, move
										from tempfile import mkstemp

										def replace(a_   :						str      ,						a_   :						str      ,						a_   :						List[str]			):
													# Create temp file
													snake_case							,  snake_case:    Union[str, Any]      							=mkstemp()
													snake_case:    str      							=False
													with fdopen(a_      ,						'w'			) as new_file:
																with open(a_			) as old_file:
																			for line in old_file:
																						new_file.write(a_			)
																						if line_to_copy_below in line:
																									snake_case:    int      							=True
																									for line_to_copy in lines_to_copy:
																												new_file.write(a_			)

													if not line_found:
																raise ValueError(F'''Line {line_to_copy_below} was not found in file.'''			)

													# Copy the file permissions from the old file to the new file
													copymode(a_      ,						a_			)
													# Remove original file
													remove(a_			)
													# Move new file
													move(a_      ,						a_			)

										def skip_units(a_   :						List[str]			):
													return (
													    ("generating PyTorch" in line and not output_pytorch)
													    or ("generating TensorFlow" in line and not output_tensorflow)
													    or ("generating Flax" in line and not output_flax)
													)

										def replace_in_files(a_   :						Any			):
													with open(a_			) as datafile:
																snake_case:    List[str]      							=[]
																snake_case:    Optional[int]      							=False
																snake_case:    Optional[int]      							=False
																for line in datafile:
																			if "# To replace in: " in line and "##" not in line:
																						snake_case:    Optional[Any]      							=line.split('"'			)[1]
																						snake_case:    int      							=skip_units(a_			)
																			elif "# Below: " in line and "##" not in line:
																						snake_case:    Optional[Any]      							=line.split('"'			)[1]
																						snake_case:    Any      							=skip_units(a_			)
																			elif "# End." in line and "##" not in line:
																						if not skip_file and not skip_snippet:
																									replace(a_      ,						a_      ,						a_			)

																						snake_case:    Tuple      							=[]
																			elif "# Replace with" in line and "##" not in line:
																						snake_case:    Any      							=[]
																			elif "##" not in line:
																						lines_to_copy.append(a_			)

													remove(a_			)

										replace_in_files(F'''{directory}/to_replace_{lowercase_model_name}.py'''			)
										os.rmdir(a_			)









'''simple docstring'''

import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

a							   =				[
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

a							   =				[
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]







def a_     (    __UpperCAmelCase				,							__UpperCAmelCase					)				->					Any:





			"""simple docstring"""







			snake_case:    Any      							={
			    'word_embeddings.weight': 'word_embeddings.weight',
			    'word_embeddings.norm.weight': 'word_embeddings_layernorm.weight',
			    'word_embeddings.norm.bias': 'word_embeddings_layernorm.bias',
			    'weight': 'ln_f.weight',
			    'bias': 'ln_f.bias',
			}

			if key in layer_rename_map:
						return layer_rename_map[key]

			# Handle transformer blocks
			snake_case:    Dict      							=int(re.match(R'.*layer_(\d*).*'				,							__UpperCAmelCase					)[1]					)
			layer_number -= 3
			return f'''h.{layer_number}.''' + key







def a_     (    __UpperCAmelCase					)				->					Union[str, Any]:





			"""simple docstring"""







			if dtype == torch.bool:
						return 1 / 8
			snake_case:    Optional[Any]      							=re.search(R'[^\d](\d+)$'				,							str(__UpperCAmelCase					)					)
			if bit_search is None:
						raise ValueError(f'''`dtype` is not a valid dtype: {dtype}.'''					)
			snake_case:    str      							=int(bit_search.groups()[0]					)
			return bit_size // 8







def a_     (    __UpperCAmelCase				,							__UpperCAmelCase				,							__UpperCAmelCase				,							__UpperCAmelCase				,							__UpperCAmelCase					)				->					Tuple:





			"""simple docstring"""







			if bloom_config_file == "":
						snake_case:    Tuple      							=BloomConfig()
			else:
						snake_case:    List[Any]      							=BloomConfig.from_json_file(__UpperCAmelCase					)

			if shard_model:
						snake_case:    Any      							=os.listdir(__UpperCAmelCase					)
						snake_case:    Any      							=sorted(filter(lambda __UpperCAmelCase					: s.startswith('layer'					) and "model_00" in s				,							__UpperCAmelCase					)					)

						snake_case:    int      							={'weight_map': {}, 'metadata': {}}
						snake_case:    Dict      							=0

						snake_case:    List[str]      							=None

						snake_case:    Any      							=BloomConfig()

						for j, file in enumerate(__UpperCAmelCase					):
									print('Processing file: {}'.format(__UpperCAmelCase					)					)
									snake_case:    Any      							=None

									for i in range(__UpperCAmelCase					):
												# load all TP files
												snake_case:    Optional[int]      							=file.replace('model_00'				,							f'''model_0{i}'''					)
												snake_case:    List[str]      							=torch.load(os.path.join(__UpperCAmelCase				,							__UpperCAmelCase					)				,							map_location='cpu'					)

												# Rename keys in the transformers names
												snake_case:    int      							=list(temp.keys()					)
												for key in keys:
															snake_case:    List[Any]      							=temp.pop(__UpperCAmelCase					)

												if tensors is None:
															snake_case:    Tuple      							=temp
												else:
															for key in tensors.keys():
																		if any(key.endswith(__UpperCAmelCase					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
																					# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																					tensors[key] += temp[key]
																		else:
																					# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																					snake_case:    Optional[int]      							=1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN					) else 0
																					# We concatenate these weights accross TP ranks
																					snake_case:    str      							=torch.cat([tensors[key], temp[key]]				,							dim=__UpperCAmelCase					)

            # Divide by the number of TP the weights we want to average
									for key in tensors.keys():
												if any(key.endswith(__UpperCAmelCase					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
															snake_case:    Dict      							=tensors[key] / pretraining_tp
									torch.save(
									    __UpperCAmelCase				,							os.path.join(
									        __UpperCAmelCase				,							'pytorch_model_{}-of-{}.bin'.format(str(j + 1					).zfill(5					)				,							str(len(__UpperCAmelCase					)					).zfill(5					)					)				,							)				,							)

									for key in tensors.keys():
												snake_case:    List[str]      							=tensors[key]
												total_size += value.numel() * get_dtype_size(value.dtype					)
												if key not in index_dict["weight_map"]:
															snake_case:    List[Any]      							='pytorch_model_{}-of-{}.bin'.format(
															    str(j + 1					).zfill(5					)				,							str(len(__UpperCAmelCase					)					).zfill(5					)					)

						snake_case:    Tuple      							=BloomConfig()
						snake_case:    Union[str, Any]      							=pytorch_dump_folder_path + '/' + CONFIG_NAME
						snake_case:    Any      							=total_size
						with open(__UpperCAmelCase				,							'w'				,							encoding='utf-8'					) as f:
									f.write(config.to_json_string()					)
						with open(os.path.join(__UpperCAmelCase				,							WEIGHTS_NAME + '.index.json'					)				,							'w'				,							encoding='utf-8'					) as f:
									snake_case:    Tuple      							=json.dumps(__UpperCAmelCase				,							indent=2				,							sort_keys=__UpperCAmelCase					) + '\n'
									f.write(__UpperCAmelCase					)
			else:
						snake_case:    Optional[Any]      							=BloomModel(__UpperCAmelCase					)

						snake_case:    Any      							=os.listdir(__UpperCAmelCase					)
						snake_case:    Optional[int]      							=sorted(filter(lambda __UpperCAmelCase					: s.startswith('layer'					) and "model_00" in s				,							__UpperCAmelCase					)					)

						snake_case:    str      							=None
						for i, file in enumerate(__UpperCAmelCase					):
									snake_case:    List[str]      							=None
									for i in range(__UpperCAmelCase					):
												# load all TP files
												snake_case:    int      							=file.replace('model_00'				,							f'''model_0{i}'''					)
												snake_case:    Optional[Any]      							=torch.load(os.path.join(__UpperCAmelCase				,							__UpperCAmelCase					)				,							map_location='cpu'					)

												# Rename keys in the transformers names
												snake_case:    List[str]      							=list(temp.keys()					)
												for key in keys:
															snake_case:    int      							=temp.pop(__UpperCAmelCase					)

												if tensors is None:
															snake_case:    Any      							=temp
												else:
															for key in tensors.keys():
																		# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																		if any(key.endswith(__UpperCAmelCase					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
																					tensors[key] += temp[key]
																		else:
																					# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																					snake_case:    List[Any]      							=1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN					) else 0
																					# We concatenate these weights accross TP ranks
																					snake_case:    Tuple      							=torch.cat([tensors[key], temp[key]]				,							dim=__UpperCAmelCase					)

            # Divide by the number of TP the weights we want to average
									for key in tensors.keys():
												if any(key.endswith(__UpperCAmelCase					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
															snake_case:    int      							=tensors[key] / pretraining_tp

									snake_case:    Optional[Any]      							=model.load_state_dict(__UpperCAmelCase				,							strict=__UpperCAmelCase					)
									assert not other_keys.unexpected_keys, f'''The keys {other_keys.unexpected_keys} are unexpected'''
									if missing_keys is None:
												snake_case:    Optional[Any]      							=set(other_keys.missing_keys					)
									else:
												snake_case:    Any      							=missing_keys.intersection(set(other_keys.missing_keys					)					)

						assert not missing_keys, f'''The keys {missing_keys} are missing'''

						# Save pytorch-model
						os.makedirs(__UpperCAmelCase				,							exist_ok=__UpperCAmelCase					)
						snake_case:    Optional[int]      							=pytorch_dump_folder_path + '/' + WEIGHTS_NAME
						snake_case:    Any      							=pytorch_dump_folder_path + '/' + CONFIG_NAME
						print(f'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''					)
						if config.torch_dtype is not None:
									snake_case:    Optional[Any]      							=model.to(config.torch_dtype					)
						torch.save(model.state_dict()				,							__UpperCAmelCase					)
						print(f'''Save configuration file to {pytorch_config_dump_path}'''					)
						with open(__UpperCAmelCase				,							'w'				,							encoding='utf-8'					) as f:
									f.write(config.to_json_string()					)


if __name__ == "__main__":
	a							   =				argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '--bloom_checkpoint_path',
	    default=None,
	    type=str,
	    required=True,
	    help='Path to the Megatron-LM checkpoint path.',
	)
	parser.add_argument(
	    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
	)
	parser.add_argument(
	    '--bloom_config_file',
	    default='',
	    type=str,
	    help=(
	        'An optional config json file corresponding to the pre-trained model. \n'
	        'This specifies the model architecture.'
	    ),
	)
	parser.add_argument(
	    '--shard_model',
	    action='store_true',
	    help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
	)
	parser.add_argument(
	    '--pretraining_tp',
	    default=4,
	    type=int,
	    help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
	)
	a							   =				parser.parse_args()
	convert_bloom_checkpoint_to_pytorch(
	    args.bloom_checkpoint_path,
	    args.bloom_config_file,
	    args.pytorch_dump_folder_path,
	    args.shard_model,
	    args.pretraining_tp,
	)




