



'''simple docstring'''

import asyncio
import os
import shutil
import subprocess
import sys
import tempfile
import unittest
from distutils.util import strtobool
from functools import partial
from pathlib import Path
from typing import List, Union
from unittest import mock

import torch

from ..state import AcceleratorState, PartialState
from ..utils import (
    gather,
    is_bnb_available,
    is_comet_ml_available,
    is_datasets_available,
    is_deepspeed_available,
    is_mps_available,
    is_safetensors_available,
    is_tensorboard_available,
    is_torch_version,
    is_tpu_available,
    is_transformers_available,
    is_wandb_available,
    is_xpu_available,
)







def a_     (    __UpperCAmelCase				,							__UpperCAmelCase=False					)				->					int:





			"""simple docstring"""







			try:
						snake_case:    List[Any]      							=os.environ[key]
			except KeyError:
						# KEY isn't set, default to `default`.
						snake_case:    Union[str, Any]      							=default
			else:
						# KEY is set, convert it to True or False.
						try:
									snake_case:    str      							=strtobool(__UpperCAmelCase					)
						except ValueError:
									# More values are supported, but let's keep the message simple.
									raise ValueError(f'''If set, {key} must be yes or no.'''					)
			return _value


a							   =				parse_flag_from_env('RUN_SLOW', default=False)







def a_     (    __UpperCAmelCase					)				->					int:





			"""simple docstring"""







			return unittest.skip('Test was skipped'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					Dict:





			"""simple docstring"""







			return unittest.skipUnless(_run_slow_tests				,							'test is slow'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					int:





			"""simple docstring"""







			return unittest.skipUnless(not torch.cuda.is_available()				,							'test requires only a CPU'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					List[str]:





			"""simple docstring"""







			return unittest.skipUnless(torch.cuda.is_available()				,							'test requires a GPU'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					List[Any]:





			"""simple docstring"""







			return unittest.skipUnless(is_xpu_available()				,							'test requires a XPU'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					str:





			"""simple docstring"""







			return unittest.skipUnless(is_mps_available()				,							'test requires a `mps` backend support in `torch`'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					int:





			"""simple docstring"""







			return unittest.skipUnless(
			    is_transformers_available() and is_datasets_available()				,							'test requires the Hugging Face suite'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					str:





			"""simple docstring"""







			return unittest.skipUnless(is_bnb_available()				,							'test requires the bitsandbytes library'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					str:





			"""simple docstring"""







			return unittest.skipUnless(is_tpu_available()				,							'test requires TPU'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					int:





			"""simple docstring"""







			return unittest.skipUnless(torch.cuda.device_count() == 1				,							'test requires a GPU'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					List[Any]:





			"""simple docstring"""







			return unittest.skipUnless(torch.xpu.device_count() == 1				,							'test requires a XPU'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					str:





			"""simple docstring"""







			return unittest.skipUnless(torch.cuda.device_count() > 1				,							'test requires multiple GPUs'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					Optional[int]:





			"""simple docstring"""







			return unittest.skipUnless(torch.xpu.device_count() > 1				,							'test requires multiple XPUs'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					Any:





			"""simple docstring"""







			return unittest.skipUnless(is_safetensors_available()				,							'test requires safetensors'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					Dict:





			"""simple docstring"""







			return unittest.skipUnless(is_deepspeed_available()				,							'test requires DeepSpeed'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					int:





			"""simple docstring"""







			return unittest.skipUnless(is_torch_version('>='				,							'1.12.0'					)				,							'test requires torch version >= 1.12.0'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase=None				,							__UpperCAmelCase=None					)				->					Optional[int]:





			"""simple docstring"""







			if test_case is None:
						return partial(__UpperCAmelCase				,							version=__UpperCAmelCase					)
			return unittest.skipUnless(is_torch_version('>='				,							__UpperCAmelCase					)				,							f'''test requires torch version >= {version}'''					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					int:





			"""simple docstring"""







			return unittest.skipUnless(is_tensorboard_available()				,							'test requires Tensorboard'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					Optional[Any]:





			"""simple docstring"""







			return unittest.skipUnless(is_wandb_available()				,							'test requires wandb'					)(__UpperCAmelCase					)







def a_     (    __UpperCAmelCase					)				->					Tuple:





			"""simple docstring"""







			return unittest.skipUnless(is_comet_ml_available()				,							'test requires comet_ml'					)(__UpperCAmelCase					)


a							   =				(
    any([is_wandb_available(), is_tensorboard_available()]) and not is_comet_ml_available()
)







def a_     (    __UpperCAmelCase					)				->					Optional[int]:





			"""simple docstring"""







			return unittest.skipUnless(
			    _atleast_one_tracker_available				,							'test requires at least one tracker to be available and for `comet_ml` to not be installed'				,							)(__UpperCAmelCase					)






class 						a_		(     unittest.TestCase    ):
							UpperCAmelCase			:		List[Any]        =				True


							@classmethod
							def    UpperCamelCase     (				cls   :						Optional[int]			)			->					Tuple:
										snake_case:    Optional[Any]      							=tempfile.mkdtemp()


							@classmethod
							def    UpperCamelCase     (				cls   :						Optional[int]			)			->					int:
										if os.path.exists(cls.tmpdir			):
													shutil.rmtree(cls.tmpdir			)




							def    UpperCamelCase     (				self   :						Optional[Any]			)			->					Optional[Any]:
										if self.clear_on_setup:
													for path in Path(self.tmpdir			).glob('**/*'			):
																if path.is_file():
																			path.unlink()
																elif path.is_dir():
																			shutil.rmtree(a_			)






class 						a_		(     unittest.TestCase    ):


							def    UpperCamelCase     (				self   :						Optional[int]			)			->					str:
										super().tearDown()
										# Reset the state of the AcceleratorState singleton.
										AcceleratorState._reset_state()
										PartialState._reset_state()






class 						a_		(     unittest.TestCase    ):


							def    UpperCamelCase     (				self   :						str      ,						a_   :						Union[mock.Mock, List[mock.Mock]]			)			->					Union[str, Any]:
										snake_case:    Any      							=mocks if isinstance(a_      ,						(tuple, list)			) else [mocks]
										for m in self.mocks:
													m.start()
													self.addCleanup(m.stop			)







def a_     (    __UpperCAmelCase					)				->					Any:





			"""simple docstring"""







			snake_case:    int      							=AcceleratorState()
			snake_case:    List[Any]      							=tensor[None].clone().to(state.device					)
			snake_case:    int      							=gather(__UpperCAmelCase					).cpu()
			snake_case:    Optional[int]      							=tensor[0].cpu()
			for i in range(tensors.shape[0]					):
						if not torch.equal(tensors[i]				,							__UpperCAmelCase					):
									return False
			return True






class 						a_		:


							def __init__(				self   :						List[Any]      ,						a_   :						List[Any]      ,						a_   :						List[str]      ,						a_   :						List[str]			)			->					str:
										snake_case:    Optional[int]      							=returncode
										snake_case:    Union[str, Any]      							=stdout
										snake_case:    Optional[int]      							=stderr







async def a_     (    __UpperCAmelCase				,							__UpperCAmelCase					)				->					str:





			"""simple docstring"""







			while True:
						snake_case:    Optional[Any]      							=await stream.readline()
						if line:
									callback(__UpperCAmelCase					)
						else:
									break







async def a_     (    __UpperCAmelCase				,							__UpperCAmelCase=None				,							__UpperCAmelCase=None				,							__UpperCAmelCase=None				,							__UpperCAmelCase=False				,							__UpperCAmelCase=False					)				->					_RunOutput:





			"""simple docstring"""







			if echo:
						print('\nRunning: '				,							' '.join(__UpperCAmelCase					)					)

			snake_case:    str      							=await asyncio.create_subprocess_exec(
			    cmd[0]				,							*cmd[1:]				,							stdin=__UpperCAmelCase				,							stdout=asyncio.subprocess.PIPE				,							stderr=asyncio.subprocess.PIPE				,							env=__UpperCAmelCase				,							)

			# note: there is a warning for a possible deadlock when using `wait` with huge amounts of data in the pipe
			# https://docs.python.org/3/library/asyncio-subprocess.html#asyncio.asyncio.subprocess.Process.wait
			#
			# If it starts hanging, will need to switch to the following code. The problem is that no data
			# will be seen until it's done and if it hangs for example there will be no debug info.
			# out, err = await p.communicate()
			# return _RunOutput(p.returncode, out, err)

			snake_case:    Union[str, Any]      							=[]
			snake_case:    str      							=[]

			def tee(__UpperCAmelCase				,							__UpperCAmelCase				,							__UpperCAmelCase				,							__UpperCAmelCase=""					):
						snake_case:    List[str]      							=line.decode('utf-8'					).rstrip()
						sink.append(__UpperCAmelCase					)
						if not quiet:
									print(__UpperCAmelCase				,							__UpperCAmelCase				,							file=__UpperCAmelCase					)

    # XXX: the timeout doesn't seem to make any difference here
			await asyncio.wait(
			    [
			        asyncio.create_task(_read_stream(p.stdout				,							lambda __UpperCAmelCase					: tee(__UpperCAmelCase				,							__UpperCAmelCase				,							sys.stdout				,							label='stdout:'					)					)					),
			        asyncio.create_task(_read_stream(p.stderr				,							lambda __UpperCAmelCase					: tee(__UpperCAmelCase				,							__UpperCAmelCase				,							sys.stderr				,							label='stderr:'					)					)					),
			    ]				,							timeout=__UpperCAmelCase				,							)
			return _RunOutput(await p.wait()				,							__UpperCAmelCase				,							__UpperCAmelCase					)







def a_     (    __UpperCAmelCase				,							__UpperCAmelCase=None				,							__UpperCAmelCase=None				,							__UpperCAmelCase=1_80				,							__UpperCAmelCase=False				,							__UpperCAmelCase=True					)				->					_RunOutput:





			"""simple docstring"""







			snake_case:    List[Any]      							=asyncio.get_event_loop()
			snake_case:    Optional[int]      							=loop.run_until_complete(
			    _stream_subprocess(__UpperCAmelCase				,							env=__UpperCAmelCase				,							stdin=__UpperCAmelCase				,							timeout=__UpperCAmelCase				,							quiet=__UpperCAmelCase				,							echo=__UpperCAmelCase					)					)

			snake_case:    Optional[int]      							=' '.join(__UpperCAmelCase					)
			if result.returncode > 0:
						snake_case:    str      							='\n'.join(result.stderr					)
						raise RuntimeError(
						    f'''\'{cmd_str}\' failed with returncode {result.returncode}\n\n'''
						    f'''The combined stderr from workers follows:\n{stderr}'''					)

			return result






class 						a_		(     snake_case    ):
							pass







def a_     (    __UpperCAmelCase				,							__UpperCAmelCase=False					)				->					Union[str, Any]:





			"""simple docstring"""







			try:
						snake_case:    Optional[Any]      							=subprocess.check_output(__UpperCAmelCase				,							stderr=subprocess.STDOUT					)
						if return_stdout:
									if hasattr(__UpperCAmelCase				,							'decode'					):
												snake_case:    Union[str, Any]      							=output.decode('utf-8'					)
									return output
			except subprocess.CalledProcessError as e:
						raise SubprocessCallException(
						    f'''Command `{" ".join(__UpperCAmelCase					)}` failed with the following error:\n\n{e.output.decode()}'''					) from e









'''simple docstring'''

import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

a							   =				[
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

a							   =				[
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]







def a_     (    __UpperCAmelCase				,							__UpperCAmelCase					)				->					Any:





			"""simple docstring"""







			snake_case:    Any      							={
			    'word_embeddings.weight': 'word_embeddings.weight',
			    'word_embeddings.norm.weight': 'word_embeddings_layernorm.weight',
			    'word_embeddings.norm.bias': 'word_embeddings_layernorm.bias',
			    'weight': 'ln_f.weight',
			    'bias': 'ln_f.bias',
			}

			if key in layer_rename_map:
						return layer_rename_map[key]

			# Handle transformer blocks
			snake_case:    Dict      							=int(re.match(R'.*layer_(\d*).*'				,							__UpperCAmelCase					)[1]					)
			layer_number -= 3
			return f'''h.{layer_number}.''' + key







def a_     (    __UpperCAmelCase					)				->					Union[str, Any]:





			"""simple docstring"""







			if dtype == torch.bool:
						return 1 / 8
			snake_case:    Optional[Any]      							=re.search(R'[^\d](\d+)$'				,							str(__UpperCAmelCase					)					)
			if bit_search is None:
						raise ValueError(f'''`dtype` is not a valid dtype: {dtype}.'''					)
			snake_case:    str      							=int(bit_search.groups()[0]					)
			return bit_size // 8







def a_     (    __UpperCAmelCase				,							__UpperCAmelCase				,							__UpperCAmelCase				,							__UpperCAmelCase				,							__UpperCAmelCase					)				->					Tuple:





			"""simple docstring"""







			if bloom_config_file == "":
						snake_case:    Tuple      							=BloomConfig()
			else:
						snake_case:    List[Any]      							=BloomConfig.from_json_file(__UpperCAmelCase					)

			if shard_model:
						snake_case:    Any      							=os.listdir(__UpperCAmelCase					)
						snake_case:    Any      							=sorted(filter(lambda __UpperCAmelCase					: s.startswith('layer'					) and "model_00" in s				,							__UpperCAmelCase					)					)

						snake_case:    int      							={'weight_map': {}, 'metadata': {}}
						snake_case:    Dict      							=0

						snake_case:    List[str]      							=None

						snake_case:    Any      							=BloomConfig()

						for j, file in enumerate(__UpperCAmelCase					):
									print('Processing file: {}'.format(__UpperCAmelCase					)					)
									snake_case:    Any      							=None

									for i in range(__UpperCAmelCase					):
												# load all TP files
												snake_case:    Optional[int]      							=file.replace('model_00'				,							f'''model_0{i}'''					)
												snake_case:    List[str]      							=torch.load(os.path.join(__UpperCAmelCase				,							__UpperCAmelCase					)				,							map_location='cpu'					)

												# Rename keys in the transformers names
												snake_case:    int      							=list(temp.keys()					)
												for key in keys:
															snake_case:    List[Any]      							=temp.pop(__UpperCAmelCase					)

												if tensors is None:
															snake_case:    Tuple      							=temp
												else:
															for key in tensors.keys():
																		if any(key.endswith(__UpperCAmelCase					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
																					# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																					tensors[key] += temp[key]
																		else:
																					# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																					snake_case:    Optional[int]      							=1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN					) else 0
																					# We concatenate these weights accross TP ranks
																					snake_case:    str      							=torch.cat([tensors[key], temp[key]]				,							dim=__UpperCAmelCase					)

            # Divide by the number of TP the weights we want to average
									for key in tensors.keys():
												if any(key.endswith(__UpperCAmelCase					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
															snake_case:    Dict      							=tensors[key] / pretraining_tp
									torch.save(
									    __UpperCAmelCase				,							os.path.join(
									        __UpperCAmelCase				,							'pytorch_model_{}-of-{}.bin'.format(str(j + 1					).zfill(5					)				,							str(len(__UpperCAmelCase					)					).zfill(5					)					)				,							)				,							)

									for key in tensors.keys():
												snake_case:    List[str]      							=tensors[key]
												total_size += value.numel() * get_dtype_size(value.dtype					)
												if key not in index_dict["weight_map"]:
															snake_case:    List[Any]      							='pytorch_model_{}-of-{}.bin'.format(
															    str(j + 1					).zfill(5					)				,							str(len(__UpperCAmelCase					)					).zfill(5					)					)

						snake_case:    Tuple      							=BloomConfig()
						snake_case:    Union[str, Any]      							=pytorch_dump_folder_path + '/' + CONFIG_NAME
						snake_case:    Any      							=total_size
						with open(__UpperCAmelCase				,							'w'				,							encoding='utf-8'					) as f:
									f.write(config.to_json_string()					)
						with open(os.path.join(__UpperCAmelCase				,							WEIGHTS_NAME + '.index.json'					)				,							'w'				,							encoding='utf-8'					) as f:
									snake_case:    Tuple      							=json.dumps(__UpperCAmelCase				,							indent=2				,							sort_keys=__UpperCAmelCase					) + '\n'
									f.write(__UpperCAmelCase					)
			else:
						snake_case:    Optional[Any]      							=BloomModel(__UpperCAmelCase					)

						snake_case:    Any      							=os.listdir(__UpperCAmelCase					)
						snake_case:    Optional[int]      							=sorted(filter(lambda __UpperCAmelCase					: s.startswith('layer'					) and "model_00" in s				,							__UpperCAmelCase					)					)

						snake_case:    str      							=None
						for i, file in enumerate(__UpperCAmelCase					):
									snake_case:    List[str]      							=None
									for i in range(__UpperCAmelCase					):
												# load all TP files
												snake_case:    int      							=file.replace('model_00'				,							f'''model_0{i}'''					)
												snake_case:    Optional[Any]      							=torch.load(os.path.join(__UpperCAmelCase				,							__UpperCAmelCase					)				,							map_location='cpu'					)

												# Rename keys in the transformers names
												snake_case:    List[str]      							=list(temp.keys()					)
												for key in keys:
															snake_case:    int      							=temp.pop(__UpperCAmelCase					)

												if tensors is None:
															snake_case:    Any      							=temp
												else:
															for key in tensors.keys():
																		# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																		if any(key.endswith(__UpperCAmelCase					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
																					tensors[key] += temp[key]
																		else:
																					# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																					snake_case:    List[Any]      							=1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN					) else 0
																					# We concatenate these weights accross TP ranks
																					snake_case:    Tuple      							=torch.cat([tensors[key], temp[key]]				,							dim=__UpperCAmelCase					)

            # Divide by the number of TP the weights we want to average
									for key in tensors.keys():
												if any(key.endswith(__UpperCAmelCase					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
															snake_case:    int      							=tensors[key] / pretraining_tp

									snake_case:    Optional[Any]      							=model.load_state_dict(__UpperCAmelCase				,							strict=__UpperCAmelCase					)
									assert not other_keys.unexpected_keys, f'''The keys {other_keys.unexpected_keys} are unexpected'''
									if missing_keys is None:
												snake_case:    Optional[Any]      							=set(other_keys.missing_keys					)
									else:
												snake_case:    Any      							=missing_keys.intersection(set(other_keys.missing_keys					)					)

						assert not missing_keys, f'''The keys {missing_keys} are missing'''

						# Save pytorch-model
						os.makedirs(__UpperCAmelCase				,							exist_ok=__UpperCAmelCase					)
						snake_case:    Optional[int]      							=pytorch_dump_folder_path + '/' + WEIGHTS_NAME
						snake_case:    Any      							=pytorch_dump_folder_path + '/' + CONFIG_NAME
						print(f'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''					)
						if config.torch_dtype is not None:
									snake_case:    Optional[Any]      							=model.to(config.torch_dtype					)
						torch.save(model.state_dict()				,							__UpperCAmelCase					)
						print(f'''Save configuration file to {pytorch_config_dump_path}'''					)
						with open(__UpperCAmelCase				,							'w'				,							encoding='utf-8'					) as f:
									f.write(config.to_json_string()					)


if __name__ == "__main__":
	a							   =				argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '--bloom_checkpoint_path',
	    default=None,
	    type=str,
	    required=True,
	    help='Path to the Megatron-LM checkpoint path.',
	)
	parser.add_argument(
	    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
	)
	parser.add_argument(
	    '--bloom_config_file',
	    default='',
	    type=str,
	    help=(
	        'An optional config json file corresponding to the pre-trained model. \n'
	        'This specifies the model architecture.'
	    ),
	)
	parser.add_argument(
	    '--shard_model',
	    action='store_true',
	    help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
	)
	parser.add_argument(
	    '--pretraining_tp',
	    default=4,
	    type=int,
	    help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
	)
	a							   =				parser.parse_args()
	convert_bloom_checkpoint_to_pytorch(
	    args.bloom_checkpoint_path,
	    args.bloom_config_file,
	    args.pytorch_dump_folder_path,
	    args.shard_model,
	    args.pretraining_tp,
	)




