





import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

lowerCAmelCase_    =    [
    '''word_embeddings_layernorm.weight''',
    '''word_embeddings_layernorm.bias''',
    '''input_layernorm.weight''',
    '''input_layernorm.bias''',
    '''post_attention_layernorm.weight''',
    '''post_attention_layernorm.bias''',
    '''self_attention.dense.bias''',
    '''mlp.dense_4h_to_h.bias''',
    '''ln_f.weight''',
    '''ln_f.bias''',
]

lowerCAmelCase_    =    [
    '''mlp.dense_4h_to_h.weight''',
    '''self_attention.dense.weight''',
]






def 					lowerCamelCase_						(    _UpperCamelCase			,    _UpperCamelCase			)		->						Optional[int]:






				"""simple docstring"""


				snake_case_						:   List[str]           =  {
				    '''word_embeddings.weight''': '''word_embeddings.weight''',
				    '''word_embeddings.norm.weight''': '''word_embeddings_layernorm.weight''',
				    '''word_embeddings.norm.bias''': '''word_embeddings_layernorm.bias''',
				    '''weight''': '''ln_f.weight''',
				    '''bias''': '''ln_f.bias''',
				}

				if key in layer_rename_map:
								return layer_rename_map[key]

				# Handle transformer blocks
				snake_case_						:   Union[str, Any]           =  int(re.match(R'''.*layer_(\d*).*'''			,    _UpperCamelCase			)[1]			)
				layer_number -= 3
				return f'''h.{layer_number}.''' + key






def 					lowerCamelCase_						(    _UpperCamelCase			)		->						int:






				"""simple docstring"""


				if dtype == torch.bool:
								return 1 / 8
				snake_case_						:   str           =  re.search(R'''[^\d](\d+)$'''			,    str(_UpperCamelCase			)			)
				if bit_search is None:
								raise ValueError(f'''`dtype` is not a valid dtype: {dtype}.'''			)
				snake_case_						:   Any           =  int(bit_search.groups()[0]			)
				return bit_size // 8






def 					lowerCamelCase_						(    _UpperCamelCase			,    _UpperCamelCase			,    _UpperCamelCase			,    _UpperCamelCase			,    _UpperCamelCase			)		->						str:






				"""simple docstring"""


				if bloom_config_file == "":
								snake_case_						:   Optional[Any]           =  BloomConfig()
				else:
								snake_case_						:   List[str]           =  BloomConfig.from_json_file(_UpperCamelCase			)

				if shard_model:
								snake_case_						:   Dict           =  os.listdir(_UpperCamelCase			)
								snake_case_						:   Tuple           =  sorted(filter(lambda _UpperCamelCase			: s.startswith('''layer'''			) and "model_00" in s			,    _UpperCamelCase			)			)

								snake_case_						:   List[Any]           =  {'''weight_map''': {}, '''metadata''': {}}
								snake_case_						:   int           =  0

								snake_case_						:   Tuple           =  None

								snake_case_						:   Tuple           =  BloomConfig()

								for j, file in enumerate(_UpperCamelCase			):
												print('''Processing file: {}'''.format(_UpperCamelCase			)			)
												snake_case_						:   List[str]           =  None

												for i in range(_UpperCamelCase			):
																# load all TP files
																snake_case_						:   Optional[int]           =  file.replace('''model_00'''			,    f'''model_0{i}'''			)
																snake_case_						:   int           =  torch.load(os.path.join(_UpperCamelCase			,    _UpperCamelCase			)			,    map_location='''cpu'''			)

																# Rename keys in the transformers names
																snake_case_						:   Dict           =  list(temp.keys()			)
																for key in keys:
																				snake_case_						:   Dict           =  temp.pop(_UpperCamelCase			)

																if tensors is None:
																				snake_case_						:   str           =  temp
																else:
																				for key in tensors.keys():
																								if any(key.endswith(_UpperCamelCase			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
																												# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																												tensors[key] += temp[key]
																								else:
																												# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																												snake_case_						:   List[Any]           =  1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN			) else 0
																												# We concatenate these weights accross TP ranks
																												snake_case_						:   List[str]           =  torch.cat([tensors[key], temp[key]]			,    dim=_UpperCamelCase			)

            # Divide by the number of TP the weights we want to average
												for key in tensors.keys():
																if any(key.endswith(_UpperCamelCase			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
																				snake_case_						:   List[Any]           =  tensors[key] / pretraining_tp
												torch.save(
												    _UpperCamelCase			,    os.path.join(
												        _UpperCamelCase			,    '''pytorch_model_{}-of-{}.bin'''.format(str(j + 1			).zfill(5			)			,    str(len(_UpperCamelCase			)			).zfill(5			)			)			,    )			,    )

												for key in tensors.keys():
																snake_case_						:   Union[str, Any]           =  tensors[key]
																total_size += value.numel() * get_dtype_size(value.dtype			)
																if key not in index_dict["weight_map"]:
																				snake_case_						:   Tuple           =  '''pytorch_model_{}-of-{}.bin'''.format(
																				    str(j + 1			).zfill(5			)			,    str(len(_UpperCamelCase			)			).zfill(5			)			)

								snake_case_						:   Dict           =  BloomConfig()
								snake_case_						:   List[str]           =  pytorch_dump_folder_path + '''/''' + CONFIG_NAME
								snake_case_						:   List[Any]           =  total_size
								with open(_UpperCamelCase			,    '''w'''			,    encoding='''utf-8'''			) as f:
												f.write(config.to_json_string()			)
								with open(os.path.join(_UpperCamelCase			,    WEIGHTS_NAME + '''.index.json'''			)			,    '''w'''			,    encoding='''utf-8'''			) as f:
												snake_case_						:   Optional[Any]           =  json.dumps(_UpperCamelCase			,    indent=2			,    sort_keys=_UpperCamelCase			) + '''\n'''
												f.write(_UpperCamelCase			)
				else:
								snake_case_						:   Dict           =  BloomModel(_UpperCamelCase			)

								snake_case_						:   Union[str, Any]           =  os.listdir(_UpperCamelCase			)
								snake_case_						:   Tuple           =  sorted(filter(lambda _UpperCamelCase			: s.startswith('''layer'''			) and "model_00" in s			,    _UpperCamelCase			)			)

								snake_case_						:   Tuple           =  None
								for i, file in enumerate(_UpperCamelCase			):
												snake_case_						:   Union[str, Any]           =  None
												for i in range(_UpperCamelCase			):
																# load all TP files
																snake_case_						:   Optional[Any]           =  file.replace('''model_00'''			,    f'''model_0{i}'''			)
																snake_case_						:   List[str]           =  torch.load(os.path.join(_UpperCamelCase			,    _UpperCamelCase			)			,    map_location='''cpu'''			)

																# Rename keys in the transformers names
																snake_case_						:   Dict           =  list(temp.keys()			)
																for key in keys:
																				snake_case_						:   Tuple           =  temp.pop(_UpperCamelCase			)

																if tensors is None:
																				snake_case_						:   List[Any]           =  temp
																else:
																				for key in tensors.keys():
																								# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																								if any(key.endswith(_UpperCamelCase			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
																												tensors[key] += temp[key]
																								else:
																												# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																												snake_case_						:   Any           =  1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN			) else 0
																												# We concatenate these weights accross TP ranks
																												snake_case_						:   Optional[Any]           =  torch.cat([tensors[key], temp[key]]			,    dim=_UpperCamelCase			)

            # Divide by the number of TP the weights we want to average
												for key in tensors.keys():
																if any(key.endswith(_UpperCamelCase			) for end in WEIGHTS_TO_AVERAGE_ENDSWITH			):
																				snake_case_						:   Optional[int]           =  tensors[key] / pretraining_tp

												snake_case_						:   Tuple           =  model.load_state_dict(_UpperCamelCase			,    strict=_UpperCamelCase			)
												assert not other_keys.unexpected_keys, f'''The keys {other_keys.unexpected_keys} are unexpected'''
												if missing_keys is None:
																snake_case_						:   Union[str, Any]           =  set(other_keys.missing_keys			)
												else:
																snake_case_						:   Tuple           =  missing_keys.intersection(set(other_keys.missing_keys			)			)

								assert not missing_keys, f'''The keys {missing_keys} are missing'''

								# Save pytorch-model
								os.makedirs(_UpperCamelCase			,    exist_ok=_UpperCamelCase			)
								snake_case_						:   List[str]           =  pytorch_dump_folder_path + '''/''' + WEIGHTS_NAME
								snake_case_						:   List[Any]           =  pytorch_dump_folder_path + '''/''' + CONFIG_NAME
								print(f'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''			)
								if config.torch_dtype is not None:
												snake_case_						:   str           =  model.to(config.torch_dtype			)
								torch.save(model.state_dict()			,    _UpperCamelCase			)
								print(f'''Save configuration file to {pytorch_config_dump_path}'''			)
								with open(_UpperCamelCase			,    '''w'''			,    encoding='''utf-8'''			) as f:
												f.write(config.to_json_string()			)


if __name__ == "__main__":
	lowerCAmelCase_    =    argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '''--bloom_checkpoint_path''',
	    default=None,
	    type=str,
	    required=True,
	    help='''Path to the Megatron-LM checkpoint path.''',
	)
	parser.add_argument(
	    '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
	)
	parser.add_argument(
	    '''--bloom_config_file''',
	    default='''''',
	    type=str,
	    help=(
	        '''An optional config json file corresponding to the pre-trained model. \n'''
	        '''This specifies the model architecture.'''
	    ),
	)
	parser.add_argument(
	    '''--shard_model''',
	    action='''store_true''',
	    help='''An optional setting to shard the output model \nThis enables sharding the converted checkpoint''',
	)
	parser.add_argument(
	    '''--pretraining_tp''',
	    default=4,
	    type=int,
	    help='''Pretraining TP rank that has been used when training the model in Megatron-LM \n''',
	)
	lowerCAmelCase_    =    parser.parse_args()
	convert_bloom_checkpoint_to_pytorch(
	    args.bloom_checkpoint_path,
	    args.bloom_config_file,
	    args.pytorch_dump_folder_path,
	    args.shard_model,
	    args.pretraining_tp,
	)







# DISCLAIMER: This file is strongly influenced by https://github.com/ermongroup/ddim

from dataclasses import dataclass
from typing import Optional, Tuple, Union

import flax
import jax
import jax.numpy as jnp

from ..configuration_utils import ConfigMixin, register_to_config
from .scheduling_utils_flax import (
    CommonSchedulerState,
    FlaxKarrasDiffusionSchedulers,
    FlaxSchedulerMixin,
    FlaxSchedulerOutput,
    add_noise_common,
    get_velocity_common,
)



@flax.struct.dataclass
class  __lowerCAmelCase	:
			lowerCamelCase_					:					CommonSchedulerState

			# setable values
			lowerCamelCase_					:					jnp.ndarray
			lowerCamelCase_					:					jnp.ndarray
			lowerCamelCase_					:					Optional[int]	      =				None






			@classmethod
			def       lowerCamelCase  (cls ,							__magic_name__ ,							__magic_name__ ,							__magic_name__   )					->       Dict:


							'''simple docstring'''



							return cls(common=__magic_name__ ,							init_noise_sigma=__magic_name__ ,							timesteps=__magic_name__   )




@dataclass
class  __lowerCAmelCase	(      _a					):
			lowerCamelCase_					:					DDPMSchedulerState




class  __lowerCAmelCase	(      _a,					_a					):
			lowerCamelCase_					:					List[Any]	      =				[e.name for e in FlaxKarrasDiffusionSchedulers]

			lowerCamelCase_					:					jnp.dtype






			@property
			def       lowerCamelCase  (self   )					->       int:


							'''simple docstring'''



							return True






			@register_to_config
			def __init__(self ,							__magic_name__ = 1000 ,							__magic_name__ = 0.0_001 ,							__magic_name__ = 0.02 ,							__magic_name__ = "linear" ,							__magic_name__ = None ,							__magic_name__ = "fixed_small" ,							__magic_name__ = True ,							__magic_name__ = "epsilon" ,							__magic_name__ = jnp.floataa ,							)					->       List[str]:


							'''simple docstring'''



							snake_case_						:   Optional[Any]           =  dtype






			def       lowerCamelCase  (self ,							__magic_name__ = None   )					->       DDPMSchedulerState:


							'''simple docstring'''



							if common is None:
											snake_case_						:   Any           =  CommonSchedulerState.create(self   )

							# standard deviation of the initial noise distribution
							snake_case_						:   Any           =  jnp.array(1.0 ,							dtype=self.dtype   )

							snake_case_						:   Optional[int]           =  jnp.arange(0 ,							self.config.num_train_timesteps   ).round()[::-1]

							return DDPMSchedulerState.create(
							    common=__magic_name__ ,							init_noise_sigma=__magic_name__ ,							timesteps=__magic_name__ ,							)






			def       lowerCamelCase  (self ,							__magic_name__ ,							__magic_name__ ,							__magic_name__ = None   )					->       jnp.ndarray:


							'''simple docstring'''



							return sample






			def       lowerCamelCase  (self ,							__magic_name__ ,							__magic_name__ ,							__magic_name__ = ()   )					->       DDPMSchedulerState:


							'''simple docstring'''



							snake_case_						:   str           =  self.config.num_train_timesteps // num_inference_steps
							# creates integer timesteps by multiplying by ratio
							# rounding to avoid issues when num_inference_step is power of 3
							snake_case_						:   List[str]           =  (jnp.arange(0 ,							__magic_name__   ) * step_ratio).round()[::-1]

							return state.replace(
							    num_inference_steps=__magic_name__ ,							timesteps=__magic_name__ ,							)






			def       lowerCamelCase  (self ,							__magic_name__ ,							__magic_name__ ,							__magic_name__=None ,							__magic_name__=None   )					->       int:


							'''simple docstring'''



							snake_case_						:   Any           =  state.common.alphas_cumprod[t]
							snake_case_						:   Dict           =  jnp.where(t > 0 ,							state.common.alphas_cumprod[t - 1] ,							jnp.array(1.0 ,							dtype=self.dtype   )   )

							# For t > 0, compute predicted variance βt (see formula (6) and (7) from https://arxiv.org/pdf/2006.11239.pdf)
							# and sample from it to get previous sample
							# x_{t-1} ~ N(pred_prev_sample, variance) == add variance to pred_sample
							snake_case_						:   Tuple           =  (1 - alpha_prod_t_prev) / (1 - alpha_prod_t) * state.common.betas[t]

							if variance_type is None:
											snake_case_						:   Any           =  self.config.variance_type

							# hacks - were probably added for training stability
							if variance_type == "fixed_small":
											snake_case_						:   int           =  jnp.clip(__magic_name__ ,							a_min=1e-20   )
							# for rl-diffuser https://arxiv.org/abs/2205.09991
							elif variance_type == "fixed_small_log":
											snake_case_						:   List[str]           =  jnp.log(jnp.clip(__magic_name__ ,							a_min=1e-20   )   )
							elif variance_type == "fixed_large":
											snake_case_						:   str           =  state.common.betas[t]
							elif variance_type == "fixed_large_log":
											# Glide max_log
											snake_case_						:   int           =  jnp.log(state.common.betas[t]   )
							elif variance_type == "learned":
											return predicted_variance
							elif variance_type == "learned_range":
											snake_case_						:   str           =  variance
											snake_case_						:   Optional[Any]           =  state.common.betas[t]
											snake_case_						:   Optional[Any]           =  (predicted_variance + 1) / 2
											snake_case_						:   Optional[Any]           =  frac * max_log + (1 - frac) * min_log

							return variance






			def       lowerCamelCase  (self ,							__magic_name__ ,							__magic_name__ ,							__magic_name__ ,							__magic_name__ ,							__magic_name__ = None ,							__magic_name__ = True ,							)					->       Union[FlaxDDPMSchedulerOutput, Tuple]:


							'''simple docstring'''



							snake_case_						:   Tuple           =  timestep

							if key is None:
											snake_case_						:   List[Any]           =  jax.random.PRNGKey(0   )

							if model_output.shape[1] == sample.shape[1] * 2 and self.config.variance_type in ["learned", "learned_range"]:
											snake_case_      ,							snake_case_						:   Any           =  jnp.split(__magic_name__ ,							sample.shape[1] ,							axis=1   )
							else:
											snake_case_						:   Optional[Any]           =  None

							# 1. compute alphas, betas
							snake_case_						:   List[Any]           =  state.common.alphas_cumprod[t]
							snake_case_						:   Union[str, Any]           =  jnp.where(t > 0 ,							state.common.alphas_cumprod[t - 1] ,							jnp.array(1.0 ,							dtype=self.dtype   )   )
							snake_case_						:   List[Any]           =  1 - alpha_prod_t
							snake_case_						:   str           =  1 - alpha_prod_t_prev

							# 2. compute predicted original sample from predicted noise also called
							# "predicted x_0" of formula (15) from https://arxiv.org/pdf/2006.11239.pdf
							if self.config.prediction_type == "epsilon":
											snake_case_						:   int           =  (sample - beta_prod_t ** 0.5 * model_output) / alpha_prod_t ** 0.5
							elif self.config.prediction_type == "sample":
											snake_case_						:   List[Any]           =  model_output
							elif self.config.prediction_type == "v_prediction":
											snake_case_						:   Optional[Any]           =  (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output
							else:
											raise ValueError(
											    F'''prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample` '''
											    ''' for the FlaxDDPMScheduler.'''   )

							# 3. Clip "predicted x_0"
							if self.config.clip_sample:
											snake_case_						:   Optional[int]           =  jnp.clip(__magic_name__ ,							-1 ,							1   )

							# 4. Compute coefficients for pred_original_sample x_0 and current sample x_t
							# See formula (7) from https://arxiv.org/pdf/2006.11239.pdf
							snake_case_						:   str           =  (alpha_prod_t_prev ** 0.5 * state.common.betas[t]) / beta_prod_t
							snake_case_						:   Dict           =  state.common.alphas[t] ** 0.5 * beta_prod_t_prev / beta_prod_t

							# 5. Compute predicted previous sample µ_t
							# See formula (7) from https://arxiv.org/pdf/2006.11239.pdf
							snake_case_						:   Optional[Any]           =  pred_original_sample_coeff * pred_original_sample + current_sample_coeff * sample

							# 6. Add noise
							def random_variance():
											snake_case_						:   List[Any]           =  jax.random.split(__magic_name__ ,							num=1   )
											snake_case_						:   Tuple           =  jax.random.normal(__magic_name__ ,							shape=model_output.shape ,							dtype=self.dtype   )
											return (self._get_variance(__magic_name__ ,							__magic_name__ ,							predicted_variance=__magic_name__   ) ** 0.5) * noise

							snake_case_						:   List[str]           =  jnp.where(t > 0 ,							random_variance() ,							jnp.zeros(model_output.shape ,							dtype=self.dtype   )   )

							snake_case_						:   Any           =  pred_prev_sample + variance

							if not return_dict:
											return (pred_prev_sample, state)

							return FlaxDDPMSchedulerOutput(prev_sample=__magic_name__ ,							state=__magic_name__   )






			def       lowerCamelCase  (self ,							__magic_name__ ,							__magic_name__ ,							__magic_name__ ,							__magic_name__ ,							)					->       jnp.ndarray:


							'''simple docstring'''



							return add_noise_common(state.common ,							__magic_name__ ,							__magic_name__ ,							__magic_name__   )






			def       lowerCamelCase  (self ,							__magic_name__ ,							__magic_name__ ,							__magic_name__ ,							__magic_name__ ,							)					->       jnp.ndarray:


							'''simple docstring'''



							return get_velocity_common(state.common ,							__magic_name__ ,							__magic_name__ ,							__magic_name__   )






			def __len__(self   )					->       int:


							'''simple docstring'''



							return self.config.num_train_timesteps
