

"""simple docstring"""





import flax.linen as nn
import jax.numpy as jnp

from .attention_flax import FlaxTransformeraDModel
from .resnet_flax import FlaxDownsampleaD, FlaxResnetBlockaD, FlaxUpsampleaD







class 							SCREAMING_SNAKE_CASE   (						nn.Module			):






					"""simple docstring"""



					lowercase__												=			42
					lowercase__												=			42
					lowercase__												=			0.0
					lowercase__												=			1
					lowercase__												=			1
					lowercase__												=			True
					lowercase__												=			False
					lowercase__												=			False
					lowercase__												=			False
					lowercase__												=			jnp.floataa







					def 			__lowerCAmelCase					(			self     :							str						):
						lowerCAmelCase__      :		str    							=      []
						lowerCAmelCase__      :		Any    							=      []

						for i in range(self.num_layers						):
							lowerCAmelCase__      :		str    							=      self.in_channels if i == 0 else self.out_channels

							lowerCAmelCase__      :		Optional[int]    							=      FlaxResnetBlockaD(
							    in_channels=__snake_case  ,out_channels=self.out_channels  ,dropout_prob=self.dropout  ,dtype=self.dtype  ,)
							resnets.append(__snake_case						)

							lowerCAmelCase__      :		List[str]    							=      FlaxTransformeraDModel(
							    in_channels=self.out_channels  ,n_heads=self.num_attention_heads  ,d_head=self.out_channels // self.num_attention_heads  ,depth=1  ,use_linear_projection=self.use_linear_projection  ,only_cross_attention=self.only_cross_attention  ,use_memory_efficient_attention=self.use_memory_efficient_attention  ,dtype=self.dtype  ,)
							attentions.append(__snake_case						)

						lowerCAmelCase__      :		Any    							=      resnets
						lowerCAmelCase__      :		str    							=      attentions

						if self.add_downsample:
							lowerCAmelCase__      :		List[str]    							=      FlaxDownsampleaD(self.out_channels  ,dtype=self.dtype						)







					def __call__(			self     :							Any  ,lowercase_     :							List[Any]  ,lowercase_     :							Any  ,lowercase_     :							Dict  ,lowercase_     :							List[str]=True						):
						lowerCAmelCase__      :		str    							=      ()

						for resnet, attn in zip(self.resnets  ,self.attentions						):
							lowerCAmelCase__      :		Optional[Any]    							=      resnet(__snake_case  ,__snake_case  ,deterministic=__snake_case						)
							lowerCAmelCase__      :		Union[str, Any]    							=      attn(__snake_case  ,__snake_case  ,deterministic=__snake_case						)
							output_states += (hidden_states,)

						if self.add_downsample:
							lowerCAmelCase__      :		Any    							=      self.downsamplers_a(__snake_case						)
							output_states += (hidden_states,)

						return hidden_states, output_states







class 							SCREAMING_SNAKE_CASE   (						nn.Module			):






					"""simple docstring"""



					lowercase__												=			42
					lowercase__												=			42
					lowercase__												=			0.0
					lowercase__												=			1
					lowercase__												=			True
					lowercase__												=			jnp.floataa







					def 			__lowerCAmelCase					(			self     :							str						):
						lowerCAmelCase__      :		Optional[Any]    							=      []

						for i in range(self.num_layers						):
							lowerCAmelCase__      :		str    							=      self.in_channels if i == 0 else self.out_channels

							lowerCAmelCase__      :		Tuple    							=      FlaxResnetBlockaD(
							    in_channels=__snake_case  ,out_channels=self.out_channels  ,dropout_prob=self.dropout  ,dtype=self.dtype  ,)
							resnets.append(__snake_case						)
						lowerCAmelCase__      :		Union[str, Any]    							=      resnets

						if self.add_downsample:
							lowerCAmelCase__      :		str    							=      FlaxDownsampleaD(self.out_channels  ,dtype=self.dtype						)







					def __call__(			self     :							str  ,lowercase_     :							int  ,lowercase_     :							List[str]  ,lowercase_     :							List[Any]=True						):
						lowerCAmelCase__      :		Optional[Any]    							=      ()

						for resnet in self.resnets:
							lowerCAmelCase__      :		Optional[int]    							=      resnet(__snake_case  ,__snake_case  ,deterministic=__snake_case						)
							output_states += (hidden_states,)

						if self.add_downsample:
							lowerCAmelCase__      :		List[str]    							=      self.downsamplers_a(__snake_case						)
							output_states += (hidden_states,)

						return hidden_states, output_states







class 							SCREAMING_SNAKE_CASE   (						nn.Module			):






					"""simple docstring"""



					lowercase__												=			42
					lowercase__												=			42
					lowercase__												=			42
					lowercase__												=			0.0
					lowercase__												=			1
					lowercase__												=			1
					lowercase__												=			True
					lowercase__												=			False
					lowercase__												=			False
					lowercase__												=			False
					lowercase__												=			jnp.floataa







					def 			__lowerCAmelCase					(			self     :							int						):
						lowerCAmelCase__      :		Optional[int]    							=      []
						lowerCAmelCase__      :		Any    							=      []

						for i in range(self.num_layers						):
							lowerCAmelCase__      :		int    							=      self.in_channels if (i == self.num_layers - 1) else self.out_channels
							lowerCAmelCase__      :		Any    							=      self.prev_output_channel if i == 0 else self.out_channels

							lowerCAmelCase__      :		Dict    							=      FlaxResnetBlockaD(
							    in_channels=resnet_in_channels + res_skip_channels  ,out_channels=self.out_channels  ,dropout_prob=self.dropout  ,dtype=self.dtype  ,)
							resnets.append(__snake_case						)

							lowerCAmelCase__      :		int    							=      FlaxTransformeraDModel(
							    in_channels=self.out_channels  ,n_heads=self.num_attention_heads  ,d_head=self.out_channels // self.num_attention_heads  ,depth=1  ,use_linear_projection=self.use_linear_projection  ,only_cross_attention=self.only_cross_attention  ,use_memory_efficient_attention=self.use_memory_efficient_attention  ,dtype=self.dtype  ,)
							attentions.append(__snake_case						)

						lowerCAmelCase__      :		List[str]    							=      resnets
						lowerCAmelCase__      :		List[str]    							=      attentions

						if self.add_upsample:
							lowerCAmelCase__      :		str    							=      FlaxUpsampleaD(self.out_channels  ,dtype=self.dtype						)







					def __call__(			self     :							Tuple  ,lowercase_     :							int  ,lowercase_     :							List[Any]  ,lowercase_     :							Dict  ,lowercase_     :							List[str]  ,lowercase_     :							Dict=True						):
						for resnet, attn in zip(self.resnets  ,self.attentions						):
							# pop res hidden states
							lowerCAmelCase__      :		Any    							=      res_hidden_states_tuple[-1]
							lowerCAmelCase__      :		Optional[int]    							=      res_hidden_states_tuple[:-1]
							lowerCAmelCase__      :		Optional[Any]    							=      jnp.concatenate((hidden_states, res_hidden_states)  ,axis=-1						)

							lowerCAmelCase__      :		Optional[Any]    							=      resnet(__snake_case  ,__snake_case  ,deterministic=__snake_case						)
							lowerCAmelCase__      :		str    							=      attn(__snake_case  ,__snake_case  ,deterministic=__snake_case						)

						if self.add_upsample:
							lowerCAmelCase__      :		Optional[Any]    							=      self.upsamplers_a(__snake_case						)

						return hidden_states







class 							SCREAMING_SNAKE_CASE   (						nn.Module			):






					"""simple docstring"""



					lowercase__												=			42
					lowercase__												=			42
					lowercase__												=			42
					lowercase__												=			0.0
					lowercase__												=			1
					lowercase__												=			True
					lowercase__												=			jnp.floataa







					def 			__lowerCAmelCase					(			self     :							int						):
						lowerCAmelCase__      :		Any    							=      []

						for i in range(self.num_layers						):
							lowerCAmelCase__      :		List[str]    							=      self.in_channels if (i == self.num_layers - 1) else self.out_channels
							lowerCAmelCase__      :		List[Any]    							=      self.prev_output_channel if i == 0 else self.out_channels

							lowerCAmelCase__      :		List[Any]    							=      FlaxResnetBlockaD(
							    in_channels=resnet_in_channels + res_skip_channels  ,out_channels=self.out_channels  ,dropout_prob=self.dropout  ,dtype=self.dtype  ,)
							resnets.append(__snake_case						)

						lowerCAmelCase__      :		Any    							=      resnets

						if self.add_upsample:
							lowerCAmelCase__      :		List[str]    							=      FlaxUpsampleaD(self.out_channels  ,dtype=self.dtype						)







					def __call__(			self     :							List[str]  ,lowercase_     :							Any  ,lowercase_     :							Tuple  ,lowercase_     :							str  ,lowercase_     :							Optional[int]=True						):
						for resnet in self.resnets:
							# pop res hidden states
							lowerCAmelCase__      :		List[Any]    							=      res_hidden_states_tuple[-1]
							lowerCAmelCase__      :		Dict    							=      res_hidden_states_tuple[:-1]
							lowerCAmelCase__      :		Dict    							=      jnp.concatenate((hidden_states, res_hidden_states)  ,axis=-1						)

							lowerCAmelCase__      :		Optional[int]    							=      resnet(__snake_case  ,__snake_case  ,deterministic=__snake_case						)

						if self.add_upsample:
							lowerCAmelCase__      :		Optional[Any]    							=      self.upsamplers_a(__snake_case						)

						return hidden_states







class 							SCREAMING_SNAKE_CASE   (						nn.Module			):






					"""simple docstring"""



					lowercase__												=			42
					lowercase__												=			0.0
					lowercase__												=			1
					lowercase__												=			1
					lowercase__												=			False
					lowercase__												=			False
					lowercase__												=			jnp.floataa







					def 			__lowerCAmelCase					(			self     :							Dict						):
						# there is always at least one resnet
						lowerCAmelCase__      :		Tuple    							=      [
						    FlaxResnetBlockaD(
						        in_channels=self.in_channels  ,out_channels=self.in_channels  ,dropout_prob=self.dropout  ,dtype=self.dtype  ,)
						]

						lowerCAmelCase__      :		str    							=      []

						for _ in range(self.num_layers						):
							lowerCAmelCase__      :		List[Any]    							=      FlaxTransformeraDModel(
							    in_channels=self.in_channels  ,n_heads=self.num_attention_heads  ,d_head=self.in_channels // self.num_attention_heads  ,depth=1  ,use_linear_projection=self.use_linear_projection  ,use_memory_efficient_attention=self.use_memory_efficient_attention  ,dtype=self.dtype  ,)
							attentions.append(__snake_case						)

							lowerCAmelCase__      :		Tuple    							=      FlaxResnetBlockaD(
							    in_channels=self.in_channels  ,out_channels=self.in_channels  ,dropout_prob=self.dropout  ,dtype=self.dtype  ,)
							resnets.append(__snake_case						)

						lowerCAmelCase__      :		Optional[Any]    							=      resnets
						lowerCAmelCase__      :		int    							=      attentions







					def __call__(			self     :							str  ,lowercase_     :							Tuple  ,lowercase_     :							Tuple  ,lowercase_     :							List[Any]  ,lowercase_     :							Optional[Any]=True						):
						lowerCAmelCase__      :		Optional[Any]    							=      self.resnets[0](__snake_case  ,__snake_case						)
						for attn, resnet in zip(self.attentions  ,self.resnets[1:]						):
							lowerCAmelCase__      :		Optional[Any]    							=      attn(__snake_case  ,__snake_case  ,deterministic=__snake_case						)
							lowerCAmelCase__      :		str    							=      resnet(__snake_case  ,__snake_case  ,deterministic=__snake_case						)

						return hidden_states












"""simple docstring"""





import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

__lowerCamelCase							=  [
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

__lowerCamelCase							=  [
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]



def   a    (     __UpperCAmelCase      :    Any      ,   __UpperCAmelCase      :    Any  )    ->							Any:
						__magic_name__:	List[str]							=       {
						    """word_embeddings.weight""": """word_embeddings.weight""",
						    """word_embeddings.norm.weight""": """word_embeddings_layernorm.weight""",
						    """word_embeddings.norm.bias""": """word_embeddings_layernorm.bias""",
						    """weight""": """ln_f.weight""",
						    """bias""": """ln_f.bias""",
						}

						if key in layer_rename_map:
												return layer_rename_map[key]

						# Handle transformer blocks
						__magic_name__:	Any							=       int(re.match(R""".*layer_(\d*).*"""      ,   __UpperCAmelCase  )[1]  )
						layer_number -= 3
						return f'h.{layer_number}.' + key



def   a    (     __UpperCAmelCase      :    List[str]  )    ->							Union[str, Any]:
						if dtype == torch.bool:
												return 1 / 8
						__magic_name__:	Optional[int]							=       re.search(R"""[^\d](\d+)$"""      ,   str(__UpperCAmelCase  )  )
						if bit_search is None:
												raise ValueError(f'`dtype` is not a valid dtype: {dtype}.'  )
						__magic_name__:	Dict							=       int(bit_search.groups()[0]  )
						return bit_size // 8



def   a    (     __UpperCAmelCase      :    Union[str, Any]      ,   __UpperCAmelCase      :    Optional[int]      ,   __UpperCAmelCase      :    List[Any]      ,   __UpperCAmelCase      :    Optional[int]      ,   __UpperCAmelCase      :    Tuple  )    ->							List[str]:
						# Construct model
						if bloom_config_file == "":
												__magic_name__:	Optional[Any]							=       BloomConfig()
						else:
												__magic_name__:	Tuple							=       BloomConfig.from_json_file(__UpperCAmelCase  )

						if shard_model:
												__magic_name__:	Union[str, Any]							=       os.listdir(__UpperCAmelCase  )
												__magic_name__:	Optional[int]							=       sorted(filter(lambda __UpperCAmelCase  : s.startswith("""layer"""  ) and "model_00" in s      ,   __UpperCAmelCase  )  )

												__magic_name__:	str							=       {"""weight_map""": {}, """metadata""": {}}
												__magic_name__:	Optional[Any]							=       0

												__magic_name__:	Dict							=       None

												__magic_name__:	Optional[int]							=       BloomConfig()

												for j, file in enumerate(__UpperCAmelCase  ):
																		print("""Processing file: {}""".format(__UpperCAmelCase  )  )
																		__magic_name__:	Tuple							=       None

																		for i in range(__UpperCAmelCase  ):
																								# load all TP files
																								__magic_name__:	int							=       file.replace("""model_00"""      ,   f'model_0{i}'  )
																								__magic_name__:	Optional[Any]							=       torch.load(os.path.join(__UpperCAmelCase      ,   __UpperCAmelCase  )      ,   map_location="""cpu"""  )

																								# Rename keys in the transformers names
																								__magic_name__:	str							=       list(temp.keys()  )
																								for key in keys:
																														__magic_name__:	Any							=       temp.pop(__UpperCAmelCase  )

																								if tensors is None:
																														__magic_name__:	Union[str, Any]							=       temp
																								else:
																														for key in tensors.keys():
																																				if any(key.endswith(__UpperCAmelCase  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
																																										# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																										tensors[key] += temp[key]
																																				else:
																																										# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																										__magic_name__:	int							=       1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN  ) else 0
																																										# We concatenate these weights accross TP ranks
																																										__magic_name__:	int							=       torch.cat([tensors[key], temp[key]]      ,   dim=__UpperCAmelCase  )

            # Divide by the number of TP the weights we want to average
																		for key in tensors.keys():
																								if any(key.endswith(__UpperCAmelCase  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
																														__magic_name__:	Union[str, Any]							=       tensors[key] / pretraining_tp
																		torch.save(
																		    __UpperCAmelCase      ,   os.path.join(
																		        __UpperCAmelCase      ,   """pytorch_model_{}-of-{}.bin""".format(str(j + 1  ).zfill(5  )      ,   str(len(__UpperCAmelCase  )  ).zfill(5  )  )      ,   )      ,   )

																		for key in tensors.keys():
																								__magic_name__:	Optional[Any]							=       tensors[key]
																								total_size += value.numel() * get_dtype_size(value.dtype  )
																								if key not in index_dict["weight_map"]:
																														__magic_name__:	Union[str, Any]							=       """pytorch_model_{}-of-{}.bin""".format(
																														    str(j + 1  ).zfill(5  )      ,   str(len(__UpperCAmelCase  )  ).zfill(5  )  )

												__magic_name__:	Tuple							=       BloomConfig()
												__magic_name__:	Optional[int]							=       pytorch_dump_folder_path + """/""" + CONFIG_NAME
												__magic_name__:	Optional[int]							=       total_size
												with open(__UpperCAmelCase      ,   """w"""      ,   encoding="""utf-8"""  ) as f:
																		f.write(config.to_json_string()  )
												with open(os.path.join(__UpperCAmelCase      ,   WEIGHTS_NAME + """.index.json"""  )      ,   """w"""      ,   encoding="""utf-8"""  ) as f:
																		__magic_name__:	str							=       json.dumps(__UpperCAmelCase      ,   indent=2      ,   sort_keys=__UpperCAmelCase  ) + """\n"""
																		f.write(__UpperCAmelCase  )
						else:
												__magic_name__:	Optional[int]							=       BloomModel(__UpperCAmelCase  )

												__magic_name__:	List[str]							=       os.listdir(__UpperCAmelCase  )
												__magic_name__:	Optional[Any]							=       sorted(filter(lambda __UpperCAmelCase  : s.startswith("""layer"""  ) and "model_00" in s      ,   __UpperCAmelCase  )  )

												__magic_name__:	Any							=       None
												for i, file in enumerate(__UpperCAmelCase  ):
																		__magic_name__:	List[Any]							=       None
																		for i in range(__UpperCAmelCase  ):
																								# load all TP files
																								__magic_name__:	Any							=       file.replace("""model_00"""      ,   f'model_0{i}'  )
																								__magic_name__:	Dict							=       torch.load(os.path.join(__UpperCAmelCase      ,   __UpperCAmelCase  )      ,   map_location="""cpu"""  )

																								# Rename keys in the transformers names
																								__magic_name__:	Optional[Any]							=       list(temp.keys()  )
																								for key in keys:
																														__magic_name__:	Optional[Any]							=       temp.pop(__UpperCAmelCase  )

																								if tensors is None:
																														__magic_name__:	Tuple							=       temp
																								else:
																														for key in tensors.keys():
																																				# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																																				if any(key.endswith(__UpperCAmelCase  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
																																										tensors[key] += temp[key]
																																				else:
																																										# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																																										__magic_name__:	Tuple							=       1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN  ) else 0
																																										# We concatenate these weights accross TP ranks
																																										__magic_name__:	Dict							=       torch.cat([tensors[key], temp[key]]      ,   dim=__UpperCAmelCase  )

            # Divide by the number of TP the weights we want to average
																		for key in tensors.keys():
																								if any(key.endswith(__UpperCAmelCase  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
																														__magic_name__:	Optional[int]							=       tensors[key] / pretraining_tp

																		__magic_name__:	Union[str, Any]							=       model.load_state_dict(__UpperCAmelCase      ,   strict=__UpperCAmelCase  )
																		assert not other_keys.unexpected_keys, f'The keys {other_keys.unexpected_keys} are unexpected'
																		if missing_keys is None:
																								__magic_name__:	Optional[Any]							=       set(other_keys.missing_keys  )
																		else:
																								__magic_name__:	Tuple							=       missing_keys.intersection(set(other_keys.missing_keys  )  )

												assert not missing_keys, f'The keys {missing_keys} are missing'

												# Save pytorch-model
												os.makedirs(__UpperCAmelCase      ,   exist_ok=__UpperCAmelCase  )
												__magic_name__:	int							=       pytorch_dump_folder_path + """/""" + WEIGHTS_NAME
												__magic_name__:	List[str]							=       pytorch_dump_folder_path + """/""" + CONFIG_NAME
												print(f'Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'  )
												if config.torch_dtype is not None:
																		__magic_name__:	Optional[Any]							=       model.to(config.torch_dtype  )
												torch.save(model.state_dict()      ,   __UpperCAmelCase  )
												print(f'Save configuration file to {pytorch_config_dump_path}'  )
												with open(__UpperCAmelCase      ,   """w"""      ,   encoding="""utf-8"""  ) as f:
																		f.write(config.to_json_string()  )


if __name__ == "__main__":
							__lowerCamelCase							=  argparse.ArgumentParser()
							# Required parameters
							parser.add_argument(
							    '--bloom_checkpoint_path',
							    default=None,
							    type=str,
							    required=True,
							    help='Path to the Megatron-LM checkpoint path.',
							)
							parser.add_argument(
							    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
							)
							parser.add_argument(
							    '--bloom_config_file',
							    default='',
							    type=str,
							    help=(
							        'An optional config json file corresponding to the pre-trained model. \n'
							        'This specifies the model architecture.'
							    ),
							)
							parser.add_argument(
							    '--shard_model',
							    action='store_true',
							    help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
							)
							parser.add_argument(
							    '--pretraining_tp',
							    default=4,
							    type=int,
							    help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
							)
							__lowerCamelCase							=  parser.parse_args()
							convert_bloom_checkpoint_to_pytorch(
							    args.bloom_checkpoint_path,
							    args.bloom_config_file,
							    args.pytorch_dump_folder_path,
							    args.shard_model,
							    args.pretraining_tp,
							)



