
import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

UpperCamelCase_   	=	[
    """word_embeddings_layernorm.weight""",
    """word_embeddings_layernorm.bias""",
    """input_layernorm.weight""",
    """input_layernorm.bias""",
    """post_attention_layernorm.weight""",
    """post_attention_layernorm.bias""",
    """self_attention.dense.bias""",
    """mlp.dense_4h_to_h.bias""",
    """ln_f.weight""",
    """ln_f.bias""",
]

UpperCamelCase_   	=	[
    """mlp.dense_4h_to_h.weight""",
    """self_attention.dense.weight""",
]






def 		_UpperCAmelCase   (	UpperCamelCase:     Dict			,							UpperCamelCase:     Optional[int]  ):






	"""simple docstring"""


	__lowerCAmelCase         =							{
	    "word_embeddings.weight": "word_embeddings.weight",
	    "word_embeddings.norm.weight": "word_embeddings_layernorm.weight",
	    "word_embeddings.norm.bias": "word_embeddings_layernorm.bias",
	    "weight": "ln_f.weight",
	    "bias": "ln_f.bias",
	}

	if key in layer_rename_map:
		return layer_rename_map[key]

	# Handle transformer blocks
	__lowerCAmelCase         =							int(re.match(R".*layer_(\d*).*"			,							UpperCamelCase  )[1]  )
	layer_number -= 3
	return F"h.{layer_number}." + key






def 		_UpperCAmelCase   (	UpperCamelCase:     Dict  ):






	"""simple docstring"""


	if dtype == torch.bool:
		return 1 / 8
	__lowerCAmelCase         =							re.search(R"[^\d](\d+)$"			,							str(UpperCamelCase  )  )
	if bit_search is None:
		raise ValueError(F"`dtype` is not a valid dtype: {dtype}."  )
	__lowerCAmelCase         =							int(bit_search.groups()[0]  )
	return bit_size // 8






def 		_UpperCAmelCase   (	UpperCamelCase:     Union[str, Any]			,							UpperCamelCase:     str			,							UpperCamelCase:     Any			,							UpperCamelCase:     List[Any]			,							UpperCamelCase:     str  ):






	"""simple docstring"""


	if bloom_config_file == "":
		__lowerCAmelCase         =							BloomConfig()
	else:
		__lowerCAmelCase         =							BloomConfig.from_json_file(UpperCamelCase  )

	if shard_model:
		__lowerCAmelCase         =							os.listdir(UpperCamelCase  )
		__lowerCAmelCase         =							sorted(filter(lambda UpperCamelCase  : s.startswith("layer"  ) and "model_00" in s			,							UpperCamelCase  )  )

		__lowerCAmelCase         =							{"weight_map": {}, "metadata": {}}
		__lowerCAmelCase         =							0

		__lowerCAmelCase         =							None

		__lowerCAmelCase         =							BloomConfig()

		for j, file in enumerate(UpperCamelCase  ):
			print("Processing file: {}".format(UpperCamelCase  )  )
			__lowerCAmelCase         =							None

			for i in range(UpperCamelCase  ):
				# load all TP files
				__lowerCAmelCase         =							file.replace("model_00"			,							F"model_0{i}"  )
				__lowerCAmelCase         =							torch.load(os.path.join(UpperCamelCase			,							UpperCamelCase  )			,							map_location="cpu"  )

				# Rename keys in the transformers names
				__lowerCAmelCase         =							list(temp.keys()  )
				for key in keys:
					__lowerCAmelCase         =							temp.pop(UpperCamelCase  )

				if tensors is None:
					__lowerCAmelCase         =							temp
				else:
					for key in tensors.keys():
						if any(key.endswith(UpperCamelCase  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
							# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
							tensors[key] += temp[key]
						else:
							# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
							__lowerCAmelCase         =							1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN  ) else 0
							# We concatenate these weights accross TP ranks
							__lowerCAmelCase         =							torch.cat([tensors[key], temp[key]]			,							dim=UpperCamelCase  )

            # Divide by the number of TP the weights we want to average
			for key in tensors.keys():
				if any(key.endswith(UpperCamelCase  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
					__lowerCAmelCase         =							tensors[key] / pretraining_tp
			torch.save(
			    UpperCamelCase			,							os.path.join(
			        UpperCamelCase			,							"pytorch_model_{}-of-{}.bin".format(str(j + 1  ).zfill(5  )			,							str(len(UpperCamelCase  )  ).zfill(5  )  )			,							)			,							)

			for key in tensors.keys():
				__lowerCAmelCase         =							tensors[key]
				total_size += value.numel() * get_dtype_size(value.dtype  )
				if key not in index_dict["weight_map"]:
					__lowerCAmelCase         =							"pytorch_model_{}-of-{}.bin".format(
					    str(j + 1  ).zfill(5  )			,							str(len(UpperCamelCase  )  ).zfill(5  )  )

		__lowerCAmelCase         =							BloomConfig()
		__lowerCAmelCase         =							pytorch_dump_folder_path + "/" + CONFIG_NAME
		__lowerCAmelCase         =							total_size
		with open(UpperCamelCase			,							"w"			,							encoding="utf-8"  ) as f:
			f.write(config.to_json_string()  )
		with open(os.path.join(UpperCamelCase			,							WEIGHTS_NAME + ".index.json"  )			,							"w"			,							encoding="utf-8"  ) as f:
			__lowerCAmelCase         =							json.dumps(UpperCamelCase			,							indent=2			,							sort_keys=UpperCamelCase  ) + "\n"
			f.write(UpperCamelCase  )
	else:
		__lowerCAmelCase         =							BloomModel(UpperCamelCase  )

		__lowerCAmelCase         =							os.listdir(UpperCamelCase  )
		__lowerCAmelCase         =							sorted(filter(lambda UpperCamelCase  : s.startswith("layer"  ) and "model_00" in s			,							UpperCamelCase  )  )

		__lowerCAmelCase         =							None
		for i, file in enumerate(UpperCamelCase  ):
			__lowerCAmelCase         =							None
			for i in range(UpperCamelCase  ):
				# load all TP files
				__lowerCAmelCase         =							file.replace("model_00"			,							F"model_0{i}"  )
				__lowerCAmelCase         =							torch.load(os.path.join(UpperCamelCase			,							UpperCamelCase  )			,							map_location="cpu"  )

				# Rename keys in the transformers names
				__lowerCAmelCase         =							list(temp.keys()  )
				for key in keys:
					__lowerCAmelCase         =							temp.pop(UpperCamelCase  )

				if tensors is None:
					__lowerCAmelCase         =							temp
				else:
					for key in tensors.keys():
						# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
						if any(key.endswith(UpperCamelCase  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
							tensors[key] += temp[key]
						else:
							# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
							__lowerCAmelCase         =							1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN  ) else 0
							# We concatenate these weights accross TP ranks
							__lowerCAmelCase         =							torch.cat([tensors[key], temp[key]]			,							dim=UpperCamelCase  )

            # Divide by the number of TP the weights we want to average
			for key in tensors.keys():
				if any(key.endswith(UpperCamelCase  ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH  ):
					__lowerCAmelCase         =							tensors[key] / pretraining_tp

			__lowerCAmelCase         =							model.load_state_dict(UpperCamelCase			,							strict=UpperCamelCase  )
			assert not other_keys.unexpected_keys, F"The keys {other_keys.unexpected_keys} are unexpected"
			if missing_keys is None:
				__lowerCAmelCase         =							set(other_keys.missing_keys  )
			else:
				__lowerCAmelCase         =							missing_keys.intersection(set(other_keys.missing_keys  )  )

		assert not missing_keys, F"The keys {missing_keys} are missing"

		# Save pytorch-model
		os.makedirs(UpperCamelCase			,							exist_ok=UpperCamelCase  )
		__lowerCAmelCase         =							pytorch_dump_folder_path + "/" + WEIGHTS_NAME
		__lowerCAmelCase         =							pytorch_dump_folder_path + "/" + CONFIG_NAME
		print(F"Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}"  )
		if config.torch_dtype is not None:
			__lowerCAmelCase         =							model.to(config.torch_dtype  )
		torch.save(model.state_dict()			,							UpperCamelCase  )
		print(F"Save configuration file to {pytorch_config_dump_path}"  )
		with open(UpperCamelCase			,							"w"			,							encoding="utf-8"  ) as f:
			f.write(config.to_json_string()  )


if __name__ == "__main__":
					UpperCamelCase_   	=	argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    "--bloom_checkpoint_path",
					    default=None,
					    type=str,
					    required=True,
					    help="Path to the Megatron-LM checkpoint path.",
					)
					parser.add_argument(
					    "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
					)
					parser.add_argument(
					    "--bloom_config_file",
					    default="",
					    type=str,
					    help=(
					        "An optional config json file corresponding to the pre-trained model. \n"
					        "This specifies the model architecture."
					    ),
					)
					parser.add_argument(
					    "--shard_model",
					    action="store_true",
					    help="An optional setting to shard the output model \nThis enables sharding the converted checkpoint",
					)
					parser.add_argument(
					    "--pretraining_tp",
					    default=4,
					    type=int,
					    help="Pretraining TP rank that has been used when training the model in Megatron-LM \n",
					)
					UpperCamelCase_   	=	parser.parse_args()
					convert_bloom_checkpoint_to_pytorch(
					    args.bloom_checkpoint_path,
					    args.bloom_config_file,
					    args.pytorch_dump_folder_path,
					    args.shard_model,
					    args.pretraining_tp,
					)








from copy import deepcopy
from typing import Optional, Union

import numpy as np

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, is_tf_available, is_torch_available


if is_torch_available():
					import torch

if is_tf_available():
					import tensorflow as tf

class       a		(   __UpperCAmelCase					):
							lowercase_		:						Tuple   							=  ['image_processor']
							lowercase_		:						Tuple   							=  'SamImageProcessor'







							def __init__(						self  :  List[str]      ,				snake_case__  :  Optional[Any] ):
								"""simple docstring"""







								super().__init__(snake_case__ )
								__lowerCAmelCase         =							self.image_processor
								__lowerCAmelCase         =							-10
								__lowerCAmelCase         =							self.image_processor.size["longest_edge"]







							def __call__(						self  :  Any      ,				snake_case__  :  Optional[int]=None      ,				snake_case__  :  Optional[int]=None      ,				snake_case__  :  str=None      ,				snake_case__  :  List[str]=None      ,				snake_case__  :  Optional[Union[str, TensorType]] = None      ,				**snake_case__  :  Any      ,				):
								"""simple docstring"""







								__lowerCAmelCase         =							self.image_processor(
								    snake_case__      ,				return_tensors=snake_case__      ,				**snake_case__      ,				)

								# pop arguments that are not used in the foward but used nevertheless
								__lowerCAmelCase         =							encoding_image_processor["original_sizes"]

								if hasattr(snake_case__      ,				"numpy" ):  # Checks if Torch or TF tensor
									__lowerCAmelCase         =							original_sizes.numpy()

								__lowerCAmelCase   ,						__lowerCAmelCase   ,						__lowerCAmelCase         =							self._check_and_preprocess_points(
								    input_points=snake_case__      ,				input_labels=snake_case__      ,				input_boxes=snake_case__      ,				)

								__lowerCAmelCase         =							self._normalize_and_convert(
								    snake_case__      ,				snake_case__      ,				input_points=snake_case__      ,				input_labels=snake_case__      ,				input_boxes=snake_case__      ,				return_tensors=snake_case__      ,				)

								return encoding_image_processor







							def       UpperCAmelCase__					(						self  :  Tuple      ,				snake_case__  :  Any      ,				snake_case__  :  Optional[Any]      ,				snake_case__  :  str=None      ,				snake_case__  :  Any=None      ,				snake_case__  :  Any=None      ,				snake_case__  :  List[str]="pt"      ,				):
								"""simple docstring"""







								if input_points is not None:
									if len(snake_case__ ) != len(snake_case__ ):
										__lowerCAmelCase         =							[
										    self._normalize_coordinates(self.target_size      ,				snake_case__      ,				original_sizes[0] ) for point in input_points
										]
									else:
										__lowerCAmelCase         =							[
										    self._normalize_coordinates(self.target_size      ,				snake_case__      ,				snake_case__ )
										    for point, original_size in zip(snake_case__      ,				snake_case__ )
										]
									# check that all arrays have the same shape
									if not all(point.shape == input_points[0].shape for point in input_points ):
										if input_labels is not None:
											__lowerCAmelCase   ,						__lowerCAmelCase         =							self._pad_points_and_labels(snake_case__      ,				snake_case__ )

									__lowerCAmelCase         =							np.array(snake_case__ )

								if input_labels is not None:
									__lowerCAmelCase         =							np.array(snake_case__ )

								if input_boxes is not None:
									if len(snake_case__ ) != len(snake_case__ ):
										__lowerCAmelCase         =							[
										    self._normalize_coordinates(self.target_size      ,				snake_case__      ,				original_sizes[0]      ,				is_bounding_box=snake_case__ )
										    for box in input_boxes
										]
									else:
										__lowerCAmelCase         =							[
										    self._normalize_coordinates(self.target_size      ,				snake_case__      ,				snake_case__      ,				is_bounding_box=snake_case__ )
										    for box, original_size in zip(snake_case__      ,				snake_case__ )
										]
									__lowerCAmelCase         =							np.array(snake_case__ )

								if input_boxes is not None:
									if return_tensors == "pt":
										__lowerCAmelCase         =							torch.from_numpy(snake_case__ )
										# boxes batch size of 1 by default
										__lowerCAmelCase         =							input_boxes.unsqueeze(1 ) if len(input_boxes.shape ) != 3 else input_boxes
									elif return_tensors == "tf":
										__lowerCAmelCase         =							tf.convert_to_tensor(snake_case__ )
										# boxes batch size of 1 by default
										__lowerCAmelCase         =							tf.expand_dims(snake_case__      ,				1 ) if len(input_boxes.shape ) != 3 else input_boxes
									encoding_image_processor.update({"input_boxes": input_boxes} )
								if input_points is not None:
									if return_tensors == "pt":
										__lowerCAmelCase         =							torch.from_numpy(snake_case__ )
										# point batch size of 1 by default
										__lowerCAmelCase         =							input_points.unsqueeze(1 ) if len(input_points.shape ) != 4 else input_points
									elif return_tensors == "tf":
										__lowerCAmelCase         =							tf.convert_to_tensor(snake_case__ )
										# point batch size of 1 by default
										__lowerCAmelCase         =							tf.expand_dims(snake_case__      ,				1 ) if len(input_points.shape ) != 4 else input_points
									encoding_image_processor.update({"input_points": input_points} )
								if input_labels is not None:
									if return_tensors == "pt":
										__lowerCAmelCase         =							torch.from_numpy(snake_case__ )
										# point batch size of 1 by default
										__lowerCAmelCase         =							input_labels.unsqueeze(1 ) if len(input_labels.shape ) != 3 else input_labels
									elif return_tensors == "tf":
										__lowerCAmelCase         =							tf.convert_to_tensor(snake_case__ )
										# point batch size of 1 by default
										__lowerCAmelCase         =							tf.expand_dims(snake_case__      ,				1 ) if len(input_labels.shape ) != 3 else input_labels
									encoding_image_processor.update({"input_labels": input_labels} )

								return encoding_image_processor







							def       UpperCAmelCase__					(						self  :  str      ,				snake_case__  :  Union[str, Any]      ,				snake_case__  :  Dict ):
								"""simple docstring"""







								__lowerCAmelCase         =							max([point.shape[0] for point in input_points] )
								__lowerCAmelCase         =							[]
								for i, point in enumerate(snake_case__ ):
									if point.shape[0] != expected_nb_points:
										__lowerCAmelCase         =							np.concatenate(
										    [point, np.zeros((expected_nb_points - point.shape[0], 2) ) + self.point_pad_value]      ,				axis=0 )
										__lowerCAmelCase         =							np.append(input_labels[i]      ,				[self.point_pad_value] )
									processed_input_points.append(snake_case__ )
								__lowerCAmelCase         =							processed_input_points
								return input_points, input_labels







							def       UpperCAmelCase__					(						self  :  Dict      ,				snake_case__  :  int      ,				snake_case__  :  np.ndarray      ,				snake_case__  :  List[str]      ,				snake_case__  :  Dict=False ):
								"""simple docstring"""







								__lowerCAmelCase   ,						__lowerCAmelCase         =							original_size
								__lowerCAmelCase   ,						__lowerCAmelCase         =							self.image_processor._get_preprocess_shape(snake_case__      ,				longest_edge=snake_case__ )
								__lowerCAmelCase         =							deepcopy(snake_case__ ).astype(snake_case__ )

								if is_bounding_box:
									__lowerCAmelCase         =							coords.reshape(-1      ,				2      ,				2 )

								__lowerCAmelCase         =							coords[..., 0] * (new_w / old_w)
								__lowerCAmelCase         =							coords[..., 1] * (new_h / old_h)

								if is_bounding_box:
									__lowerCAmelCase         =							coords.reshape(-1      ,				4 )

								return coords







							def       UpperCAmelCase__					(						self  :  List[str]      ,				snake_case__  :  Union[str, Any]=None      ,				snake_case__  :  int=None      ,				snake_case__  :  Any=None      ,				):
								"""simple docstring"""







								if input_points is not None:
									if hasattr(snake_case__      ,				"numpy" ):  # Checks for TF or Torch tensor
										__lowerCAmelCase         =							input_points.numpy().tolist()

									if not isinstance(snake_case__      ,				snake_case__ ) or not isinstance(input_points[0]      ,				snake_case__ ):
										raise ValueError("Input points must be a list of list of floating points." )
									__lowerCAmelCase         =							[np.array(snake_case__ ) for input_point in input_points]
								else:
									__lowerCAmelCase         =							None

								if input_labels is not None:
									if hasattr(snake_case__      ,				"numpy" ):
										__lowerCAmelCase         =							input_labels.numpy().tolist()

									if not isinstance(snake_case__      ,				snake_case__ ) or not isinstance(input_labels[0]      ,				snake_case__ ):
										raise ValueError("Input labels must be a list of list integers." )
									__lowerCAmelCase         =							[np.array(snake_case__ ) for label in input_labels]
								else:
									__lowerCAmelCase         =							None

								if input_boxes is not None:
									if hasattr(snake_case__      ,				"numpy" ):
										__lowerCAmelCase         =							input_boxes.numpy().tolist()

									if (
									    not isinstance(snake_case__      ,				snake_case__ )
									    or not isinstance(input_boxes[0]      ,				snake_case__ )
									    or not isinstance(input_boxes[0][0]      ,				snake_case__ )
									):
										raise ValueError("Input boxes must be a list of list of list of floating points." )
									__lowerCAmelCase         =							[np.array(snake_case__ ).astype(np.floataa ) for box in input_boxes]
								else:
									__lowerCAmelCase         =							None

								return input_points, input_labels, input_boxes







							@property
							def       UpperCAmelCase__					(						self  :  Union[str, Any] ):
								"""simple docstring"""







								__lowerCAmelCase         =							self.image_processor.model_input_names
								return list(dict.fromkeys(snake_case__ ) )







							def       UpperCAmelCase__					(						self  :  Optional[Any]      ,				*snake_case__  :  Dict      ,				**snake_case__  :  Union[str, Any] ):
								"""simple docstring"""







								return self.image_processor.post_process_masks(*snake_case__      ,				**snake_case__ )






