

import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

A_			:					Any    			=     [
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

A_			:					Optional[Any]    			=     [
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]

def 		UpperCamelCase   (lowercase_:					List[Any]       , lowercase_:					List[str]   )	-> Union[str, Any]:
	A__				:       List[Any]	  =						{
	    """word_embeddings.weight""": """word_embeddings.weight""",
	    """word_embeddings.norm.weight""": """word_embeddings_layernorm.weight""",
	    """word_embeddings.norm.bias""": """word_embeddings_layernorm.bias""",
	    """weight""": """ln_f.weight""",
	    """bias""": """ln_f.bias""",
	}

	if key in layer_rename_map:
		return layer_rename_map[key]

	# Handle transformer blocks
	A__				:       Optional[int]	  =						int(re.match(r""".*layer_(\d*).*"""       , lowercase_   )[1]   )
	layer_number -= 3
	return f"""h.{layer_number}.""" + key

def 		UpperCamelCase   (lowercase_:					str   )	-> Union[str, Any]:
	if dtype == torch.bool:
		return 1 / 8
	A__				:       Dict	  =						re.search(r"""[^\d](\d+)$"""       , str(lowercase_   )   )
	if bit_search is None:
		raise ValueError(f"""`dtype` is not a valid dtype: {dtype}."""   )
	A__				:       str	  =						int(bit_search.groups()[0]   )
	return bit_size // 8

def 		UpperCamelCase   (lowercase_:					List[Any]       , lowercase_:					Optional[int]       , lowercase_:					Dict       , lowercase_:					Optional[Any]       , lowercase_:					Optional[int]   )	-> Optional[Any]:
	# Construct model
	if bloom_config_file == "":
		A__				:       Any	  =						BloomConfig()
	else:
		A__				:       Optional[int]	  =						BloomConfig.from_json_file(lowercase_   )

	if shard_model:
		A__				:       int	  =						os.listdir(lowercase_   )
		A__				:       Optional[Any]	  =						sorted(filter(lambda lowercase_   : s.startswith("""layer"""   ) and "model_00" in s       , lowercase_   )   )

		A__				:       List[Any]	  =						{"""weight_map""": {}, """metadata""": {}}
		A__				:       int	  =						0

		A__				:       Optional[int]	  =						None

		A__				:       str	  =						BloomConfig()

		for j, file in enumerate(lowercase_   ):
			print("""Processing file: {}""".format(lowercase_   )   )
			A__				:       Union[str, Any]	  =						None

			for i in range(lowercase_   ):
				# load all TP files
				A__				:       Union[str, Any]	  =						file.replace("""model_00"""       , f"""model_0{i}"""   )
				A__				:       int	  =						torch.load(os.path.join(lowercase_       , lowercase_   )       , map_location="""cpu"""   )

				# Rename keys in the transformers names
				A__				:       Optional[int]	  =						list(temp.keys()   )
				for key in keys:
					A__				:       List[str]	  =						temp.pop(lowercase_   )

				if tensors is None:
					A__				:       Any	  =						temp
				else:
					for key in tensors.keys():
						if any(key.endswith(lowercase_   ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH   ):
							# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
							tensors[key] += temp[key]
						else:
							# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
							A__				:       int	  =						1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN   ) else 0
							# We concatenate these weights accross TP ranks
							A__				:       Dict	  =						torch.cat([tensors[key], temp[key]]       , dim=lowercase_   )

            # Divide by the number of TP the weights we want to average
			for key in tensors.keys():
				if any(key.endswith(lowercase_   ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH   ):
					A__				:       Optional[Any]	  =						tensors[key] / pretraining_tp
			torch.save(
			    lowercase_       , os.path.join(
			        lowercase_       , """pytorch_model_{}-of-{}.bin""".format(str(j + 1   ).zfill(5   )       , str(len(lowercase_   )   ).zfill(5   )   )       , )       , )

			for key in tensors.keys():
				A__				:       Optional[Any]	  =						tensors[key]
				total_size += value.numel() * get_dtype_size(value.dtype   )
				if key not in index_dict["weight_map"]:
					A__				:       Optional[Any]	  =						"""pytorch_model_{}-of-{}.bin""".format(
					    str(j + 1   ).zfill(5   )       , str(len(lowercase_   )   ).zfill(5   )   )

		A__				:       Tuple	  =						BloomConfig()
		A__				:       List[Any]	  =						pytorch_dump_folder_path + """/""" + CONFIG_NAME
		A__				:       Optional[Any]	  =						total_size
		with open(lowercase_       , """w"""       , encoding="""utf-8"""   ) as f:
			f.write(config.to_json_string()   )
		with open(os.path.join(lowercase_       , WEIGHTS_NAME + """.index.json"""   )       , """w"""       , encoding="""utf-8"""   ) as f:
			A__				:       int	  =						json.dumps(lowercase_       , indent=2       , sort_keys=lowercase_   ) + """\n"""
			f.write(lowercase_   )
	else:
		A__				:       Dict	  =						BloomModel(lowercase_   )

		A__				:       Union[str, Any]	  =						os.listdir(lowercase_   )
		A__				:       Tuple	  =						sorted(filter(lambda lowercase_   : s.startswith("""layer"""   ) and "model_00" in s       , lowercase_   )   )

		A__				:       str	  =						None
		for i, file in enumerate(lowercase_   ):
			A__				:       Dict	  =						None
			for i in range(lowercase_   ):
				# load all TP files
				A__				:       Dict	  =						file.replace("""model_00"""       , f"""model_0{i}"""   )
				A__				:       Union[str, Any]	  =						torch.load(os.path.join(lowercase_       , lowercase_   )       , map_location="""cpu"""   )

				# Rename keys in the transformers names
				A__				:       List[str]	  =						list(temp.keys()   )
				for key in keys:
					A__				:       Optional[Any]	  =						temp.pop(lowercase_   )

				if tensors is None:
					A__				:       List[str]	  =						temp
				else:
					for key in tensors.keys():
						# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
						if any(key.endswith(lowercase_   ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH   ):
							tensors[key] += temp[key]
						else:
							# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
							A__				:       int	  =						1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN   ) else 0
							# We concatenate these weights accross TP ranks
							A__				:       Optional[Any]	  =						torch.cat([tensors[key], temp[key]]       , dim=lowercase_   )

            # Divide by the number of TP the weights we want to average
			for key in tensors.keys():
				if any(key.endswith(lowercase_   ) for end in WEIGHTS_TO_AVERAGE_ENDSWITH   ):
					A__				:       List[Any]	  =						tensors[key] / pretraining_tp

			A__				:       List[Any]	  =						model.load_state_dict(lowercase_       , strict=lowercase_   )
			assert not other_keys.unexpected_keys, f"""The keys {other_keys.unexpected_keys} are unexpected"""
			if missing_keys is None:
				A__				:       str	  =						set(other_keys.missing_keys   )
			else:
				A__				:       Optional[Any]	  =						missing_keys.intersection(set(other_keys.missing_keys   )   )

		assert not missing_keys, f"""The keys {missing_keys} are missing"""

		# Save pytorch-model
		os.makedirs(lowercase_       , exist_ok=lowercase_   )
		A__				:       List[str]	  =						pytorch_dump_folder_path + """/""" + WEIGHTS_NAME
		A__				:       Optional[int]	  =						pytorch_dump_folder_path + """/""" + CONFIG_NAME
		print(f"""Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}"""   )
		if config.torch_dtype is not None:
			A__				:       Any	  =						model.to(config.torch_dtype   )
		torch.save(model.state_dict()       , lowercase_   )
		print(f"""Save configuration file to {pytorch_config_dump_path}"""   )
		with open(lowercase_       , """w"""       , encoding="""utf-8"""   ) as f:
			f.write(config.to_json_string()   )


if __name__ == "__main__":
	A_			:					Optional[int]    			=     argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '--bloom_checkpoint_path',
	    default=None,
	    type=str,
	    required=True,
	    help='Path to the Megatron-LM checkpoint path.',
	)
	parser.add_argument(
	    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
	)
	parser.add_argument(
	    '--bloom_config_file',
	    default='',
	    type=str,
	    help=(
	        'An optional config json file corresponding to the pre-trained model. \n'
	        'This specifies the model architecture.'
	    ),
	)
	parser.add_argument(
	    '--shard_model',
	    action='store_true',
	    help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
	)
	parser.add_argument(
	    '--pretraining_tp',
	    default=4,
	    type=int,
	    help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
	)
	A_			:					Union[str, Any]    			=     parser.parse_args()
	convert_bloom_checkpoint_to_pytorch(
	    args.bloom_checkpoint_path,
	    args.bloom_config_file,
	    args.pytorch_dump_folder_path,
	    args.shard_model,
	    args.pretraining_tp,
	)






import argparse
import copy

def 		UpperCamelCase   (lowercase_:					Any   )	-> Union[str, Any]:
	A__				:       int	  =						{}

	with open(lowercase_   ) as f:
		for line in f:
			if line.split()[0] not in dict_of_neighbours:
				A__				:       int	  =						[]
				_list.append([line.split()[1], line.split()[2]]   )
				A__				:       Any	  =						_list
			else:
				dict_of_neighbours[line.split()[0]].append(
				    [line.split()[1], line.split()[2]]   )
			if line.split()[1] not in dict_of_neighbours:
				A__				:       Optional[int]	  =						[]
				_list.append([line.split()[0], line.split()[2]]   )
				A__				:       Any	  =						_list
			else:
				dict_of_neighbours[line.split()[1]].append(
				    [line.split()[0], line.split()[2]]   )

	return dict_of_neighbours

def 		UpperCamelCase   (lowercase_:					Tuple       , lowercase_:					List[str]   )	-> Tuple:

	with open(lowercase_   ) as f:
		A__				:       List[Any]	  =						f.read(1   )
	A__				:       Union[str, Any]	  =						start_node

	A__				:       List[Any]	  =						[]

	A__				:       str	  =						start_node

	A__				:       List[str]	  =						0
	while visiting not in first_solution:
		A__				:       Dict	  =						10000
		for k in dict_of_neighbours[visiting]:
			if int(k[1]   ) < int(lowercase_   ) and k[0] not in first_solution:
				A__				:       Tuple	  =						k[1]
				A__				:       Optional[Any]	  =						k[0]

		first_solution.append(lowercase_   )
		A__				:       Optional[int]	  =						distance_of_first_solution + int(lowercase_   )
		A__				:       str	  =						best_node

	first_solution.append(lowercase_   )

	A__				:       Any	  =						0
	for k in dict_of_neighbours[first_solution[-2]]:
		if k[0] == start_node:
			break
		position += 1

	A__				:       int	  =						(
	    distance_of_first_solution
	    + int(dict_of_neighbours[first_solution[-2]][position][1]   )
	    - 10000
	)
	return first_solution, distance_of_first_solution

def 		UpperCamelCase   (lowercase_:					Any       , lowercase_:					str   )	-> Any:
	A__				:       int	  =						[]

	for n in solution[1:-1]:
		A__				:       Union[str, Any]	  =						solution.index(lowercase_   )
		for kn in solution[1:-1]:
			A__				:       List[Any]	  =						solution.index(lowercase_   )
			if n == kn:
				continue

			A__				:       Union[str, Any]	  =						copy.deepcopy(lowercase_   )
			A__				:       int	  =						kn
			A__				:       Optional[int]	  =						n

			A__				:       Dict	  =						0

			for k in _tmp[:-1]:
				A__				:       Dict	  =						_tmp[_tmp.index(lowercase_   ) + 1]
				for i in dict_of_neighbours[k]:
					if i[0] == next_node:
						A__				:       List[Any]	  =						distance + int(i[1]   )
			_tmp.append(lowercase_   )

			if _tmp not in neighborhood_of_solution:
				neighborhood_of_solution.append(_tmp   )

	A__				:       Optional[Any]	  =						len(neighborhood_of_solution[0]   ) - 1

	neighborhood_of_solution.sort(key=lambda lowercase_   : x[index_of_last_item_in_the_list]   )
	return neighborhood_of_solution

def 		UpperCamelCase   (lowercase_:					List[str]       , lowercase_:					Tuple       , lowercase_:					Dict       , lowercase_:					Dict       , lowercase_:					Tuple   )	-> Union[str, Any]:
	A__				:       Union[str, Any]	  =						1
	A__				:       List[str]	  =						first_solution
	A__				:       Dict	  =						[]
	A__				:       Union[str, Any]	  =						distance_of_first_solution
	A__				:       Any	  =						solution

	while count <= iters:
		A__				:       Optional[int]	  =						find_neighborhood(lowercase_       , lowercase_   )
		A__				:       Union[str, Any]	  =						0
		A__				:       str	  =						neighborhood[index_of_best_solution]
		A__				:       int	  =						len(lowercase_   ) - 1

		A__				:       List[str]	  =						False
		while not found:
			A__				:       Dict	  =						0
			while i < len(lowercase_   ):
				if best_solution[i] != solution[i]:
					A__				:       Any	  =						best_solution[i]
					A__				:       Optional[int]	  =						solution[i]
					break
				A__				:       Any	  =						i + 1

			if [first_exchange_node, second_exchange_node] not in tabu_list and [
			    second_exchange_node,
			    first_exchange_node,
			] not in tabu_list:
				tabu_list.append([first_exchange_node, second_exchange_node]   )
				A__				:       List[str]	  =						True
				A__				:       List[Any]	  =						best_solution[:-1]
				A__				:       Optional[Any]	  =						neighborhood[index_of_best_solution][best_cost_index]
				if cost < best_cost:
					A__				:       Tuple	  =						cost
					A__				:       int	  =						solution
			else:
				A__				:       List[Any]	  =						index_of_best_solution + 1
				A__				:       Tuple	  =						neighborhood[index_of_best_solution]

		if len(lowercase_   ) >= size:
			tabu_list.pop(0   )

		A__				:       Any	  =						count + 1

	return best_solution_ever, best_cost

def 		UpperCamelCase   (lowercase_:					Union[str, Any]=None   )	-> List[str]:
	A__				:       Union[str, Any]	  =						generate_neighbours(args.File   )

	A__  ,  A__				:       int	  =						generate_first_solution(
	    args.File       , lowercase_   )

	A__  ,  A__				:       str	  =						tabu_search(
	    lowercase_       , lowercase_       , lowercase_       , args.Iterations       , args.Size       , )

	print(f"""Best solution: {best_sol}, with total distance: {best_cost}."""   )


if __name__ == "__main__":
	A_			:					str    			=     argparse.ArgumentParser(description='Tabu Search')
	parser.add_argument(
	    '-f',
	    '--File',
	    type=str,
	    help='Path to the file containing the data',
	    required=True,
	)
	parser.add_argument(
	    '-i',
	    '--Iterations',
	    type=int,
	    help='How many iterations the algorithm should perform',
	    required=True,
	)
	parser.add_argument(
	    '-s', '--Size', type=int, help='Size of the tabu list', required=True
	)

	# Pass the arguments to main method
	main(parser.parse_args())



