



'''simple docstring'''

import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
	import torch


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


a							   =				logging.get_logger(__name__)
a							   =				{'vocab_file': 'spiece.model'}

a							   =				{
    'vocab_file': {
        'AI-Sweden/gpt-sw3-126m': 'https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-350m': 'https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-1.6b': 'https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-6.7b': 'https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-20b': 'https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model',
    }
}

a							   =				{
    'AI-Sweden/gpt-sw3-126m': 2_048,
    'AI-Sweden/gpt-sw3-350m': 2_048,
    'AI-Sweden/gpt-sw3-1.6b': 2_048,
    'AI-Sweden/gpt-sw3-6.7b': 2_048,
    'AI-Sweden/gpt-sw3-20b': 2_048,
}






class 						a_		(     snake_case    ):
							UpperCAmelCase			:		Optional[Any]        =				VOCAB_FILES_NAMES
							UpperCAmelCase			:		Tuple        =				PRETRAINED_VOCAB_FILES_MAP
							UpperCAmelCase			:		Tuple        =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCAmelCase			:		List[Any]        =				["""input_ids""", """attention_mask"""]


							def __init__(				self   :						Union[str, Any]      ,						a_   :						Tuple      ,						a_   :						Union[str, Any]=False      ,						a_   :						Optional[Any]=False      ,						a_   :						Optional[int]=False      ,						a_   :						int=None      ,						a_   :						List[str]=None      ,						a_   :						Any=None      ,						a_   :						str=None      ,						a_   :						Optional[Dict[str, Any]] = None      ,						**a_   :						Any      ,						)			->					None:
										snake_case:    List[Any]      							={} if sp_model_kwargs is None else sp_model_kwargs

										snake_case:    List[Any]      							=kwargs.get('name_or_path'			)
										if name_or_path is None:
													logger.warning(
													    'name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'
													    ' you are testing the model, this can safely be ignored'			)
													snake_case:    Any      							='None'

										# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
										snake_case:    List[str]      							='<|endoftext|>' if eos_token is None else eos_token
										snake_case:    List[Any]      							='<unk>' if unk_token is None else unk_token
										if "gpt-sw3-7b" in name_or_path:
													snake_case:    List[str]      							=unk_token if pad_token is None else pad_token
													snake_case:    Dict      							=eos_token if bos_token is None else bos_token
										else:
													snake_case:    str      							='<pad>' if pad_token is None else pad_token
													snake_case:    Optional[int]      							='<s>' if bos_token is None else bos_token

										super().__init__(
										    do_lower_case=a_      ,						remove_space=a_      ,						keep_accents=a_      ,						bos_token=a_      ,						eos_token=a_      ,						unk_token=a_      ,						pad_token=a_      ,						sp_model_kwargs=self.sp_model_kwargs      ,						**a_      ,						)

										snake_case:    Any      							=do_lower_case
										snake_case:    Optional[Any]      							=remove_space
										snake_case:    Union[str, Any]      							=keep_accents
										snake_case:    List[Any]      							=vocab_file

										snake_case:    int      							=spm.SentencePieceProcessor(**self.sp_model_kwargs			)
										self.sp_model.Load(a_			)

										# Used for whitespace normalization in input texts
										# fmt : off
										snake_case:    int      							={' ', ' ', ' ', ' ', ' ', '　', ' ', ' ', ' ', ' ', '￼', ''}
										# fmt : on

										# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
										snake_case:    Optional[int]      							=re.compile(
										    F'''[{"".join(map(a_      ,						list(range(0      ,						9			)			) + list(range(1_1      ,						3_2			)			) + list(range(1_2_7      ,						1_6_0			)			) + [1_6_0, 1_7_3, 8_2_0_3]			)			)}]'''			)


							def __getstate__(				self   :						str			)			->					Optional[Any]:
										snake_case:    List[str]      							=self.__dict__.copy()
										snake_case:    List[str]      							=None
										return state


							def __setstate__(				self   :						List[str]      ,						a_   :						List[Any]			)			->					str:
										snake_case:    Tuple      							=d

										# for backward compatibility
										if not hasattr(self      ,						'sp_model_kwargs'			):
													snake_case:    Optional[Any]      							={}

										snake_case:    int      							=spm.SentencePieceProcessor(**self.sp_model_kwargs			)
										self.sp_model.Load(self.vocab_file			)


							@property
							# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
							def    UpperCamelCase     (				self   :						Union[str, Any]			)			->					int:
										return len(self.sp_model			)


							def    UpperCamelCase     (				self   :						List[str]      ,						a_   :						str			)			->					str:
										snake_case:    Optional[Any]      							=self.non_printing_characters_re.sub(''      ,						a_			)

										# Normalize whitespaces
										snake_case:    List[str]      							=''.join([char if char not in self.whitespaces else ' ' for char in text]			)

										# NFC Unicode normalization
										snake_case:    Any      							=unicodedata.normalize('NFC'      ,						a_			)
										return text


							def    UpperCamelCase     (				self   :						str      ,						a_   :						str      ,						**a_   :						Union[str, Any]			)			->					List[str]:
										snake_case:    Optional[int]      							=self.preprocess_text(a_			)
										return self.sp_model.encode(a_      ,						out_type=a_			)


							def    UpperCamelCase     (				self   :						Tuple      ,						a_   :						str			)			->					int:
										return self.sp_model.PieceToId(a_			)


							def    UpperCamelCase     (				self   :						int      ,						a_   :						int			)			->					str:
										return self.sp_model.IdToPiece(a_			)


							@staticmethod
							def    UpperCamelCase     (				a_   :						str			)			->					str:
										return out_string


							def    UpperCamelCase     (				self   :						List[Any]      ,						a_   :						List[str]			)			->					str:
										snake_case:    List[Any]      							=[]
										snake_case:    Dict      							=''
										snake_case:    int      							=False
										for token in tokens:
													# make sure that special tokens are not decoded using sentencepiece model
													if token in self.all_special_tokens:
																# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																if not prev_is_special:
																			out_string += " "

																out_string += self.sp_model.decode(a_			) + token
																snake_case:    Dict      							=True
																snake_case:    Optional[int]      							=[]
													else:
																current_sub_tokens.append(a_			)
																snake_case:    Optional[Any]      							=False
										out_string += self.sp_model.decode(a_			)

										return out_string


							def    UpperCamelCase     (				self   :						List[str]			)			->					Dict[str, int]:
										snake_case:    List[str]      							={self.convert_ids_to_tokens(a_			): i for i in range(self.vocab_size			)}
										vocab.update(self.added_tokens_encoder			)
										return vocab


							def    UpperCamelCase     (				self   :						List[str]      ,						a_   :						str      ,						a_   :						Optional[str] = None			)			->					Tuple[str]:
										if not os.path.isdir(a_			):
													logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''			)
													return
										snake_case:    Tuple      							=os.path.join(
										    a_      ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)

										if os.path.abspath(self.vocab_file			) != os.path.abspath(a_			) and os.path.isfile(self.vocab_file			):
													copyfile(self.vocab_file      ,						a_			)
										elif not os.path.isfile(self.vocab_file			):
													with open(a_      ,						'wb'			) as fi:
																snake_case:    Tuple      							=self.sp_model.serialized_model_proto()
																fi.write(a_			)

										return (out_vocab_file,)


							def    UpperCamelCase     (				self   :						Any      ,						a_   :						Union[str, List[str]]      ,						a_   :						Union[str, bool] = False			)			->					Union[List[int], List[List[int]], "torch.Tensor"]:

										if isinstance(a_      ,						a_			):
													snake_case:    Tuple      							=self.preprocess_text(a_			)
													snake_case:    str      							=self.sp_model.encode(a_			)
										else:
													snake_case:    str      							=[self.preprocess_text(a_			) for t in text]
													snake_case:    Union[str, Any]      							=self.sp_model.encode(a_			)

										if return_tensors is True or return_tensors == "pt":
													snake_case:    Dict      							=torch.tensor(a_			)

										return token_ids


							def    UpperCamelCase     (				self   :						Dict      ,						a_   :						Union[int, List[int]]			)			->					str:
										return self.sp_model.decode(a_			)




							def    UpperCamelCase     (				self   :						Dict      ,						a_   :						"Conversation"			)			->					List[int]:
										snake_case:    Dict      							=[F'''User: {text}''' if is_user else F'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
										snake_case:    Union[str, Any]      							=(
										    F'''{self.eos_token}{self.bos_token}''' + F'''{self.bos_token}'''.join(a_			) + F'''{self.bos_token}Bot:'''
										)
										return self.encode(text=a_			)









'''simple docstring'''

import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

a							   =				[
    'word_embeddings_layernorm.weight',
    'word_embeddings_layernorm.bias',
    'input_layernorm.weight',
    'input_layernorm.bias',
    'post_attention_layernorm.weight',
    'post_attention_layernorm.bias',
    'self_attention.dense.bias',
    'mlp.dense_4h_to_h.bias',
    'ln_f.weight',
    'ln_f.bias',
]

a							   =				[
    'mlp.dense_4h_to_h.weight',
    'self_attention.dense.weight',
]







def a_     (    __UpperCAmelCase				,							__UpperCAmelCase					)				->					Any:





			"""simple docstring"""







			snake_case:    Any      							={
			    'word_embeddings.weight': 'word_embeddings.weight',
			    'word_embeddings.norm.weight': 'word_embeddings_layernorm.weight',
			    'word_embeddings.norm.bias': 'word_embeddings_layernorm.bias',
			    'weight': 'ln_f.weight',
			    'bias': 'ln_f.bias',
			}

			if key in layer_rename_map:
						return layer_rename_map[key]

			# Handle transformer blocks
			snake_case:    Dict      							=int(re.match(R'.*layer_(\d*).*'				,							__UpperCAmelCase					)[1]					)
			layer_number -= 3
			return f'''h.{layer_number}.''' + key







def a_     (    __UpperCAmelCase					)				->					Union[str, Any]:





			"""simple docstring"""







			if dtype == torch.bool:
						return 1 / 8
			snake_case:    Optional[Any]      							=re.search(R'[^\d](\d+)$'				,							str(__UpperCAmelCase					)					)
			if bit_search is None:
						raise ValueError(f'''`dtype` is not a valid dtype: {dtype}.'''					)
			snake_case:    str      							=int(bit_search.groups()[0]					)
			return bit_size // 8







def a_     (    __UpperCAmelCase				,							__UpperCAmelCase				,							__UpperCAmelCase				,							__UpperCAmelCase				,							__UpperCAmelCase					)				->					Tuple:





			"""simple docstring"""







			if bloom_config_file == "":
						snake_case:    Tuple      							=BloomConfig()
			else:
						snake_case:    List[Any]      							=BloomConfig.from_json_file(__UpperCAmelCase					)

			if shard_model:
						snake_case:    Any      							=os.listdir(__UpperCAmelCase					)
						snake_case:    Any      							=sorted(filter(lambda __UpperCAmelCase					: s.startswith('layer'					) and "model_00" in s				,							__UpperCAmelCase					)					)

						snake_case:    int      							={'weight_map': {}, 'metadata': {}}
						snake_case:    Dict      							=0

						snake_case:    List[str]      							=None

						snake_case:    Any      							=BloomConfig()

						for j, file in enumerate(__UpperCAmelCase					):
									print('Processing file: {}'.format(__UpperCAmelCase					)					)
									snake_case:    Any      							=None

									for i in range(__UpperCAmelCase					):
												# load all TP files
												snake_case:    Optional[int]      							=file.replace('model_00'				,							f'''model_0{i}'''					)
												snake_case:    List[str]      							=torch.load(os.path.join(__UpperCAmelCase				,							__UpperCAmelCase					)				,							map_location='cpu'					)

												# Rename keys in the transformers names
												snake_case:    int      							=list(temp.keys()					)
												for key in keys:
															snake_case:    List[Any]      							=temp.pop(__UpperCAmelCase					)

												if tensors is None:
															snake_case:    Tuple      							=temp
												else:
															for key in tensors.keys():
																		if any(key.endswith(__UpperCAmelCase					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
																					# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																					tensors[key] += temp[key]
																		else:
																					# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																					snake_case:    Optional[int]      							=1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN					) else 0
																					# We concatenate these weights accross TP ranks
																					snake_case:    str      							=torch.cat([tensors[key], temp[key]]				,							dim=__UpperCAmelCase					)

            # Divide by the number of TP the weights we want to average
									for key in tensors.keys():
												if any(key.endswith(__UpperCAmelCase					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
															snake_case:    Dict      							=tensors[key] / pretraining_tp
									torch.save(
									    __UpperCAmelCase				,							os.path.join(
									        __UpperCAmelCase				,							'pytorch_model_{}-of-{}.bin'.format(str(j + 1					).zfill(5					)				,							str(len(__UpperCAmelCase					)					).zfill(5					)					)				,							)				,							)

									for key in tensors.keys():
												snake_case:    List[str]      							=tensors[key]
												total_size += value.numel() * get_dtype_size(value.dtype					)
												if key not in index_dict["weight_map"]:
															snake_case:    List[Any]      							='pytorch_model_{}-of-{}.bin'.format(
															    str(j + 1					).zfill(5					)				,							str(len(__UpperCAmelCase					)					).zfill(5					)					)

						snake_case:    Tuple      							=BloomConfig()
						snake_case:    Union[str, Any]      							=pytorch_dump_folder_path + '/' + CONFIG_NAME
						snake_case:    Any      							=total_size
						with open(__UpperCAmelCase				,							'w'				,							encoding='utf-8'					) as f:
									f.write(config.to_json_string()					)
						with open(os.path.join(__UpperCAmelCase				,							WEIGHTS_NAME + '.index.json'					)				,							'w'				,							encoding='utf-8'					) as f:
									snake_case:    Tuple      							=json.dumps(__UpperCAmelCase				,							indent=2				,							sort_keys=__UpperCAmelCase					) + '\n'
									f.write(__UpperCAmelCase					)
			else:
						snake_case:    Optional[Any]      							=BloomModel(__UpperCAmelCase					)

						snake_case:    Any      							=os.listdir(__UpperCAmelCase					)
						snake_case:    Optional[int]      							=sorted(filter(lambda __UpperCAmelCase					: s.startswith('layer'					) and "model_00" in s				,							__UpperCAmelCase					)					)

						snake_case:    str      							=None
						for i, file in enumerate(__UpperCAmelCase					):
									snake_case:    List[str]      							=None
									for i in range(__UpperCAmelCase					):
												# load all TP files
												snake_case:    int      							=file.replace('model_00'				,							f'''model_0{i}'''					)
												snake_case:    Optional[Any]      							=torch.load(os.path.join(__UpperCAmelCase				,							__UpperCAmelCase					)				,							map_location='cpu'					)

												# Rename keys in the transformers names
												snake_case:    List[str]      							=list(temp.keys()					)
												for key in keys:
															snake_case:    int      							=temp.pop(__UpperCAmelCase					)

												if tensors is None:
															snake_case:    Any      							=temp
												else:
															for key in tensors.keys():
																		# We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
																		if any(key.endswith(__UpperCAmelCase					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
																					tensors[key] += temp[key]
																		else:
																					# Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
																					snake_case:    List[Any]      							=1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN					) else 0
																					# We concatenate these weights accross TP ranks
																					snake_case:    Tuple      							=torch.cat([tensors[key], temp[key]]				,							dim=__UpperCAmelCase					)

            # Divide by the number of TP the weights we want to average
									for key in tensors.keys():
												if any(key.endswith(__UpperCAmelCase					) for end in WEIGHTS_TO_AVERAGE_ENDSWITH					):
															snake_case:    int      							=tensors[key] / pretraining_tp

									snake_case:    Optional[Any]      							=model.load_state_dict(__UpperCAmelCase				,							strict=__UpperCAmelCase					)
									assert not other_keys.unexpected_keys, f'''The keys {other_keys.unexpected_keys} are unexpected'''
									if missing_keys is None:
												snake_case:    Optional[Any]      							=set(other_keys.missing_keys					)
									else:
												snake_case:    Any      							=missing_keys.intersection(set(other_keys.missing_keys					)					)

						assert not missing_keys, f'''The keys {missing_keys} are missing'''

						# Save pytorch-model
						os.makedirs(__UpperCAmelCase				,							exist_ok=__UpperCAmelCase					)
						snake_case:    Optional[int]      							=pytorch_dump_folder_path + '/' + WEIGHTS_NAME
						snake_case:    Any      							=pytorch_dump_folder_path + '/' + CONFIG_NAME
						print(f'''Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'''					)
						if config.torch_dtype is not None:
									snake_case:    Optional[Any]      							=model.to(config.torch_dtype					)
						torch.save(model.state_dict()				,							__UpperCAmelCase					)
						print(f'''Save configuration file to {pytorch_config_dump_path}'''					)
						with open(__UpperCAmelCase				,							'w'				,							encoding='utf-8'					) as f:
									f.write(config.to_json_string()					)


if __name__ == "__main__":
	a							   =				argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '--bloom_checkpoint_path',
	    default=None,
	    type=str,
	    required=True,
	    help='Path to the Megatron-LM checkpoint path.',
	)
	parser.add_argument(
	    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
	)
	parser.add_argument(
	    '--bloom_config_file',
	    default='',
	    type=str,
	    help=(
	        'An optional config json file corresponding to the pre-trained model. \n'
	        'This specifies the model architecture.'
	    ),
	)
	parser.add_argument(
	    '--shard_model',
	    action='store_true',
	    help='An optional setting to shard the output model \nThis enables sharding the converted checkpoint',
	)
	parser.add_argument(
	    '--pretraining_tp',
	    default=4,
	    type=int,
	    help='Pretraining TP rank that has been used when training the model in Megatron-LM \n',
	)
	a							   =				parser.parse_args()
	convert_bloom_checkpoint_to_pytorch(
	    args.bloom_checkpoint_path,
	    args.bloom_config_file,
	    args.pytorch_dump_folder_path,
	    args.shard_model,
	    args.pretraining_tp,
	)




