

'''simple docstring'''





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase_							=					logging.get_logger(__name__)
UpperCamelCase_							=					{"""vocab_file""": """spiece.model"""}

UpperCamelCase_							=					{
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

UpperCamelCase_							=					{
    """albert-base-v1""": 512,
    """albert-large-v1""": 512,
    """albert-xlarge-v1""": 512,
    """albert-xxlarge-v1""": 512,
    """albert-base-v2""": 512,
    """albert-large-v2""": 512,
    """albert-xlarge-v2""": 512,
    """albert-xxlarge-v2""": 512,
}

UpperCamelCase_							=					"""▁"""





class 		__SCREAMING_SNAKE_CASE       ( lowercase__			):
				lowerCamelCase_							=						VOCAB_FILES_NAMES
				lowerCamelCase_							=						PRETRAINED_VOCAB_FILES_MAP
				lowerCamelCase_							=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







				def __init__(					self    :		int		,				UpperCAmelCase__    :		List[str]		,				UpperCAmelCase__    :		str=True		,				UpperCAmelCase__    :		Union[str, Any]=True		,				UpperCAmelCase__    :		int=False		,				UpperCAmelCase__    :		Tuple="[CLS]"		,				UpperCAmelCase__    :		Any="[SEP]"		,				UpperCAmelCase__    :		Union[str, Any]="<unk>"		,				UpperCAmelCase__    :		Union[str, Any]="[SEP]"		,				UpperCAmelCase__    :		Optional[Any]="<pad>"		,				UpperCAmelCase__    :		str="[CLS]"		,				UpperCAmelCase__    :		Dict="[MASK]"		,				UpperCAmelCase__    :		Optional[Dict[str, Any]] = None		,				**UpperCAmelCase__    :		Optional[int]		,				):
										'''simple docstring'''
										lowercase   :       str								=(
										    AddedToken(UpperCAmelCase__		,				lstrip=UpperCAmelCase__		,				rstrip=UpperCAmelCase__		,				normalized=UpperCAmelCase__      )
										    if isinstance(UpperCAmelCase__		,				UpperCAmelCase__      )
										    else mask_token
										)

										lowercase   :       Optional[Any]								={} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    do_lower_case=UpperCAmelCase__		,				remove_space=UpperCAmelCase__		,				keep_accents=UpperCAmelCase__		,				bos_token=UpperCAmelCase__		,				eos_token=UpperCAmelCase__		,				unk_token=UpperCAmelCase__		,				sep_token=UpperCAmelCase__		,				pad_token=UpperCAmelCase__		,				cls_token=UpperCAmelCase__		,				mask_token=UpperCAmelCase__		,				sp_model_kwargs=self.sp_model_kwargs		,				**UpperCAmelCase__		,				)

										lowercase   :       int								=do_lower_case
										lowercase   :       List[str]								=remove_space
										lowercase   :       Any								=keep_accents
										lowercase   :       Dict								=vocab_file

										lowercase   :       int								=spm.SentencePieceProcessor(**self.sp_model_kwargs      )
										self.sp_model.Load(UpperCAmelCase__      )







				@property
				def    lowerCamelCase_  (					self    :		Optional[Any]      ):
										'''simple docstring'''
										return len(self.sp_model      )







				def    lowerCamelCase_  (					self    :		Optional[int]      ):
										'''simple docstring'''
										lowercase   :       Optional[Any]								={self.convert_ids_to_tokens(UpperCAmelCase__      ): i for i in range(self.vocab_size      )}
										vocab.update(self.added_tokens_encoder      )
										return vocab







				def __getstate__(					self    :		Optional[Any]      ):
										'''simple docstring'''
										lowercase   :       Union[str, Any]								=self.__dict__.copy()
										lowercase   :       str								=None
										return state







				def __setstate__(					self    :		Tuple		,				UpperCAmelCase__    :		Optional[Any]      ):
										'''simple docstring'''
										lowercase   :       Optional[Any]								=d

										# for backward compatibility
										if not hasattr(self		,				'''sp_model_kwargs'''      ):
																lowercase   :       Union[str, Any]								={}

										lowercase   :       int								=spm.SentencePieceProcessor(**self.sp_model_kwargs      )
										self.sp_model.Load(self.vocab_file      )







				def    lowerCamelCase_  (					self    :		Any		,				UpperCAmelCase__    :		Optional[Any]      ):
										'''simple docstring'''
										if self.remove_space:
																lowercase   :       Tuple								=''' '''.join(inputs.strip().split()      )
										else:
																lowercase   :       Optional[int]								=inputs
										lowercase   :       int								=outputs.replace('''``'''		,				'''"'''      ).replace('''\'\''''		,				'''"'''      )

										if not self.keep_accents:
																lowercase   :       List[Any]								=unicodedata.normalize('''NFKD'''		,				UpperCAmelCase__      )
																lowercase   :       Optional[int]								=''''''.join([c for c in outputs if not unicodedata.combining(UpperCAmelCase__      )]      )
										if self.do_lower_case:
																lowercase   :       Any								=outputs.lower()

										return outputs







				def    lowerCamelCase_  (					self    :		str		,				UpperCAmelCase__    :		str      ):
										'''simple docstring'''
										lowercase   :       Optional[Any]								=self.preprocess_text(UpperCAmelCase__      )
										lowercase   :       Tuple								=self.sp_model.encode(UpperCAmelCase__		,				out_type=UpperCAmelCase__      )
										lowercase   :       Dict								=[]
										for piece in pieces:
																if len(UpperCAmelCase__      ) > 1 and piece[-1] == str(''','''      ) and piece[-2].isdigit():
																						lowercase   :       Tuple								=self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCAmelCase__		,				''''''      )      )
																						if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																												if len(cur_pieces[0]      ) == 1:
																																		lowercase   :       int								=cur_pieces[1:]
																												else:
																																		lowercase   :       Any								=cur_pieces[0][1:]
																						cur_pieces.append(piece[-1]      )
																						new_pieces.extend(UpperCAmelCase__      )
																else:
																						new_pieces.append(UpperCAmelCase__      )

										return new_pieces







				def    lowerCamelCase_  (					self    :		Tuple		,				UpperCAmelCase__    :		Dict      ):
										'''simple docstring'''
										return self.sp_model.PieceToId(UpperCAmelCase__      )







				def    lowerCamelCase_  (					self    :		Any		,				UpperCAmelCase__    :		List[Any]      ):
										'''simple docstring'''
										return self.sp_model.IdToPiece(UpperCAmelCase__      )







				def    lowerCamelCase_  (					self    :		int		,				UpperCAmelCase__    :		List[Any]      ):
										'''simple docstring'''
										lowercase   :       Any								=[]
										lowercase   :       str								=''''''
										lowercase   :       List[str]								=False
										for token in tokens:
																# make sure that special tokens are not decoded using sentencepiece model
																if token in self.all_special_tokens:
																						if not prev_is_special:
																												out_string += " "
																						out_string += self.sp_model.decode(UpperCAmelCase__      ) + token
																						lowercase   :       str								=True
																						lowercase   :       Tuple								=[]
																else:
																						current_sub_tokens.append(UpperCAmelCase__      )
																						lowercase   :       Optional[int]								=False
										out_string += self.sp_model.decode(UpperCAmelCase__      )
										return out_string.strip()







				def    lowerCamelCase_  (					self    :		List[str]		,				UpperCAmelCase__    :		List[int]		,				UpperCAmelCase__    :		Optional[List[int]] = None      ):
										'''simple docstring'''
										lowercase   :       Tuple								=[self.sep_token_id]
										lowercase   :       int								=[self.cls_token_id]
										if token_ids_a is None:
																return cls + token_ids_a + sep
										return cls + token_ids_a + sep + token_ids_a + sep







				def    lowerCamelCase_  (					self    :		List[Any]		,				UpperCAmelCase__    :		List[int]		,				UpperCAmelCase__    :		Optional[List[int]] = None		,				UpperCAmelCase__    :		bool = False      ):
										'''simple docstring'''

										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=UpperCAmelCase__		,				token_ids_a=UpperCAmelCase__		,				already_has_special_tokens=UpperCAmelCase__      )

										if token_ids_a is not None:
																return [1] + ([0] * len(UpperCAmelCase__      )) + [1] + ([0] * len(UpperCAmelCase__      )) + [1]
										return [1] + ([0] * len(UpperCAmelCase__      )) + [1]







				def    lowerCamelCase_  (					self    :		Optional[int]		,				UpperCAmelCase__    :		List[int]		,				UpperCAmelCase__    :		Optional[List[int]] = None      ):
										'''simple docstring'''
										lowercase   :       Optional[Any]								=[self.sep_token_id]
										lowercase   :       Union[str, Any]								=[self.cls_token_id]

										if token_ids_a is None:
																return len(cls + token_ids_a + sep      ) * [0]
										return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]







				def    lowerCamelCase_  (					self    :		Any		,				UpperCAmelCase__    :		str		,				UpperCAmelCase__    :		Optional[str] = None      ):
										'''simple docstring'''
										if not os.path.isdir(UpperCAmelCase__      ):
																logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''      )
																return
										lowercase   :       Union[str, Any]								=os.path.join(
										    UpperCAmelCase__		,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )

										if os.path.abspath(self.vocab_file      ) != os.path.abspath(UpperCAmelCase__      ) and os.path.isfile(self.vocab_file      ):
																copyfile(self.vocab_file		,				UpperCAmelCase__      )
										elif not os.path.isfile(self.vocab_file      ):
																with open(UpperCAmelCase__		,				'''wb'''      ) as fi:
																						lowercase   :       Any								=self.sp_model.serialized_model_proto()
																						fi.write(UpperCAmelCase__      )

										return (out_vocab_file,)








'''simple docstring'''





import argparse
import json
import os
from collections import OrderedDict

import numpy as np
import tensorflow as tf
import torch






def 							_lowerCAmelCase (      __magic_name__	:    str	)							->     Union[str, Any]:
						lowercase   :       Union[str, Any]								=os.path.join(args.tf_model_dir      ,   '''parameters.json'''	)
						lowercase   :       List[str]								=json.loads(open(__magic_name__	).read()	)
						if not params:
												raise ValueError(
												    f'''It seems that the json file at {parameter_file} is empty. Make sure you have a correct json file.'''	)
						if not args.output.endswith('''.pt'''	):
												lowercase   :       Tuple								=args.output + '''.pt'''
						lowercase   :       int								=OrderedDict()
						with tf.device('''/CPU:0'''	):
												lowercase   :       List[Any]								=tf.train.load_checkpoint(args.tf_model_dir	)
												lowercase   :       int								=reader.get_variable_to_shape_map()
												for key_name in shapes.keys():
																		lowercase   :       Any								=reader.get_tensor(__magic_name__	).astype(np.floataa	)
																		if key_name.endswith('''/adam_m'''	) or key_name.endswith('''/adam_v'''	):
																								continue
																		if key_name.startswith('''pasts/'''	):
																								if key_name.startswith('''pasts/mlp'''	):
																														lowercase   :       int								=int(key_name[9]	)
																								elif key_name.startswith('''pasts/out'''	):
																														lowercase   :       Union[str, Any]								=8
																								lowercase   :       Any								='''model.sqout.%d.weight''' % (player * 2)  # enter to nn.Sequencial with Tanh, so 2 at a time
																								lowercase   :       Dict								=vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																								lowercase   :       List[str]								=torch.tensor(__magic_name__	)
																		elif key_name.startswith('''model/moe'''	):
																								lowercase   :       Union[str, Any]								=int(key_name[9:].split('''/'''	)[0]	)
																								if key_name.endswith('''/switch_gating/kernel'''	):
																														lowercase   :       Dict								='''model.blocks.%d.feed_forward.mlp.router.classifier.weight''' % player
																														lowercase   :       Any								=vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																														lowercase   :       List[Any]								=torch.tensor(__magic_name__	)
																								elif key_name.endswith('''/softmlp/kernel'''	):
																														lowercase   :       Optional[int]								='''model.blocks.%d.feed_forward.soft_bypass_mlp.weight''' % player
																														lowercase   :       int								=vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																														lowercase   :       Optional[Any]								=torch.tensor(__magic_name__	)
																								elif key_name.endswith('''/wo/kernel'''	) or key_name.endswith('''/wi/kernel'''	):
																														lowercase   :       Union[str, Any]								=key_name[-9:-7]
																														for i in range(16	):
																																				lowercase   :       Dict								='''model.blocks.%d.feed_forward.mlp.experts.expert_%d.%s.weight''' % (player, i, nlayer)
																																				lowercase   :       Any								=(
																																				    vnp[i].transpose([1, 0]	).copy()
																																				)  # In Mesh-Tensorflow, it is one array, so it is divided
																																				lowercase   :       List[str]								=torch.tensor(__magic_name__	)
																		elif key_name.startswith('''model/mlp'''	):
																								lowercase   :       Dict								=int(key_name[9:].split('''/'''	)[0]	)
																								if key_name.endswith('''/p1/kernel'''	):
																														lowercase   :       Any								='''model.blocks.%d.feed_forward.mlp.wi.weight''' % player
																														lowercase   :       str								=vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																														lowercase   :       Any								=torch.tensor(__magic_name__	)
																								elif key_name.endswith('''/p1/bias'''	):
																														lowercase   :       List[Any]								='''model.blocks.%d.feed_forward.mlp.wi.bias''' % player
																														lowercase   :       Union[str, Any]								=vnp.copy()  # same because it is one dimensional
																														lowercase   :       Union[str, Any]								=torch.tensor(__magic_name__	)
																								elif key_name.endswith('''/p2/kernel'''	):
																														lowercase   :       int								='''model.blocks.%d.feed_forward.mlp.wo.weight''' % player
																														lowercase   :       Tuple								=vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																														lowercase   :       int								=torch.tensor(__magic_name__	)
																								elif key_name.endswith('''/p2/bias'''	):
																														lowercase   :       str								='''model.blocks.%d.feed_forward.mlp.wo.bias''' % player
																														lowercase   :       Optional[int]								=vnp.copy()  # same because it is one dimensional
																														lowercase   :       List[Any]								=torch.tensor(__magic_name__	)
																		elif key_name.startswith('''model/ln'''	):
																								lowercase   :       int								=int(key_name[8:].split('''/'''	)[0]	)
																								if key_name.endswith('''/b'''	):
																														lowercase   :       Any								='''model.blocks.%d.feed_forward.norm.bias''' % player
																														lowercase   :       Optional[int]								=vnp.copy()  # same because it is one dimensional
																														lowercase   :       Union[str, Any]								=torch.tensor(__magic_name__	)
																								elif key_name.endswith('''/g'''	):
																														lowercase   :       Optional[Any]								='''model.blocks.%d.feed_forward.norm.weight''' % player
																														lowercase   :       Any								=vnp.copy()  # same because it is one dimensional
																														lowercase   :       List[Any]								=torch.tensor(__magic_name__	)
																		elif key_name.startswith('''model/att'''	):
																								lowercase   :       int								=int(key_name[9:].split('''/'''	)[0]	)
																								if key_name.endswith('''/qkv/kernel'''	):
																														lowercase   :       Optional[int]								=vnp.copy()  # Compute same dimension as Mesh-tensorflow using einsum
																														lowercase   :       Dict								=state[:, 0, :, :]
																														lowercase   :       Tuple								=state[:, 1, :, :]
																														lowercase   :       List[Any]								=state[:, 2, :, :]
																														lowercase   :       Optional[int]								=(
																														    state_q.reshape([state_q.shape[0], state_q.shape[1] * state_q.shape[2]]	)
																														    .transpose([1, 0]	)
																														    .copy()
																														)  # Mesh-Tensorflow is a diagonal matrix
																														lowercase   :       Optional[Any]								=(
																														    state_k.reshape([state_k.shape[0], state_k.shape[1] * state_k.shape[2]]	)
																														    .transpose([1, 0]	)
																														    .copy()
																														)  # Mesh-Tensorflow is a diagonal matrix
																														lowercase   :       Optional[int]								=(
																														    state_v.reshape([state_v.shape[0], state_v.shape[1] * state_v.shape[2]]	)
																														    .transpose([1, 0]	)
																														    .copy()
																														)  # Mesh-Tensorflow is a diagonal matrix
																														lowercase   :       Dict								='''model.blocks.%d.self_attn.self_attn.q_proj.weight''' % player
																														lowercase   :       Dict								=torch.tensor(__magic_name__	)
																														lowercase   :       List[Any]								='''model.blocks.%d.self_attn.self_attn.k_proj.weight''' % player
																														lowercase   :       Optional[Any]								=torch.tensor(__magic_name__	)
																														lowercase   :       Optional[Any]								='''model.blocks.%d.self_attn.self_attn.v_proj.weight''' % player
																														lowercase   :       Tuple								=torch.tensor(__magic_name__	)
																								elif key_name.endswith('''/o/kernel'''	):
																														lowercase   :       Dict								='''model.blocks.%d.self_attn.self_attn.out_proj.weight''' % player
																														lowercase   :       List[Any]								=(
																														    vnp.reshape([vnp.shape[0] * vnp.shape[1], vnp.shape[2]]	).transpose([1, 0]	).copy()
																														)  # Mesh-Tensorflow is a diagonal matrix
																														lowercase   :       str								=torch.tensor(__magic_name__	)
																		elif key_name.startswith('''model/an'''	):
																								lowercase   :       Optional[Any]								=int(key_name[8:].split('''/'''	)[0]	)
																								if key_name.endswith('''/b'''	):
																														lowercase   :       List[str]								='''model.blocks.%d.self_attn.norm.bias''' % player
																														lowercase   :       Union[str, Any]								=vnp.copy()  # same because it is one dimensional
																														lowercase   :       List[str]								=torch.tensor(__magic_name__	)
																								elif key_name.endswith('''/g'''	):
																														lowercase   :       Any								='''model.blocks.%d.self_attn.norm.weight''' % player
																														lowercase   :       Any								=vnp.copy()  # same because it is one dimensional
																														lowercase   :       Optional[Any]								=torch.tensor(__magic_name__	)
																		elif (
																		    key_name.startswith('''model/wte'''	)
																		    or key_name.startswith('''model/wpe'''	)
																		    or key_name.startswith('''model/ete'''	)
																		):
																								lowercase   :       Any								={'''wte''': '''embed_tokens''', '''wpe''': '''position_embeddings''', '''ete''': '''extra_position_embeddings'''}[
																								    key_name[-3:]
																								]
																								lowercase   :       Optional[Any]								='''model.%s.weight''' % nlayer
																								lowercase   :       Optional[int]								=vnp.copy()  # same in embedded
																								lowercase   :       List[Any]								=torch.tensor(__magic_name__	)
																								if key_name.startswith('''model/wte'''	):
																														lowercase   :       Tuple								='''lm_head.weight'''
																														lowercase   :       str								=vnp.copy()  # same in embedded
																														lowercase   :       Union[str, Any]								=torch.tensor(__magic_name__	)
																		elif key_name.startswith('''model/wob'''	):
																								lowercase   :       List[str]								='''final_logits_bias'''
																								lowercase   :       Dict								=vnp.copy()  # same in embedded
																								lowercase   :       Tuple								=state.reshape((1, -1)	)
																								lowercase   :       Dict								=torch.tensor(__magic_name__	)
																		elif key_name == "model/dense/kernel":
																								lowercase   :       Dict								='''model.last_project.weight'''
																								lowercase   :       int								=vnp.transpose([1, 0]	).copy()  # Mesh-Tensorflow is a diagonal matrix
																								lowercase   :       Optional[Any]								=torch.tensor(__magic_name__	)
																		elif key_name == "model/dense_1/bias":
																								lowercase   :       List[Any]								='''model.last_project.bias'''
																								lowercase   :       str								=vnp.copy()  # same because it is one dimensional
																								lowercase   :       Optional[Any]								=torch.tensor(__magic_name__	)
						torch.save(__magic_name__      ,   args.output	)


if __name__ == "__main__":
							UpperCamelCase_							=					argparse.ArgumentParser(
							    description="""model converter.""", formatter_class=argparse.ArgumentDefaultsHelpFormatter
							)
							parser.add_argument("""--tf_model_dir""", metavar="""PATH""", type=str, required=True, help="""import model""")
							parser.add_argument("""--output""", metavar="""PATH""", type=str, required=True, help="""output model""")
							UpperCamelCase_							=					parser.parse_args()
							convert_tf_gptsan_to_pt(args)





