





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__											= logging.get_logger(__name__)
UpperCamelCase__											= {"vocab_file": "spiece.model"}

UpperCamelCase__											= {
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

UpperCamelCase__											= {
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

UpperCamelCase__											= "▁"

class        __SCREAMING_SNAKE_CASE					(     __SCREAMING_SNAKE_CASE       ):
							snake_case     :      Optional[Any]						     =       VOCAB_FILES_NAMES
							snake_case     :      int						     =       PRETRAINED_VOCAB_FILES_MAP
							snake_case     :      Dict						     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES






							def __init__(						self  , __lowerCAmelCase  , __lowerCAmelCase=True  , __lowerCAmelCase=True  , __lowerCAmelCase=False  , __lowerCAmelCase="[CLS]"  , __lowerCAmelCase="[SEP]"  , __lowerCAmelCase="<unk>"  , __lowerCAmelCase="[SEP]"  , __lowerCAmelCase="<pad>"  , __lowerCAmelCase="[CLS]"  , __lowerCAmelCase="[MASK]"  , __lowerCAmelCase = None  , **__lowerCAmelCase  , ):
														# Mask token behave like a normal word, i.e. include the space before it and
														# is included in the raw text, there should be a match in a non-normalized sentence.
														UpperCamelCase__				     =			(
														    AddedToken(_a  , lstrip=_a  , rstrip=_a  , normalized=_a  )
														    if isinstance(_a  , _a  )
														    else mask_token
														)

														UpperCamelCase__				     =			{} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    do_lower_case=_a  , remove_space=_a  , keep_accents=_a  , bos_token=_a  , eos_token=_a  , unk_token=_a  , sep_token=_a  , pad_token=_a  , cls_token=_a  , mask_token=_a  , sp_model_kwargs=self.sp_model_kwargs  , **_a  , )

														UpperCamelCase__				     =			do_lower_case
														UpperCamelCase__				     =			remove_space
														UpperCamelCase__				     =			keep_accents
														UpperCamelCase__				     =			vocab_file

														UpperCamelCase__				     =			spm.SentencePieceProcessor(**self.sp_model_kwargs  )
														self.sp_model.Load(_a  )






							@property
							def 				_lowerCamelCase		(						self  ):
														return len(self.sp_model  )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			{self.convert_ids_to_tokens(_a  ): i for i in range(self.vocab_size  )}
														vocab.update(self.added_tokens_encoder  )
														return vocab






							def __getstate__(						self  ):
														UpperCamelCase__				     =			self.__dict__.copy()
														UpperCamelCase__				     =			None
														return state






							def __setstate__(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			d

														# for backward compatibility
														if not hasattr(self  , """sp_model_kwargs"""  ):
																					UpperCamelCase__				     =			{}

														UpperCamelCase__				     =			spm.SentencePieceProcessor(**self.sp_model_kwargs  )
														self.sp_model.Load(self.vocab_file  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														if self.remove_space:
																					UpperCamelCase__				     =			""" """.join(inputs.strip().split()  )
														else:
																					UpperCamelCase__				     =			inputs
														UpperCamelCase__				     =			outputs.replace("""``"""  , """\""""  ).replace("""\'\'"""  , """\""""  )

														if not self.keep_accents:
																					UpperCamelCase__				     =			unicodedata.normalize("""NFKD"""  , _a  )
																					UpperCamelCase__				     =			"""""".join([c for c in outputs if not unicodedata.combining(_a  )]  )
														if self.do_lower_case:
																					UpperCamelCase__				     =			outputs.lower()

														return outputs






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			self.preprocess_text(_a  )
														UpperCamelCase__				     =			self.sp_model.encode(_a  , out_type=_a  )
														UpperCamelCase__				     =			[]
														for piece in pieces:
																					if len(_a  ) > 1 and piece[-1] == str(""","""  ) and piece[-2].isdigit():
																												UpperCamelCase__				     =			self.sp_model.EncodeAsPieces(piece[:-1].replace(_a  , """"""  )  )
																												if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																																			if len(cur_pieces[0]  ) == 1:
																																										UpperCamelCase__				     =			cur_pieces[1:]
																																			else:
																																										UpperCamelCase__				     =			cur_pieces[0][1:]
																												cur_pieces.append(piece[-1]  )
																												new_pieces.extend(_a  )
																					else:
																												new_pieces.append(_a  )

														return new_pieces






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														return self.sp_model.PieceToId(_a  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														return self.sp_model.IdToPiece(_a  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			[]
														UpperCamelCase__				     =			""""""
														UpperCamelCase__				     =			False
														for token in tokens:
																					# make sure that special tokens are not decoded using sentencepiece model
																					if token in self.all_special_tokens:
																												if not prev_is_special:
																																			out_string += " "
																												out_string += self.sp_model.decode(_a  ) + token
																												UpperCamelCase__				     =			True
																												UpperCamelCase__				     =			[]
																					else:
																												current_sub_tokens.append(_a  )
																												UpperCamelCase__				     =			False
														out_string += self.sp_model.decode(_a  )
														return out_string.strip()






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = None  ):
														UpperCamelCase__				     =			[self.sep_token_id]
														UpperCamelCase__				     =			[self.cls_token_id]
														if token_ids_a is None:
																					return cls + token_ids_a + sep
														return cls + token_ids_a + sep + token_ids_a + sep






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = None  , __lowerCAmelCase = False  ):

														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=_a  , token_ids_a=_a  , already_has_special_tokens=_a  )

														if token_ids_a is not None:
																					return [1] + ([0] * len(_a  )) + [1] + ([0] * len(_a  )) + [1]
														return [1] + ([0] * len(_a  )) + [1]






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = None  ):
														UpperCamelCase__				     =			[self.sep_token_id]
														UpperCamelCase__				     =			[self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep  ) * [0]
														return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = None  ):
														if not os.path.isdir(_a  ):
																					logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""  )
																					return
														UpperCamelCase__				     =			os.path.join(
														    _a  , (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )

														if os.path.abspath(self.vocab_file  ) != os.path.abspath(_a  ) and os.path.isfile(self.vocab_file  ):
																					copyfile(self.vocab_file  , _a  )
														elif not os.path.isfile(self.vocab_file  ):
																					with open(_a  , """wb"""  ) as fi:
																												UpperCamelCase__				     =			self.sp_model.serialized_model_proto()
																												fi.write(_a  )

														return (out_vocab_file,)








from typing import Any





def     _UpperCamelCase     (a__   :list       ):






							"""simple docstring"""





							if not input_list:
														return []
							UpperCamelCase__				     =			[input_list.count(a__       ) for value in input_list]
							UpperCamelCase__				     =			max(a__       )  # Gets the maximum count in the input list.
							# Gets values of modes
							return sorted({input_list[i] for i, value in enumerate(a__       ) if value == y}       )


if __name__ == "__main__":
					import doctest

					doctest.testmod()




