

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__A			:					List[str]						=			logging.get_logger(__name__)
__A			:					Dict						=			{'vocab_file': 'spiece.model'}

__A			:					int						=			{
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

__A			:					Union[str, Any]						=			{
    'albert-base-v1': 5_1_2,
    'albert-large-v1': 5_1_2,
    'albert-xlarge-v1': 5_1_2,
    'albert-xxlarge-v1': 5_1_2,
    'albert-base-v2': 5_1_2,
    'albert-large-v2': 5_1_2,
    'albert-xlarge-v2': 5_1_2,
    'albert-xxlarge-v2': 5_1_2,
}

__A			:					Optional[Any]						=			'▁'





class 			_SCREAMING_SNAKE_CASE (   _lowerCamelCase		):







   '''simple docstring'''




   lowerCamelCase__				=      VOCAB_FILES_NAMES
   lowerCamelCase__				=      PRETRAINED_VOCAB_FILES_MAP
   lowerCamelCase__				=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES





   def __init__(       self					:			Any     , __lowerCamelCase					:			Any     , __lowerCamelCase					:			Union[str, Any]=True     , __lowerCamelCase					:			Any=True     , __lowerCamelCase					:			List[str]=False     , __lowerCamelCase					:			int="[CLS]"     , __lowerCamelCase					:			Any="[SEP]"     , __lowerCamelCase					:			Optional[Any]="<unk>"     , __lowerCamelCase					:			Optional[Any]="[SEP]"     , __lowerCamelCase					:			Dict="<pad>"     , __lowerCamelCase					:			Any="[CLS]"     , __lowerCamelCase					:			Optional[Any]="[MASK]"     , __lowerCamelCase					:			Optional[Dict[str, Any]] = None     , **__lowerCamelCase					:			Any     , ):
       # Mask token behave like a normal word, i.e. include the space before it and
       # is included in the raw text, there should be a match in a non-normalized sentence.
       SCREAMING_SNAKE_CASE										=   (
           AddedToken(__UpperCamelCase     , lstrip=__UpperCamelCase     , rstrip=__UpperCamelCase     , normalized=__UpperCamelCase			)
           if isinstance(__UpperCamelCase     , __UpperCamelCase			)
           else mask_token
       )

       SCREAMING_SNAKE_CASE										=   {} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           do_lower_case=__UpperCamelCase     , remove_space=__UpperCamelCase     , keep_accents=__UpperCamelCase     , bos_token=__UpperCamelCase     , eos_token=__UpperCamelCase     , unk_token=__UpperCamelCase     , sep_token=__UpperCamelCase     , pad_token=__UpperCamelCase     , cls_token=__UpperCamelCase     , mask_token=__UpperCamelCase     , sp_model_kwargs=self.sp_model_kwargs     , **__UpperCamelCase     , )

       SCREAMING_SNAKE_CASE										=   do_lower_case
       SCREAMING_SNAKE_CASE										=   remove_space
       SCREAMING_SNAKE_CASE										=   keep_accents
       SCREAMING_SNAKE_CASE										=   vocab_file

       SCREAMING_SNAKE_CASE										=   spm.SentencePieceProcessor(**self.sp_model_kwargs			)
       self.sp_model.Load(__UpperCamelCase			)





   @property
   def      _snake_case		(       self					:			int			):
       return len(self.sp_model			)





   def      _snake_case		(       self					:			Dict			):
       SCREAMING_SNAKE_CASE										=   {self.convert_ids_to_tokens(__UpperCamelCase			): i for i in range(self.vocab_size			)}
       vocab.update(self.added_tokens_encoder			)
       return vocab





   def __getstate__(       self					:			Any			):
       SCREAMING_SNAKE_CASE										=   self.__dict__.copy()
       SCREAMING_SNAKE_CASE										=   None
       return state





   def __setstate__(       self					:			Optional[Any]     , __lowerCamelCase					:			Tuple			):
       SCREAMING_SNAKE_CASE										=   d

       # for backward compatibility
       if not hasattr(self     , "sp_model_kwargs"			):
           SCREAMING_SNAKE_CASE										=   {}

       SCREAMING_SNAKE_CASE										=   spm.SentencePieceProcessor(**self.sp_model_kwargs			)
       self.sp_model.Load(self.vocab_file			)





   def      _snake_case		(       self					:			int     , __lowerCamelCase					:			Union[str, Any]			):
       if self.remove_space:
           SCREAMING_SNAKE_CASE										=   " ".join(inputs.strip().split()			)
       else:
           SCREAMING_SNAKE_CASE										=   inputs
       SCREAMING_SNAKE_CASE										=   outputs.replace("``"     , "\""			).replace("\'\'"     , "\""			)

       if not self.keep_accents:
           SCREAMING_SNAKE_CASE										=   unicodedata.normalize("NFKD"     , __UpperCamelCase			)
           SCREAMING_SNAKE_CASE										=   "".join([c for c in outputs if not unicodedata.combining(__UpperCamelCase			)]			)
       if self.do_lower_case:
           SCREAMING_SNAKE_CASE										=   outputs.lower()

       return outputs





   def      _snake_case		(       self					:			Tuple     , __lowerCamelCase					:			str			):
       SCREAMING_SNAKE_CASE										=   self.preprocess_text(__UpperCamelCase			)
       SCREAMING_SNAKE_CASE										=   self.sp_model.encode(__UpperCamelCase     , out_type=__UpperCamelCase			)
       SCREAMING_SNAKE_CASE										=   []
       for piece in pieces:
           if len(__UpperCamelCase			) > 1 and piece[-1] == str(","			) and piece[-2].isdigit():
               SCREAMING_SNAKE_CASE										=   self.sp_model.EncodeAsPieces(piece[:-1].replace(__UpperCamelCase     , ""			)			)
               if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                   if len(cur_pieces[0]			) == 1:
                       SCREAMING_SNAKE_CASE										=   cur_pieces[1:]
                   else:
                       SCREAMING_SNAKE_CASE										=   cur_pieces[0][1:]
               cur_pieces.append(piece[-1]			)
               new_pieces.extend(__UpperCamelCase			)
           else:
               new_pieces.append(__UpperCamelCase			)

       return new_pieces





   def      _snake_case		(       self					:			Optional[int]     , __lowerCamelCase					:			Any			):
       return self.sp_model.PieceToId(__UpperCamelCase			)





   def      _snake_case		(       self					:			Optional[Any]     , __lowerCamelCase					:			Optional[int]			):
       return self.sp_model.IdToPiece(__UpperCamelCase			)





   def      _snake_case		(       self					:			List[Any]     , __lowerCamelCase					:			List[str]			):
       SCREAMING_SNAKE_CASE										=   []
       SCREAMING_SNAKE_CASE										=   ""
       SCREAMING_SNAKE_CASE										=   False
       for token in tokens:
           # make sure that special tokens are not decoded using sentencepiece model
           if token in self.all_special_tokens:
               if not prev_is_special:
                   out_string += " "
               out_string += self.sp_model.decode(__UpperCamelCase			) + token
               SCREAMING_SNAKE_CASE										=   True
               SCREAMING_SNAKE_CASE										=   []
           else:
               current_sub_tokens.append(__UpperCamelCase			)
               SCREAMING_SNAKE_CASE										=   False
       out_string += self.sp_model.decode(__UpperCamelCase			)
       return out_string.strip()





   def      _snake_case		(       self					:			Dict     , __lowerCamelCase					:			List[int]     , __lowerCamelCase					:			Optional[List[int]] = None			):
       SCREAMING_SNAKE_CASE										=   [self.sep_token_id]
       SCREAMING_SNAKE_CASE										=   [self.cls_token_id]
       if token_ids_a is None:
           return cls + token_ids_a + sep
       return cls + token_ids_a + sep + token_ids_a + sep





   def      _snake_case		(       self					:			Tuple     , __lowerCamelCase					:			List[int]     , __lowerCamelCase					:			Optional[List[int]] = None     , __lowerCamelCase					:			bool = False			):

       if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=__UpperCamelCase     , token_ids_a=__UpperCamelCase     , already_has_special_tokens=__UpperCamelCase			)

       if token_ids_a is not None:
           return [1] + ([0] * len(__UpperCamelCase			)) + [1] + ([0] * len(__UpperCamelCase			)) + [1]
       return [1] + ([0] * len(__UpperCamelCase			)) + [1]





   def      _snake_case		(       self					:			int     , __lowerCamelCase					:			List[int]     , __lowerCamelCase					:			Optional[List[int]] = None			):
       SCREAMING_SNAKE_CASE										=   [self.sep_token_id]
       SCREAMING_SNAKE_CASE										=   [self.cls_token_id]

       if token_ids_a is None:
           return len(cls + token_ids_a + sep			) * [0]
       return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]





   def      _snake_case		(       self					:			Union[str, Any]     , __lowerCamelCase					:			str     , __lowerCamelCase					:			Optional[str] = None			):
       if not os.path.isdir(__UpperCamelCase			):
           logger.error(f"Vocabulary path ({save_directory}) should be a directory"			)
           return
       SCREAMING_SNAKE_CASE										=   os.path.join(
           __UpperCamelCase     , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)

       if os.path.abspath(self.vocab_file			) != os.path.abspath(__UpperCamelCase			) and os.path.isfile(self.vocab_file			):
           copyfile(self.vocab_file     , __UpperCamelCase			)
       elif not os.path.isfile(self.vocab_file			):
           with open(__UpperCamelCase     , "wb"			) as fi:
               SCREAMING_SNAKE_CASE										=   self.sp_model.serialized_model_proto()
               fi.write(__UpperCamelCase			)

       return (out_vocab_file,)




import logging
import os
import threading
import time


try:
				import warnings
except ImportError:
				__snake_case     :Any			       =None

try:
				import msvcrt
except ImportError:
				__snake_case     :Union[str, Any]			       =None

try:
				import fcntl
except ImportError:
				__snake_case     :str			       =None


# Backward compatibility
# ------------------------------------------------
try:
				TimeoutError
except NameError:
				__snake_case     :str			       =OSError


# Data
# ------------------------------------------------
__snake_case     :Any			       =[
    'Timeout',
    'BaseFileLock',
    'WindowsFileLock',
    'UnixFileLock',
    'SoftFileLock',
    'FileLock',
]

__snake_case     :str			       ='3.0.12'


__snake_case     :str			       =None






def  lowerCamelCase_	(      )       ->							List[str]:

							'''simple docstring'''






							global _logger
							A      						=    _logger or logging.getLogger(__name__				)
							return _logger



class        lowerCAmelCase__					(  _lowerCamelCase       ):


				def __init__(							self      :  Tuple						, __UpperCamelCase      :  Union[str, Any]       )		->     List[Any]:
											A      						=    lock_file
											return None


				def __str__(							self      :  List[Any]       )		->     int:
											A      						=    f'''The file lock \'{self.lock_file}\' could not be acquired.'''
											return temp





class        lowerCAmelCase__					:


				def __init__(							self      :  int						, __UpperCamelCase      :  Union[str, Any]       )		->     List[str]:
											A      						=    lock
											return None


				def __enter__(							self      :  Dict       )		->     Dict:
											return self.lock


				def __exit__(							self      :  int						, __UpperCamelCase      :  str						, __UpperCamelCase      :  Optional[int]						, __UpperCamelCase      :  Any       )		->     Optional[int]:
											self.lock.release()
											return None





class        lowerCAmelCase__					:


				def __init__(							self      :  List[Any]						, __UpperCamelCase      :  List[str]						, __UpperCamelCase      :  List[Any]=-1						, __UpperCamelCase      :  Optional[Any]=None       )		->     Dict:
											A      						=    max_filename_length if max_filename_length is not None else 255
											# Hash the filename if it's too long
											A      						=    self.hash_filename_if_too_long(__UpperCamelCase						, __UpperCamelCase       )
											# The path to the lock file.
											A      						=    lock_file

											# The file descriptor for the *_lock_file* as it is returned by the
											# os.open() function.
											# This file lock is only NOT None, if the object currently holds the
											# lock.
											A      						=    None

											# The default timeout value.
											A      						=    timeout

											# We use this lock primarily for the lock counter.
											A      						=    threading.Lock()

											# The lock counter is used for implementing the nested locking
											# mechanism. Whenever the lock is acquired, the counter is increased and
											# the lock is only released, when this value is 0 again.
											A      						=    0
											return None


				@property
				def __UpperCamelCase						(							self      :  str       )		->     Union[str, Any]:
											return self._lock_file


				@property
				def __UpperCamelCase						(							self      :  Tuple       )		->     Union[str, Any]:
											return self._timeout


				@timeout.setter
				def __UpperCamelCase						(							self      :  Union[str, Any]						, __UpperCamelCase      :  Any       )		->     Tuple:
											A      						=    float(__UpperCamelCase       )
											return None


				def __UpperCamelCase						(							self      :  Optional[Any]       )		->     Any:
											raise NotImplementedError()


				def __UpperCamelCase						(							self      :  int       )		->     str:
											raise NotImplementedError()


				@property
				def __UpperCamelCase						(							self      :  List[Any]       )		->     Optional[Any]:
											return self._lock_file_fd is not None


				def __UpperCamelCase						(							self      :  int						, __UpperCamelCase      :  List[str]=None						, __UpperCamelCase      :  Any=0.0_5       )		->     Any:
											# Use the default timeout, if no timeout is provided.
											if timeout is None:
																		A      						=    self.timeout

											# Increment the number right at the beginning.
											# We can still undo it, if something fails.
											with self._thread_lock:
																		self._lock_counter += 1

											A      						=    id(self       )
											A      						=    self._lock_file
											A      						=    time.time()
											try:
																		while True:
																									with self._thread_lock:
																																if not self.is_locked:
																																							logger().debug(f'''Attempting to acquire lock {lock_id} on {lock_filename}'''       )
																																							self._acquire()

																									if self.is_locked:
																																logger().debug(f'''Lock {lock_id} acquired on {lock_filename}'''       )
																																break
																									elif timeout >= 0 and time.time() - start_time > timeout:
																																logger().debug(f'''Timeout on acquiring lock {lock_id} on {lock_filename}'''       )
																																raise Timeout(self._lock_file       )
																									else:
																																logger().debug(
																																    f'''Lock {lock_id} not acquired on {lock_filename}, waiting {poll_intervall} seconds ...'''       )
																																time.sleep(__UpperCamelCase       )
											except:  # noqa
																		# Something did go wrong, so decrement the counter.
																		with self._thread_lock:
																									A      						=    max(0						, self._lock_counter - 1       )

																		raise
											return _Acquire_ReturnProxy(lock=self       )


				def __UpperCamelCase						(							self      :  Optional[int]						, __UpperCamelCase      :  Tuple=False       )		->     Tuple:
											with self._thread_lock:
																		if self.is_locked:
																									self._lock_counter -= 1

																									if self._lock_counter == 0 or force:
																																A      						=    id(self       )
																																A      						=    self._lock_file

																																logger().debug(f'''Attempting to release lock {lock_id} on {lock_filename}'''       )
																																self._release()
																																A      						=    0
																																logger().debug(f'''Lock {lock_id} released on {lock_filename}'''       )

											return None


				def __enter__(							self      :  int       )		->     Dict:
											self.acquire()
											return self


				def __exit__(							self      :  List[str]						, __UpperCamelCase      :  Dict						, __UpperCamelCase      :  Optional[Any]						, __UpperCamelCase      :  str       )		->     Dict:
											self.release()
											return None


				def __del__(							self      :  Union[str, Any]       )		->     Optional[int]:
											self.release(force=__UpperCamelCase       )
											return None


				def __UpperCamelCase						(							self      :  Union[str, Any]						, __UpperCamelCase      :  str						, __UpperCamelCase      :  int       )		->     str:
											A      						=    os.path.basename(__UpperCamelCase       )
											if len(__UpperCamelCase       ) > max_length and max_length > 0:
																		A      						=    os.path.dirname(__UpperCamelCase       )
																		A      						=    str(hash(__UpperCamelCase       )       )
																		A      						=    filename[: max_length - len(__UpperCamelCase       ) - 8] + '...' + hashed_filename + '.lock'
																		return os.path.join(__UpperCamelCase						, __UpperCamelCase       )
											else:
																		return path





class        lowerCAmelCase__					(  _lowerCamelCase       ):


				def __init__(							self      :  Tuple						, __UpperCamelCase      :  List[str]						, __UpperCamelCase      :  Tuple=-1						, __UpperCamelCase      :  Optional[Any]=None       )		->     Union[str, Any]:
											from .file_utils import relative_to_absolute_path

											super().__init__(__UpperCamelCase						, timeout=__UpperCamelCase						, max_filename_length=__UpperCamelCase       )
											A      						=    '\\\\?\\' + relative_to_absolute_path(self.lock_file       )


				def __UpperCamelCase						(							self      :  Any       )		->     Any:
											A      						=    os.O_RDWR | os.O_CREAT | os.O_TRUNC

											try:
																		A      						=    os.open(self._lock_file						, __UpperCamelCase       )
											except OSError:
																		pass
											else:
																		try:
																									msvcrt.locking(__UpperCamelCase						, msvcrt.LK_NBLCK						, 1       )
																		except OSError:
																									os.close(__UpperCamelCase       )
																		else:
																									A      						=    fd
											return None


				def __UpperCamelCase						(							self      :  Optional[Any]       )		->     List[Any]:
											A      						=    self._lock_file_fd
											A      						=    None
											msvcrt.locking(__UpperCamelCase						, msvcrt.LK_UNLCK						, 1       )
											os.close(__UpperCamelCase       )

											try:
																		os.remove(self._lock_file       )
											# Probably another instance of the application
											# that acquired the file lock.
											except OSError:
																		pass
											return None





class        lowerCAmelCase__					(  _lowerCamelCase       ):


				def __init__(							self      :  List[Any]						, __UpperCamelCase      :  Any						, __UpperCamelCase      :  List[Any]=-1						, __UpperCamelCase      :  Dict=None       )		->     Dict:
											A      						=    os.statvfs(os.path.dirname(__UpperCamelCase       )       ).f_namemax
											super().__init__(__UpperCamelCase						, timeout=__UpperCamelCase						, max_filename_length=__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Any       )		->     int:
											A      						=    os.O_RDWR | os.O_CREAT | os.O_TRUNC
											A      						=    os.open(self._lock_file						, __UpperCamelCase       )

											try:
																		fcntl.flock(__UpperCamelCase						, fcntl.LOCK_EX | fcntl.LOCK_NB       )
											except OSError:
																		os.close(__UpperCamelCase       )
											else:
																		A      						=    fd
											return None


				def __UpperCamelCase						(							self      :  Optional[int]       )		->     int:
											# Do not remove the lockfile:
											#
											#   https://github.com/benediktschmitt/py-filelock/issues/31
											#   https://stackoverflow.com/questions/17708885/flock-removing-locked-file-without-race-condition
											A      						=    self._lock_file_fd
											A      						=    None
											fcntl.flock(__UpperCamelCase						, fcntl.LOCK_UN       )
											os.close(__UpperCamelCase       )
											return None





class        lowerCAmelCase__					(  _lowerCamelCase       ):


				def __UpperCamelCase						(							self      :  int       )		->     Optional[int]:
											A      						=    os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
											try:
																		A      						=    os.open(self._lock_file						, __UpperCamelCase       )
											except OSError:
																		pass
											else:
																		A      						=    fd
											return None


				def __UpperCamelCase						(							self      :  Optional[Any]       )		->     List[str]:
											os.close(self._lock_file_fd       )
											A      						=    None

											try:
																		os.remove(self._lock_file       )
											# The file is already deleted and that's what we want.
											except OSError:
																		pass
											return None





__snake_case     :List[str]			       =None

if msvcrt:
				__snake_case     :List[Any]			       =WindowsFileLock
elif fcntl:
				__snake_case     :Any			       =UnixFileLock
else:
				__snake_case     :Tuple			       =SoftFileLock

				if warnings is not None:
								warnings.warn('only soft file lock is available')