



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__snake_case							:						Any      =logging.get_logger(__name__)
__snake_case							:						Any      ={'vocab_file': 'spiece.model'}

__snake_case							:						Union[str, Any]      ={
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

__snake_case							:						List[Any]      ={
    'albert-base-v1': 5_1_2,
    'albert-large-v1': 5_1_2,
    'albert-xlarge-v1': 5_1_2,
    'albert-xxlarge-v1': 5_1_2,
    'albert-base-v2': 5_1_2,
    'albert-large-v2': 5_1_2,
    'albert-xlarge-v2': 5_1_2,
    'albert-xxlarge-v2': 5_1_2,
}

__snake_case							:						Union[str, Any]      ='▁'



class 			lowerCamelCase__     (				__A):




		'''simple docstring'''



		snake_case_      =VOCAB_FILES_NAMES
		snake_case_      =PRETRAINED_VOCAB_FILES_MAP
		snake_case_      =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



		def __init__(self						,__lowerCamelCase						,__lowerCamelCase=True						,__lowerCamelCase=True						,__lowerCamelCase=False						,__lowerCamelCase="[CLS]"						,__lowerCamelCase="[SEP]"						,__lowerCamelCase="<unk>"						,__lowerCamelCase="[SEP]"						,__lowerCamelCase="<pad>"						,__lowerCamelCase="[CLS]"						,__lowerCamelCase="[MASK]"						,__lowerCamelCase = None						,**__lowerCamelCase						,)	->     List[str]:

						"""simple docstring"""






						lowerCAmelCase__	:							int				=	(
						    AddedToken(__A						,lstrip=__A						,rstrip=__A						,normalized=__A					)
						    if isinstance(__A						,__A					)
						    else mask_token
						)

						lowerCAmelCase__	:							int				=	{} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    do_lower_case=__A						,remove_space=__A						,keep_accents=__A						,bos_token=__A						,eos_token=__A						,unk_token=__A						,sep_token=__A						,pad_token=__A						,cls_token=__A						,mask_token=__A						,sp_model_kwargs=self.sp_model_kwargs						,**__A						,)

						lowerCAmelCase__	:							Dict				=	do_lower_case
						lowerCAmelCase__	:							Dict				=	remove_space
						lowerCAmelCase__	:							Any				=	keep_accents
						lowerCAmelCase__	:							Optional[Any]				=	vocab_file

						lowerCAmelCase__	:							str				=	spm.SentencePieceProcessor(**self.sp_model_kwargs					)
						self.sp_model.Load(__A					)



		@property
		def 						lowerCAmelCase__ (self					)	->     Tuple:

						"""simple docstring"""






						return len(self.sp_model					)



		def 						lowerCAmelCase__ (self					)	->     Dict:

						"""simple docstring"""






						lowerCAmelCase__	:							Any				=	{self.convert_ids_to_tokens(__A					): i for i in range(self.vocab_size					)}
						vocab.update(self.added_tokens_encoder					)
						return vocab



		def __getstate__(self					)	->     Optional[int]:

						"""simple docstring"""






						lowerCAmelCase__	:							Union[str, Any]				=	self.__dict__.copy()
						lowerCAmelCase__	:							Dict				=	None
						return state



		def __setstate__(self						,__lowerCamelCase					)	->     int:

						"""simple docstring"""






						lowerCAmelCase__	:							List[str]				=	d

						# for backward compatibility
						if not hasattr(self						,'''sp_model_kwargs'''					):
										lowerCAmelCase__	:							Union[str, Any]				=	{}

						lowerCAmelCase__	:							Tuple				=	spm.SentencePieceProcessor(**self.sp_model_kwargs					)
						self.sp_model.Load(self.vocab_file					)



		def 						lowerCAmelCase__ (self						,__lowerCamelCase					)	->     List[str]:

						"""simple docstring"""






						if self.remove_space:
										lowerCAmelCase__	:							List[str]				=	''' '''.join(inputs.strip().split()					)
						else:
										lowerCAmelCase__	:							Tuple				=	inputs
						lowerCAmelCase__	:							Tuple				=	outputs.replace('''``'''						,'''\"'''					).replace('''\'\''''						,'''\"'''					)

						if not self.keep_accents:
										lowerCAmelCase__	:							Tuple				=	unicodedata.normalize('''NFKD'''						,__A					)
										lowerCAmelCase__	:							Optional[Any]				=	''''''.join([c for c in outputs if not unicodedata.combining(__A					)]					)
						if self.do_lower_case:
										lowerCAmelCase__	:							Dict				=	outputs.lower()

						return outputs



		def 						lowerCAmelCase__ (self						,__lowerCamelCase					)	->     List[str]:

						"""simple docstring"""






						lowerCAmelCase__	:							str				=	self.preprocess_text(__A					)
						lowerCAmelCase__	:							Tuple				=	self.sp_model.encode(__A						,out_type=__A					)
						lowerCAmelCase__	:							Dict				=	[]
						for piece in pieces:
										if len(__A					) > 1 and piece[-1] == str(''','''					) and piece[-2].isdigit():
														lowerCAmelCase__	:							Optional[int]				=	self.sp_model.EncodeAsPieces(piece[:-1].replace(__A						,''''''					)					)
														if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																		if len(cur_pieces[0]					) == 1:
																						lowerCAmelCase__	:							List[str]				=	cur_pieces[1:]
																		else:
																						lowerCAmelCase__	:							Union[str, Any]				=	cur_pieces[0][1:]
														cur_pieces.append(piece[-1]					)
														new_pieces.extend(__A					)
										else:
														new_pieces.append(__A					)

						return new_pieces



		def 						lowerCAmelCase__ (self						,__lowerCamelCase					)	->     Optional[Any]:

						"""simple docstring"""






						return self.sp_model.PieceToId(__A					)



		def 						lowerCAmelCase__ (self						,__lowerCamelCase					)	->     Dict:

						"""simple docstring"""






						return self.sp_model.IdToPiece(__A					)



		def 						lowerCAmelCase__ (self						,__lowerCamelCase					)	->     str:

						"""simple docstring"""






						lowerCAmelCase__	:							Dict				=	[]
						lowerCAmelCase__	:							str				=	''''''
						lowerCAmelCase__	:							int				=	False
						for token in tokens:
										# make sure that special tokens are not decoded using sentencepiece model
										if token in self.all_special_tokens:
														if not prev_is_special:
																		out_string += " "
														out_string += self.sp_model.decode(__A					) + token
														lowerCAmelCase__	:							List[Any]				=	True
														lowerCAmelCase__	:							Any				=	[]
										else:
														current_sub_tokens.append(__A					)
														lowerCAmelCase__	:							Dict				=	False
						out_string += self.sp_model.decode(__A					)
						return out_string.strip()



		def 						lowerCAmelCase__ (self						,__lowerCamelCase						,__lowerCamelCase = None					)	->     int:

						"""simple docstring"""






						lowerCAmelCase__	:							List[str]				=	[self.sep_token_id]
						lowerCAmelCase__	:							Dict				=	[self.cls_token_id]
						if token_ids_a is None:
										return cls + token_ids_a + sep
						return cls + token_ids_a + sep + token_ids_a + sep



		def 						lowerCAmelCase__ (self						,__lowerCamelCase						,__lowerCamelCase = None						,__lowerCamelCase = False					)	->     List[Any]:

						"""simple docstring"""






						if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=__A						,token_ids_a=__A						,already_has_special_tokens=__A					)

						if token_ids_a is not None:
										return [1] + ([0] * len(__A					)) + [1] + ([0] * len(__A					)) + [1]
						return [1] + ([0] * len(__A					)) + [1]



		def 						lowerCAmelCase__ (self						,__lowerCamelCase						,__lowerCamelCase = None					)	->     Dict:

						"""simple docstring"""






						lowerCAmelCase__	:							Tuple				=	[self.sep_token_id]
						lowerCAmelCase__	:							Optional[int]				=	[self.cls_token_id]

						if token_ids_a is None:
										return len(cls + token_ids_a + sep					) * [0]
						return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]



		def 						lowerCAmelCase__ (self						,__lowerCamelCase						,__lowerCamelCase = None					)	->     int:

						"""simple docstring"""






						if not os.path.isdir(__A					):
										logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""					)
										return
						lowerCAmelCase__	:							Optional[Any]				=	os.path.join(
						    __A						,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

						if os.path.abspath(self.vocab_file					) != os.path.abspath(__A					) and os.path.isfile(self.vocab_file					):
										copyfile(self.vocab_file						,__A					)
						elif not os.path.isfile(self.vocab_file					):
										with open(__A						,'''wb'''					) as fi:
														lowerCAmelCase__	:							Any				=	self.sp_model.serialized_model_proto()
														fi.write(__A					)

						return (out_vocab_file,)





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


SCREAMING_SNAKE_CASE							    =							logging.get_logger(__name__)

SCREAMING_SNAKE_CASE							    =							{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE							    =							{
    'vocab_file': {'mobilebert-uncased': 'https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt'},
    'tokenizer_file': {
        'mobilebert-uncased': 'https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json'
    },
}

SCREAMING_SNAKE_CASE							    =							{'mobilebert-uncased': 5_1_2}


SCREAMING_SNAKE_CASE							    =							{}


class     __UpperCAmelCase							(       __A							):


     """simple docstring"""
     _lowerCamelCase   =      VOCAB_FILES_NAMES
     _lowerCamelCase   =      PRETRAINED_VOCAB_FILES_MAP
     _lowerCamelCase   =      PRETRAINED_INIT_CONFIGURATION
     _lowerCamelCase   =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _lowerCamelCase   =      MobileBertTokenizer


     def __init__(	self  ,       __A=None  ,       __A=None  ,       __A=True  ,       __A="[UNK]"  ,       __A="[SEP]"  ,       __A="[PAD]"  ,       __A="[CLS]"  ,       __A="[MASK]"  ,       __A=True  ,       __A=None  ,       **__A  ,       ):
        super().__init__(
            __A  ,       tokenizer_file=__A  ,       do_lower_case=__A  ,       unk_token=__A  ,       sep_token=__A  ,       pad_token=__A  ,       cls_token=__A  ,       mask_token=__A  ,       tokenize_chinese_chars=__A  ,       strip_accents=__A  ,       **__A  ,       )

        __a					=  json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
        if (
            normalizer_state.get("""lowercase"""  ,       __A    ) != do_lower_case
            or normalizer_state.get("""strip_accents"""  ,       __A    ) != strip_accents
            or normalizer_state.get("""handle_chinese_chars"""  ,       __A    ) != tokenize_chinese_chars
        ):
           __a					=  getattr(__A  ,       normalizer_state.pop("""type"""    )    )
           __a					=  do_lower_case
           __a					=  strip_accents
           __a					=  tokenize_chinese_chars
           __a					=  normalizer_class(**__A    )

        __a					=  do_lower_case


     def 	snake_case_  (	self  ,       __A  ,       __A=None    ):
        __a					=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

        return output


     def 	snake_case_  (	self  ,       __A  ,       __A = None    ):
        __a					=  [self.sep_token_id]
        __a					=  [self.cls_token_id]
        if token_ids_a is None:
           return len(cls + token_ids_a + sep    ) * [0]
        return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]


     def 	snake_case_  (	self  ,       __A  ,       __A = None    ):
        __a					=  self._tokenizer.model.save(__A  ,       name=__A    )
        return tuple(__A    )



