

'''simple docstring'''





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCamelCase_    						=       logging.get_logger(__name__)

UpperCamelCase_    						=       {'''vocab_file''': '''spiece.model'''}

UpperCamelCase_    						=       {
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

UpperCamelCase_    						=       {
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
UpperCamelCase_    						=       0
UpperCamelCase_    						=       1
UpperCamelCase_    						=       2
UpperCamelCase_    						=       3
UpperCamelCase_    						=       4





class 		__SCREAMING_SNAKE_CASE							(			_lowercase      ):
				lowerCamelCase_		       =    VOCAB_FILES_NAMES
				lowerCamelCase_		       =    PRETRAINED_VOCAB_FILES_MAP
				lowerCamelCase_		       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCamelCase_		       =    '''left'''







				def __init__(       self     :		List[str]		,					UpperCAmelCase__     :		List[Any]		,					UpperCAmelCase__     :		Optional[int]=False		,					UpperCAmelCase__     :		str=True		,					UpperCAmelCase__     :		Optional[Any]=False		,					UpperCAmelCase__     :		int="<s>"		,					UpperCAmelCase__     :		str="</s>"		,					UpperCAmelCase__     :		List[str]="<unk>"		,					UpperCAmelCase__     :		Any="<sep>"		,					UpperCAmelCase__     :		Tuple="<pad>"		,					UpperCAmelCase__     :		Optional[Any]="<cls>"		,					UpperCAmelCase__     :		Dict="<mask>"		,					UpperCAmelCase__     :		str=["<eop>", "<eod>"]		,					UpperCAmelCase__     :		Optional[Dict[str, Any]] = None		,					**UpperCAmelCase__     :		Any		,					):
										'''simple docstring'''
										# Mask token behave like a normal word, i.e. include the space before it
										lowercase :							Union[str, Any]    		=AddedToken(__A		,					lstrip=__A		,					rstrip=__A   ) if isinstance(__A		,					__A   ) else mask_token

										lowercase :							Optional[int]    		={} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    do_lower_case=__A		,					remove_space=__A		,					keep_accents=__A		,					bos_token=__A		,					eos_token=__A		,					unk_token=__A		,					sep_token=__A		,					pad_token=__A		,					cls_token=__A		,					mask_token=__A		,					additional_special_tokens=__A		,					sp_model_kwargs=self.sp_model_kwargs		,					**__A		,					)

										lowercase :							int    		=3

										lowercase :							List[str]    		=do_lower_case
										lowercase :							Optional[Any]    		=remove_space
										lowercase :							Any    		=keep_accents
										lowercase :							int    		=vocab_file

										lowercase :							Union[str, Any]    		=spm.SentencePieceProcessor(**self.sp_model_kwargs   )
										self.sp_model.Load(__A   )







				@property
				def  lowerCamelCase_	(       self     :		Union[str, Any]   ):
										'''simple docstring'''
										return len(self.sp_model   )







				def  lowerCamelCase_	(       self     :		Any   ):
										'''simple docstring'''
										lowercase :							int    		={self.convert_ids_to_tokens(__A   ): i for i in range(self.vocab_size   )}
										vocab.update(self.added_tokens_encoder   )
										return vocab







				def __getstate__(       self     :		List[Any]   ):
										'''simple docstring'''
										lowercase :							Optional[int]    		=self.__dict__.copy()
										lowercase :							Dict    		=None
										return state







				def __setstate__(       self     :		List[str]		,					UpperCAmelCase__     :		Union[str, Any]   ):
										'''simple docstring'''
										lowercase :							str    		=d

										# for backward compatibility
										if not hasattr(self		,					'''sp_model_kwargs'''   ):
																lowercase :							List[Any]    		={}

										lowercase :							List[Any]    		=spm.SentencePieceProcessor(**self.sp_model_kwargs   )
										self.sp_model.Load(self.vocab_file   )







				def  lowerCamelCase_	(       self     :		Optional[Any]		,					UpperCAmelCase__     :		List[str]   ):
										'''simple docstring'''
										if self.remove_space:
																lowercase :							str    		=""" """.join(inputs.strip().split()   )
										else:
																lowercase :							Union[str, Any]    		=inputs
										lowercase :							Optional[int]    		=outputs.replace('''``'''		,					'''\"'''   ).replace('''\'\''''		,					'''\"'''   )

										if not self.keep_accents:
																lowercase :							str    		=unicodedata.normalize('''NFKD'''		,					__A   )
																lowercase :							Union[str, Any]    		="""""".join([c for c in outputs if not unicodedata.combining(__A   )]   )
										if self.do_lower_case:
																lowercase :							Optional[Any]    		=outputs.lower()

										return outputs







				def  lowerCamelCase_	(       self     :		List[str]		,					UpperCAmelCase__     :		str   ):
										'''simple docstring'''
										lowercase :							Dict    		=self.preprocess_text(__A   )
										lowercase :							str    		=self.sp_model.encode(__A		,					out_type=__A   )
										lowercase :							Any    		=[]
										for piece in pieces:
																if len(__A   ) > 1 and piece[-1] == str(''','''   ) and piece[-2].isdigit():
																						lowercase :							Dict    		=self.sp_model.EncodeAsPieces(piece[:-1].replace(__A		,					''''''   )   )
																						if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																												if len(cur_pieces[0]   ) == 1:
																																		lowercase :							List[str]    		=cur_pieces[1:]
																												else:
																																		lowercase :							int    		=cur_pieces[0][1:]
																						cur_pieces.append(piece[-1]   )
																						new_pieces.extend(__A   )
																else:
																						new_pieces.append(__A   )

										return new_pieces







				def  lowerCamelCase_	(       self     :		str		,					UpperCAmelCase__     :		int   ):
										'''simple docstring'''
										return self.sp_model.PieceToId(__A   )







				def  lowerCamelCase_	(       self     :		Optional[Any]		,					UpperCAmelCase__     :		List[Any]   ):
										'''simple docstring'''
										return self.sp_model.IdToPiece(__A   )







				def  lowerCamelCase_	(       self     :		Optional[Any]		,					UpperCAmelCase__     :		List[Any]   ):
										'''simple docstring'''
										lowercase :							Optional[Any]    		="""""".join(__A   ).replace(__A		,					''' '''   ).strip()
										return out_string







				def  lowerCamelCase_	(       self     :		str		,					UpperCAmelCase__     :		List[int]		,					UpperCAmelCase__     :		bool = False		,					UpperCAmelCase__     :		bool = None		,					UpperCAmelCase__     :		bool = True		,					**UpperCAmelCase__     :		Optional[Any]		,					):
										'''simple docstring'''
										lowercase :							Any    		=kwargs.pop('''use_source_tokenizer'''		,					__A   )

										lowercase :							str    		=self.convert_ids_to_tokens(__A		,					skip_special_tokens=__A   )

										# To avoid mixing byte-level and unicode for byte-level BPT
										# we need to build string separately for added tokens and byte-level tokens
										# cf. https://github.com/huggingface/transformers/issues/1133
										lowercase :							Union[str, Any]    		=[]
										lowercase :							Optional[int]    		=[]
										for token in filtered_tokens:
																if skip_special_tokens and token in self.all_special_ids:
																						continue
																if token in self.added_tokens_encoder:
																						if current_sub_text:
																												sub_texts.append(self.convert_tokens_to_string(__A   )   )
																												lowercase :							Any    		=[]
																						sub_texts.append(__A   )
																else:
																						current_sub_text.append(__A   )
										if current_sub_text:
																sub_texts.append(self.convert_tokens_to_string(__A   )   )

										# Mimic the behavior of the Rust tokenizer:
										# By default, there are no spaces between special tokens
										lowercase :							int    		="""""".join(__A   )

										lowercase :							List[Any]    		=(
										    clean_up_tokenization_spaces
										    if clean_up_tokenization_spaces is not None
										    else self.clean_up_tokenization_spaces
										)
										if clean_up_tokenization_spaces:
																lowercase :							int    		=self.clean_up_tokenization(__A   )
																return clean_text
										else:
																return text







				def  lowerCamelCase_	(       self     :		Dict		,					UpperCAmelCase__     :		List[int]		,					UpperCAmelCase__     :		Optional[List[int]] = None   ):
										'''simple docstring'''
										lowercase :							Union[str, Any]    		=[self.sep_token_id]
										lowercase :							Any    		=[self.cls_token_id]
										if token_ids_a is None:
																return token_ids_a + sep + cls
										return token_ids_a + sep + token_ids_a + sep + cls







				def  lowerCamelCase_	(       self     :		Dict		,					UpperCAmelCase__     :		List[int]		,					UpperCAmelCase__     :		Optional[List[int]] = None		,					UpperCAmelCase__     :		bool = False   ):
										'''simple docstring'''

										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=__A		,					token_ids_a=__A		,					already_has_special_tokens=__A   )

										if token_ids_a is not None:
																return ([0] * len(__A   )) + [1] + ([0] * len(__A   )) + [1, 1]
										return ([0] * len(__A   )) + [1, 1]







				def  lowerCamelCase_	(       self     :		List[Any]		,					UpperCAmelCase__     :		List[int]		,					UpperCAmelCase__     :		Optional[List[int]] = None   ):
										'''simple docstring'''
										lowercase :							Dict    		=[self.sep_token_id]
										lowercase :							Union[str, Any]    		=[2]

										if token_ids_a is None:
																return len(token_ids_a + sep   ) * [0] + cls_segment_id
										return len(token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1] + cls_segment_id







				def  lowerCamelCase_	(       self     :		Tuple		,					UpperCAmelCase__     :		str		,					UpperCAmelCase__     :		Optional[str] = None   ):
										'''simple docstring'''
										if not os.path.isdir(__A   ):
																logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''   )
																return
										lowercase :							Dict    		=os.path.join(
										    __A		,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )

										if os.path.abspath(self.vocab_file   ) != os.path.abspath(__A   ) and os.path.isfile(self.vocab_file   ):
																copyfile(self.vocab_file		,					__A   )
										elif not os.path.isfile(self.vocab_file   ):
																with open(__A		,					'''wb'''   ) as fi:
																						lowercase :							List[str]    		=self.sp_model.serialized_model_proto()
																						fi.write(__A   )

										return (out_vocab_file,)








from __future__ import annotations
def 							__SCREAMING_SNAKE_CASE     (   a__    :   List[str]			,a__    :   Dict			,a__    :   Union[str, Any]			,a__    :   Any       )					->				Optional[int]:  # noqa: E741
	while r - l > 1:
		__A :			Any		    =			(l + r) // 2
		if v[m] >= key:
			__A :			Optional[int]		    =			m
		else:
			__A :			List[Any]		    =			m  # noqa: E741
	return r
def 							__SCREAMING_SNAKE_CASE     (   a__    :   list[int]       )					->				int:
	if len(a__       ) == 0:
		return 0

	__A :			str		    =			[0] * len(a__       )
	__A :			List[str]		    =			1

	__A :			List[Any]		    =			v[0]

	for i in range(1			,len(a__       )       ):
		if v[i] < tail[0]:
			__A :			int		    =			v[i]
		elif v[i] > tail[length - 1]:
			__A :			Union[str, Any]		    =			v[i]
			length += 1
		else:
			__A :			Any		    =			v[i]

	return length


if __name__ == "__main__":
			import doctest

			doctest.testmod()





