





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCAmelCase_      							=     logging.get_logger(__name__)

UpperCAmelCase_      							=     {"""vocab_file""": """spiece.model"""}

UpperCAmelCase_      							=     {
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    }
}

UpperCAmelCase_      							=     {
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

# Segments (not really needed)
UpperCAmelCase_      							=     0
UpperCAmelCase_      							=     1
UpperCAmelCase_      							=     2
UpperCAmelCase_      							=     3
UpperCAmelCase_      							=     4
class 			UpperCamelCase_		(  _lowerCamelCase					):
					lowerCAmelCase_							=	VOCAB_FILES_NAMES
					lowerCAmelCase_							=	PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_							=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_							=	'''left'''




					def __init__(		self				,						lowerCAmelCase_				,						lowerCAmelCase_=False				,						lowerCAmelCase_=True				,						lowerCAmelCase_=False				,						lowerCAmelCase_="<s>"				,						lowerCAmelCase_="</s>"				,						lowerCAmelCase_="<unk>"				,						lowerCAmelCase_="<sep>"				,						lowerCAmelCase_="<pad>"				,						lowerCAmelCase_="<cls>"				,						lowerCAmelCase_="<mask>"				,						lowerCAmelCase_=["<eop>", "<eod>"]				,						lowerCAmelCase_ = None				,						**lowerCAmelCase_				,						)		->							None:
										# Mask token behave like a normal word, i.e. include the space before it
										_snake_case									=				AddedToken(lowerCAmelCase_				,						lstrip=lowerCAmelCase_				,						rstrip=lowerCAmelCase_  ) if isinstance(lowerCAmelCase_				,						lowerCAmelCase_  ) else mask_token

										_snake_case									=				{} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    do_lower_case=lowerCAmelCase_				,						remove_space=lowerCAmelCase_				,						keep_accents=lowerCAmelCase_				,						bos_token=lowerCAmelCase_				,						eos_token=lowerCAmelCase_				,						unk_token=lowerCAmelCase_				,						sep_token=lowerCAmelCase_				,						pad_token=lowerCAmelCase_				,						cls_token=lowerCAmelCase_				,						mask_token=lowerCAmelCase_				,						additional_special_tokens=lowerCAmelCase_				,						sp_model_kwargs=self.sp_model_kwargs				,						**lowerCAmelCase_				,						)

										_snake_case									=				3

										_snake_case									=				do_lower_case
										_snake_case									=				remove_space
										_snake_case									=				keep_accents
										_snake_case									=				vocab_file

										_snake_case									=				spm.SentencePieceProcessor(**self.sp_model_kwargs  )
										self.sp_model.Load(lowerCAmelCase_  )




					@property
					def 	lowerCAmelCase			(		self  )		->							Tuple:
										return len(self.sp_model  )




					def 	lowerCAmelCase			(		self  )		->							Dict:
										_snake_case									=				{self.convert_ids_to_tokens(lowerCAmelCase_  ): i for i in range(self.vocab_size  )}
										vocab.update(self.added_tokens_encoder  )
										return vocab




					def __getstate__(		self  )		->							Dict:
										_snake_case									=				self.__dict__.copy()
										_snake_case									=				None
										return state




					def __setstate__(		self				,						lowerCAmelCase_  )		->							str:
										_snake_case									=				d

										# for backward compatibility
										if not hasattr(self				,						'sp_model_kwargs'  ):
															_snake_case									=				{}

										_snake_case									=				spm.SentencePieceProcessor(**self.sp_model_kwargs  )
										self.sp_model.Load(self.vocab_file  )




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_  )		->							Union[str, Any]:
										if self.remove_space:
															_snake_case									=				' '.join(inputs.strip().split()  )
										else:
															_snake_case									=				inputs
										_snake_case									=				outputs.replace('``'				,						'"'  ).replace('\'\''				,						'"'  )

										if not self.keep_accents:
															_snake_case									=				unicodedata.normalize('NFKD'				,						lowerCAmelCase_  )
															_snake_case									=				''.join([c for c in outputs if not unicodedata.combining(lowerCAmelCase_  )]  )
										if self.do_lower_case:
															_snake_case									=				outputs.lower()

										return outputs




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_  )		->							List[str]:
										_snake_case									=				self.preprocess_text(lowerCAmelCase_  )
										_snake_case									=				self.sp_model.encode(lowerCAmelCase_				,						out_type=lowerCAmelCase_  )
										_snake_case									=				[]
										for piece in pieces:
															if len(lowerCAmelCase_  ) > 1 and piece[-1] == str(','  ) and piece[-2].isdigit():
																				_snake_case									=				self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase_				,						''  )  )
																				if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																									if len(cur_pieces[0]  ) == 1:
																														_snake_case									=				cur_pieces[1:]
																									else:
																														_snake_case									=				cur_pieces[0][1:]
																				cur_pieces.append(piece[-1]  )
																				new_pieces.extend(lowerCAmelCase_  )
															else:
																				new_pieces.append(lowerCAmelCase_  )

										return new_pieces




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_  )		->							Dict:
										return self.sp_model.PieceToId(lowerCAmelCase_  )




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_  )		->							List[str]:
										return self.sp_model.IdToPiece(lowerCAmelCase_  )




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_  )		->							Optional[int]:
										_snake_case									=				''.join(lowerCAmelCase_  ).replace(lowerCAmelCase_				,						' '  ).strip()
										return out_string




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = False				,						lowerCAmelCase_ = None				,						lowerCAmelCase_ = True				,						**lowerCAmelCase_				,						)		->							str:
										_snake_case									=				kwargs.pop('use_source_tokenizer'				,						lowerCAmelCase_  )

										_snake_case									=				self.convert_ids_to_tokens(lowerCAmelCase_				,						skip_special_tokens=lowerCAmelCase_  )

										# To avoid mixing byte-level and unicode for byte-level BPT
										# we need to build string separately for added tokens and byte-level tokens
										# cf. https://github.com/huggingface/transformers/issues/1133
										_snake_case									=				[]
										_snake_case									=				[]
										for token in filtered_tokens:
															if skip_special_tokens and token in self.all_special_ids:
																				continue
															if token in self.added_tokens_encoder:
																				if current_sub_text:
																									sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase_  )  )
																									_snake_case									=				[]
																				sub_texts.append(lowerCAmelCase_  )
															else:
																				current_sub_text.append(lowerCAmelCase_  )
										if current_sub_text:
															sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase_  )  )

										# Mimic the behavior of the Rust tokenizer:
										# By default, there are no spaces between special tokens
										_snake_case									=				''.join(lowerCAmelCase_  )

										_snake_case									=				(
										    clean_up_tokenization_spaces
										    if clean_up_tokenization_spaces is not None
										    else self.clean_up_tokenization_spaces
										)
										if clean_up_tokenization_spaces:
															_snake_case									=				self.clean_up_tokenization(lowerCAmelCase_  )
															return clean_text
										else:
															return text




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = None  )		->							List[int]:
										_snake_case									=				[self.sep_token_id]
										_snake_case									=				[self.cls_token_id]
										if token_ids_a is None:
															return token_ids_a + sep + cls
										return token_ids_a + sep + token_ids_a + sep + cls




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = None				,						lowerCAmelCase_ = False  )		->							List[int]:

										if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=lowerCAmelCase_				,						token_ids_a=lowerCAmelCase_				,						already_has_special_tokens=lowerCAmelCase_  )

										if token_ids_a is not None:
															return ([0] * len(lowerCAmelCase_  )) + [1] + ([0] * len(lowerCAmelCase_  )) + [1, 1]
										return ([0] * len(lowerCAmelCase_  )) + [1, 1]




					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = None  )		->							List[int]:
										_snake_case									=				[self.sep_token_id]
										_snake_case									=				[2]

										if token_ids_a is None:
															return len(token_ids_a + sep  ) * [0] + cls_segment_id
										return len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1] + cls_segment_id






					def 	lowerCAmelCase			(		self				,						lowerCAmelCase_				,						lowerCAmelCase_ = None  )		->							Tuple[str]:
										if not os.path.isdir(lowerCAmelCase_  ):
															logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
															return
										_snake_case									=				os.path.join(
										    lowerCAmelCase_				,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )

										if os.path.abspath(self.vocab_file  ) != os.path.abspath(lowerCAmelCase_  ) and os.path.isfile(self.vocab_file  ):
															copyfile(self.vocab_file				,						lowerCAmelCase_  )
										elif not os.path.isfile(self.vocab_file  ):
															with open(lowerCAmelCase_				,						'wb'  ) as fi:
																				_snake_case									=				self.sp_model.serialized_model_proto()
																				fi.write(lowerCAmelCase_  )

										return (out_vocab_file,)










UpperCAmelCase_      							=     range(2, 20 + 1)
UpperCAmelCase_      							=     [10**k for k in range(ks[-1] + 1)]
UpperCAmelCase_      							=     {}





def 						lowerCamelCase__     (     UpperCamelCase__			:       Optional[Any]       ,    UpperCamelCase__			:       List[Any]       ,    UpperCamelCase__			:       Union[str, Any]       ,    UpperCamelCase__			:       str						)			-> List[str]:



					'''simple docstring'''




					_snake_case									=				sum(a_i[j] for j in range(UpperCamelCase__       ,    len(UpperCamelCase__						)						)						)
					_snake_case									=				sum(a_i[j] * base[j] for j in range(min(len(UpperCamelCase__						)       ,    UpperCamelCase__						)						)						)

					_snake_case							,     _snake_case									=				0, 0
					_snake_case									=				n - i

					_snake_case									=				memo.get(UpperCamelCase__						)

					if sub_memo is not None:
										_snake_case									=				sub_memo.get(UpperCamelCase__						)

										if jumps is not None and len(UpperCamelCase__						) > 0:
															# find and make the largest jump without going over
															_snake_case									=				-1
															for _k in range(len(UpperCamelCase__						) - 1       ,    -1       ,    -1						):
																				if jumps[_k][2] <= k and jumps[_k][1] <= max_dn:
																									_snake_case									=				_k
																									break

															if max_jump >= 0:
																				_snake_case							,     _snake_case							,     _snake_case									=				jumps[max_jump]
																				# since the difference between jumps is cached, add c
																				_snake_case									=				diff + c
																				for j in range(min(UpperCamelCase__       ,    len(UpperCamelCase__						)						)						):
																									_snake_case							,     _snake_case									=				divmod(UpperCamelCase__       ,    10						)
																				if new_c > 0:
																									add(UpperCamelCase__       ,    UpperCamelCase__       ,    UpperCamelCase__						)

										else:
															_snake_case									=				[]
					else:
										_snake_case									=				{c: []}
										_snake_case									=				sub_memo

					if dn >= max_dn or c + diff >= base[k]:
										return diff, dn

					if k > ks[0]:
										while True:
															# keep doing smaller jumps
															_snake_case							,     _snake_case									=				next_term(UpperCamelCase__       ,    k - 1       ,    i + dn       ,    UpperCamelCase__						)
															diff += _diff
															dn += terms_jumped

															if dn >= max_dn or c + diff >= base[k]:
																				break
					else:
										# would be too small a jump, just compute sequential terms instead
										_snake_case							,     _snake_case									=				compute(UpperCamelCase__       ,    UpperCamelCase__       ,    i + dn       ,    UpperCamelCase__						)
										diff += _diff
										dn += terms_jumped

					_snake_case									=				sub_memo[c]

					# keep jumps sorted by # of terms skipped
					_snake_case									=				0
					while j < len(UpperCamelCase__						):
										if jumps[j][1] > dn:
															break
										j += 1

					# cache the jump for this value digitsum(b) and c
					sub_memo[c].insert(UpperCamelCase__       ,    (diff, dn, k)						)
					return (diff, dn)





def 						lowerCamelCase__     (     UpperCamelCase__			:       str       ,    UpperCamelCase__			:       str       ,    UpperCamelCase__			:       List[str]       ,    UpperCamelCase__			:       Union[str, Any]						)			-> Optional[Any]:



					'''simple docstring'''




					if i >= n:
										return 0, i
					if k > len(UpperCamelCase__						):
										a_i.extend([0 for _ in range(k - len(UpperCamelCase__						)						)]						)

					# note: a_i -> b * 10^k + c
					# ds_b -> digitsum(b)
					# ds_c -> digitsum(c)
					_snake_case									=				i
					_snake_case							,     _snake_case							,     _snake_case									=				0, 0, 0
					for j in range(len(UpperCamelCase__						)						):
										if j >= k:
															ds_b += a_i[j]
										else:
															ds_c += a_i[j]

					while i < n:
										i += 1
										_snake_case									=				ds_c + ds_b
										diff += addend
										_snake_case									=				0
										for j in range(UpperCamelCase__						):
															_snake_case									=				a_i[j] + addend
															_snake_case							,     _snake_case									=				divmod(UpperCamelCase__       ,    10						)

															ds_c += a_i[j]

										if addend > 0:
															break

					if addend > 0:
										add(UpperCamelCase__       ,    UpperCamelCase__       ,    UpperCamelCase__						)
					return diff, i - start_i





def 						lowerCamelCase__     (     UpperCamelCase__			:       Any       ,    UpperCamelCase__			:       Union[str, Any]       ,    UpperCamelCase__			:       Optional[Any]						)			-> Any:



					'''simple docstring'''




					for j in range(UpperCamelCase__       ,    len(UpperCamelCase__						)						):
										_snake_case									=				digits[j] + addend
										if s >= 10:
															_snake_case							,     _snake_case									=				divmod(UpperCamelCase__       ,    10						)
															_snake_case									=				addend // 10 + quotient
										else:
															_snake_case									=				s
															_snake_case									=				addend // 10

										if addend == 0:
															break

					while addend > 0:
										_snake_case							,     _snake_case									=				divmod(UpperCamelCase__       ,    10						)
										digits.append(UpperCamelCase__						)





def 						lowerCamelCase__     (     UpperCamelCase__			:       int = 10**15						)			-> int:



					'''simple docstring'''




					_snake_case									=				[1]
					_snake_case									=				1
					_snake_case									=				0
					while True:
										_snake_case							,     _snake_case									=				next_term(UpperCamelCase__       ,    20       ,    i + dn       ,    UpperCamelCase__						)
										dn += terms_jumped
										if dn == n - i:
															break

					_snake_case									=				0
					for j in range(len(UpperCamelCase__						)						):
										a_n += digits[j] * 10**j
					return a_n


if __name__ == "__main__":
					print(F"{solution() = }")



