






"""simple docstring"""







import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__a       :	List[str]             = logging.get_logger(__name__)

__a       :	Optional[int]             = {'vocab_file': 'spiece.model'}

__a       :	List[Any]             = {
    'vocab_file': {
        'xlnet-base-cased': 'https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model',
        'xlnet-large-cased': 'https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model',
    }
}

__a       :	List[str]             = {
    'xlnet-base-cased': None,
    'xlnet-large-cased': None,
}

# Segments (not really needed)
__a       :	Dict             = 0
__a       :	int             = 1
__a       :	Union[str, Any]             = 2
__a       :	Dict             = 3
__a       :	List[str]             = 4







class        _SCREAMING_SNAKE_CASE						(  __snake_case							):





							"""simple docstring"""





							_SCREAMING_SNAKE_CASE    =VOCAB_FILES_NAMES
							_SCREAMING_SNAKE_CASE    =PRETRAINED_VOCAB_FILES_MAP
							_SCREAMING_SNAKE_CASE    =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_SCREAMING_SNAKE_CASE    ='left'




							def __init__(   self:     List[str]				,		__A:     int				,		__A:     int=False				,		__A:     int=True				,		__A:     Union[str, Any]=False				,		__A:     Dict="<s>"				,		__A:     Optional[Any]="</s>"				,		__A:     Dict="<unk>"				,		__A:     int="<sep>"				,		__A:     Union[str, Any]="<pad>"				,		__A:     Any="<cls>"				,		__A:     int="<mask>"				,		__A:     str=["<eop>", "<eod>"]				,		__A:     Optional[Dict[str, Any]] = None				,		**__A:     List[str]				,		):

												'''simple docstring'''

												a__          =  AddedToken(__A				,		lstrip=__A				,		rstrip=__A    ) if isinstance(__A				,		__A    ) else mask_token

												a__          =  {} if sp_model_kwargs is None else sp_model_kwargs

												super().__init__(
												    do_lower_case=__A				,		remove_space=__A				,		keep_accents=__A				,		bos_token=__A				,		eos_token=__A				,		unk_token=__A				,		sep_token=__A				,		pad_token=__A				,		cls_token=__A				,		mask_token=__A				,		additional_special_tokens=__A				,		sp_model_kwargs=self.sp_model_kwargs				,		**__A				,		)

												a__          =  3

												a__          =  do_lower_case
												a__          =  remove_space
												a__          =  keep_accents
												a__          =  vocab_file

												a__          =  spm.SentencePieceProcessor(**self.sp_model_kwargs    )
												self.sp_model.Load(__A    )




							@property
							def  lowercase		(   self:     List[Any]    ):

												'''simple docstring'''

												return len(self.sp_model    )




							def  lowercase		(   self:     Optional[int]    ):

												'''simple docstring'''

												a__          =  {self.convert_ids_to_tokens(__A    ): i for i in range(self.vocab_size    )}
												vocab.update(self.added_tokens_encoder    )
												return vocab




							def __getstate__(   self:     List[Any]    ):

												'''simple docstring'''

												a__          =  self.__dict__.copy()
												a__          =  None
												return state




							def __setstate__(   self:     Optional[Any]				,		__A:     List[str]    ):

												'''simple docstring'''

												a__          =  d

												# for backward compatibility
												if not hasattr(self				,		'''sp_model_kwargs'''    ):
																	a__          =  {}

												a__          =  spm.SentencePieceProcessor(**self.sp_model_kwargs    )
												self.sp_model.Load(self.vocab_file    )




							def  lowercase		(   self:     Dict				,		__A:     Optional[Any]    ):

												'''simple docstring'''

												if self.remove_space:
																	a__          =  ''' '''.join(inputs.strip().split()    )
												else:
																	a__          =  inputs
												a__          =  outputs.replace('''``'''				,		'''"'''    ).replace('''\'\''''				,		'''"'''    )

												if not self.keep_accents:
																	a__          =  unicodedata.normalize('''NFKD'''				,		__A    )
																	a__          =  ''''''.join([c for c in outputs if not unicodedata.combining(__A    )]    )
												if self.do_lower_case:
																	a__          =  outputs.lower()

												return outputs




							def  lowercase		(   self:     Dict				,		__A:     str    ):

												'''simple docstring'''

												a__          =  self.preprocess_text(__A    )
												a__          =  self.sp_model.encode(__A				,		out_type=__A    )
												a__          =  []
												for piece in pieces:
																	if len(__A    ) > 1 and piece[-1] == str(''','''    ) and piece[-2].isdigit():
																						a__          =  self.sp_model.EncodeAsPieces(piece[:-1].replace(__A				,		''''''    )    )
																						if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																											if len(cur_pieces[0]    ) == 1:
																																a__          =  cur_pieces[1:]
																											else:
																																a__          =  cur_pieces[0][1:]
																						cur_pieces.append(piece[-1]    )
																						new_pieces.extend(__A    )
																	else:
																						new_pieces.append(__A    )

												return new_pieces




							def  lowercase		(   self:     Optional[int]				,		__A:     Tuple    ):

												'''simple docstring'''

												return self.sp_model.PieceToId(__A    )




							def  lowercase		(   self:     Any				,		__A:     Union[str, Any]    ):

												'''simple docstring'''

												return self.sp_model.IdToPiece(__A    )




							def  lowercase		(   self:     Dict				,		__A:     Tuple    ):

												'''simple docstring'''

												a__          =  ''''''.join(__A    ).replace(__A				,		''' '''    ).strip()
												return out_string




							def  lowercase		(   self:     List[Any]				,		__A:     List[int]				,		__A:     bool = False				,		__A:     bool = None				,		__A:     bool = True				,		**__A:     str				,		):

												'''simple docstring'''

												a__          =  kwargs.pop('''use_source_tokenizer'''				,		__A    )

												a__          =  self.convert_ids_to_tokens(__A				,		skip_special_tokens=__A    )

												# To avoid mixing byte-level and unicode for byte-level BPT
												# we need to build string separately for added tokens and byte-level tokens
												# cf. https://github.com/huggingface/transformers/issues/1133
												a__          =  []
												a__          =  []
												for token in filtered_tokens:
																	if skip_special_tokens and token in self.all_special_ids:
																						continue
																	if token in self.added_tokens_encoder:
																						if current_sub_text:
																											sub_texts.append(self.convert_tokens_to_string(__A    )    )
																											a__          =  []
																						sub_texts.append(__A    )
																	else:
																						current_sub_text.append(__A    )
												if current_sub_text:
																	sub_texts.append(self.convert_tokens_to_string(__A    )    )

												# Mimic the behavior of the Rust tokenizer:
												# By default, there are no spaces between special tokens
												a__          =  ''''''.join(__A    )

												a__          =  (
												    clean_up_tokenization_spaces
												    if clean_up_tokenization_spaces is not None
												    else self.clean_up_tokenization_spaces
												)
												if clean_up_tokenization_spaces:
																	a__          =  self.clean_up_tokenization(__A    )
																	return clean_text
												else:
																	return text




							def  lowercase		(   self:     List[str]				,		__A:     List[int]				,		__A:     Optional[List[int]] = None    ):

												'''simple docstring'''

												a__          =  [self.sep_token_id]
												a__          =  [self.cls_token_id]
												if token_ids_a is None:
																	return token_ids_a + sep + cls
												return token_ids_a + sep + token_ids_a + sep + cls




							def  lowercase		(   self:     List[Any]				,		__A:     List[int]				,		__A:     Optional[List[int]] = None				,		__A:     bool = False    ):

												'''simple docstring'''

												if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=__A				,		token_ids_a=__A				,		already_has_special_tokens=__A    )

												if token_ids_a is not None:
																	return ([0] * len(__A    )) + [1] + ([0] * len(__A    )) + [1, 1]
												return ([0] * len(__A    )) + [1, 1]




							def  lowercase		(   self:     Optional[Any]				,		__A:     List[int]				,		__A:     Optional[List[int]] = None    ):

												'''simple docstring'''

												a__          =  [self.sep_token_id]
												a__          =  [2]

												if token_ids_a is None:
																	return len(token_ids_a + sep    ) * [0] + cls_segment_id
												return len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1] + cls_segment_id





							def  lowercase		(   self:     int				,		__A:     str				,		__A:     Optional[str] = None    ):

												'''simple docstring'''

												if not os.path.isdir(__A    ):
																	logger.error(F'Vocabulary path ({save_directory}) should be a directory'    )
																	return
												a__          =  os.path.join(
												    __A				,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )

												if os.path.abspath(self.vocab_file    ) != os.path.abspath(__A    ) and os.path.isfile(self.vocab_file    ):
																	copyfile(self.vocab_file				,		__A    )
												elif not os.path.isfile(self.vocab_file    ):
																	with open(__A				,		'''wb'''    ) as fi:
																						a__          =  self.sp_model.serialized_model_proto()
																						fi.write(__A    )

												return (out_vocab_file,)










"""simple docstring"""







__a       :	Union[str, Any]             = range(2, 20 + 1)
__a       :	Any             = [10**k for k in range(ks[-1] + 1)]
__a       :	dict[int, dict[int, list[list[int]]]]             = {}





def        SCREAMING_SNAKE_CASE       (      lowerCamelCase_ ,     lowerCamelCase_ ,     lowerCamelCase_ ,     lowerCamelCase_):
					a__          =  sum(a_i[j] for j in range(lowerCamelCase_ ,     len(lowerCamelCase_)))
					a__          =  sum(a_i[j] * base[j] for j in range(min(len(lowerCamelCase_) ,     lowerCamelCase_)))

					a__       ,a__          =  0, 0
					a__          =  n - i

					a__          =  memo.get(lowerCamelCase_)

					if sub_memo is not None:
										a__          =  sub_memo.get(lowerCamelCase_)

										if jumps is not None and len(lowerCamelCase_) > 0:
															# find and make the largest jump without going over
															a__          =  -1
															for _k in range(len(lowerCamelCase_) - 1 ,     -1 ,     -1):
																				if jumps[_k][2] <= k and jumps[_k][1] <= max_dn:
																									a__          =  _k
																									break

															if max_jump >= 0:
																				a__       ,a__       ,a__          =  jumps[max_jump]
																				# since the difference between jumps is cached, add c
																				a__          =  diff + c
																				for j in range(min(lowerCamelCase_ ,     len(lowerCamelCase_))):
																									a__       ,a__          =  divmod(lowerCamelCase_ ,     10)
																				if new_c > 0:
																									add(lowerCamelCase_ ,     lowerCamelCase_ ,     lowerCamelCase_)

										else:
															a__          =  []
					else:
										a__          =  {c: []}
										a__          =  sub_memo

					if dn >= max_dn or c + diff >= base[k]:
										return diff, dn

					if k > ks[0]:
										while True:
															# keep doing smaller jumps
															a__       ,a__          =  next_term(lowerCamelCase_ ,     k - 1 ,     i + dn ,     lowerCamelCase_)
															diff += _diff
															dn += terms_jumped

															if dn >= max_dn or c + diff >= base[k]:
																				break
					else:
										# would be too small a jump, just compute sequential terms instead
										a__       ,a__          =  compute(lowerCamelCase_ ,     lowerCamelCase_ ,     i + dn ,     lowerCamelCase_)
										diff += _diff
										dn += terms_jumped

					a__          =  sub_memo[c]

					# keep jumps sorted by # of terms skipped
					a__          =  0
					while j < len(lowerCamelCase_):
										if jumps[j][1] > dn:
															break
										j += 1

					# cache the jump for this value digitsum(b) and c
					sub_memo[c].insert(lowerCamelCase_ ,     (diff, dn, k))
					return (diff, dn)





def        SCREAMING_SNAKE_CASE       (      lowerCamelCase_ ,     lowerCamelCase_ ,     lowerCamelCase_ ,     lowerCamelCase_):
					if i >= n:
										return 0, i
					if k > len(lowerCamelCase_):
										a_i.extend([0 for _ in range(k - len(lowerCamelCase_))])

					# note: a_i -> b * 10^k + c
					# ds_b -> digitsum(b)
					# ds_c -> digitsum(c)
					a__          =  i
					a__       ,a__       ,a__          =  0, 0, 0
					for j in range(len(lowerCamelCase_)):
										if j >= k:
															ds_b += a_i[j]
										else:
															ds_c += a_i[j]

					while i < n:
										i += 1
										a__          =  ds_c + ds_b
										diff += addend
										a__          =  0
										for j in range(lowerCamelCase_):
															a__          =  a_i[j] + addend
															a__       ,a__          =  divmod(lowerCamelCase_ ,     10)

															ds_c += a_i[j]

										if addend > 0:
															break

					if addend > 0:
										add(lowerCamelCase_ ,     lowerCamelCase_ ,     lowerCamelCase_)
					return diff, i - start_i





def        SCREAMING_SNAKE_CASE       (      lowerCamelCase_ ,     lowerCamelCase_ ,     lowerCamelCase_):
					for j in range(lowerCamelCase_ ,     len(lowerCamelCase_)):
										a__          =  digits[j] + addend
										if s >= 10:
															a__       ,a__          =  divmod(lowerCamelCase_ ,     10)
															a__          =  addend // 10 + quotient
										else:
															a__          =  s
															a__          =  addend // 10

										if addend == 0:
															break

					while addend > 0:
										a__       ,a__          =  divmod(lowerCamelCase_ ,     10)
										digits.append(lowerCamelCase_)





def        SCREAMING_SNAKE_CASE       (      lowerCamelCase_ = 10**15):
					a__          =  [1]
					a__          =  1
					a__          =  0
					while True:
										a__       ,a__          =  next_term(lowerCamelCase_ ,     20 ,     i + dn ,     lowerCamelCase_)
										dn += terms_jumped
										if dn == n - i:
															break

					a__          =  0
					for j in range(len(lowerCamelCase_)):
										a_n += digits[j] * 10**j
					return a_n


if __name__ == "__main__":
	print(F'''{solution() = }''')


