




import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin





class 					__lowerCamelCase			(       UpperCamelCase__      ,							unittest.TestCase       ):




    """simple docstring"""

    snake_case__   = PhobertTokenizer
    snake_case__   = False


    def   a (			self	:	Union[str, Any]  ) ->			str:
         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         lowerCAmelCase__            =					["T@@", "i", "I", "R@@", "r", "e@@"]
         lowerCAmelCase__            =					dict(zip(SCREAMING_SNAKE_CASE__  ,      range(len(SCREAMING_SNAKE_CASE__  )  )  )  )
         lowerCAmelCase__            =					["#version: 0.2", "l à</w>"]
         lowerCAmelCase__            =					{"unk_token": "<unk>"}

         lowerCAmelCase__            =					os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES["vocab_file"]  )
         lowerCAmelCase__            =					os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES["merges_file"]  )

         with open(self.vocab_file  ,      "w"  ,      encoding="utf-8"  ) as fp:
              for token in vocab_tokens:
                   fp.write(f'{token} {vocab_tokens[token]}\n'  )
         with open(self.merges_file  ,      "w"  ,      encoding="utf-8"  ) as fp:
              fp.write("\n".join(SCREAMING_SNAKE_CASE__  )  )


    def   a (			self	:	Tuple  ,      **SCREAMING_SNAKE_CASE__	:	int  ) ->			List[Any]:
         kwargs.update(self.special_tokens_map  )
         return PhobertTokenizer.from_pretrained(self.tmpdirname  ,      **SCREAMING_SNAKE_CASE__  )


    def   a (			self	:	Optional[Any]  ,      SCREAMING_SNAKE_CASE__	:	Optional[int]  ) ->			Optional[int]:
         lowerCAmelCase__            =					"Tôi là VinAI Research"
         lowerCAmelCase__            =					"T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>"
         return input_text, output_text


    def   a (			self	:	List[str]  ) ->			str:
         lowerCAmelCase__            =					PhobertTokenizer(self.vocab_file  ,      self.merges_file  ,      **self.special_tokens_map  )
         lowerCAmelCase__            =					"Tôi là VinAI Research"
         lowerCAmelCase__            =					"T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h".split()
         lowerCAmelCase__            =					tokenizer.tokenize(SCREAMING_SNAKE_CASE__  )
         print(SCREAMING_SNAKE_CASE__  )
         self.assertListEqual(SCREAMING_SNAKE_CASE__  ,      SCREAMING_SNAKE_CASE__  )

         lowerCAmelCase__            =					tokens + [tokenizer.unk_token]

         lowerCAmelCase__            =					[4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__  )  ,      SCREAMING_SNAKE_CASE__  )

"""simple docstring"""


a				=range(2, 20 + 1)
a				=[10**k for k in range(ks[-1] + 1)]
a				={}





def      lowerCamelCase_					(   __lowerCAmelCase  ,       __lowerCAmelCase  ,       __lowerCAmelCase  ,       __lowerCAmelCase							)				->   Any:




						'''simple docstring'''



						lowerCamelCase__   =sum(a_i[j] for j in range(__lowerCAmelCase  ,       len(__lowerCAmelCase							)							)							)
						lowerCamelCase__   =sum(a_i[j] * base[j] for j in range(min(len(__lowerCAmelCase							)  ,       __lowerCAmelCase							)							)							)

						lowerCamelCase__   , lowerCamelCase__   =0, 0
						lowerCamelCase__   =n - i

						lowerCamelCase__   =memo.get(__lowerCAmelCase							)

						if sub_memo is not None:
												lowerCamelCase__   =sub_memo.get(__lowerCAmelCase							)

												if jumps is not None and len(__lowerCAmelCase							) > 0:
																		# find and make the largest jump without going over
																		lowerCamelCase__   =-1
																		for _k in range(len(__lowerCAmelCase							) - 1  ,       -1  ,       -1							):
																								if jumps[_k][2] <= k and jumps[_k][1] <= max_dn:
																														lowerCamelCase__   =_k
																														break

																		if max_jump >= 0:
																								lowerCamelCase__   , lowerCamelCase__   , lowerCamelCase__   =jumps[max_jump]
																								# since the difference between jumps is cached, add c
																								lowerCamelCase__   =diff + c
																								for j in range(min(__lowerCAmelCase  ,       len(__lowerCAmelCase							)							)							):
																														lowerCamelCase__   , lowerCamelCase__   =divmod(__lowerCAmelCase  ,       10							)
																								if new_c > 0:
																														add(__lowerCAmelCase  ,       __lowerCAmelCase  ,       __lowerCAmelCase							)

												else:
																		lowerCamelCase__   =[]
						else:
												lowerCamelCase__   ={c: []}
												lowerCamelCase__   =sub_memo

						if dn >= max_dn or c + diff >= base[k]:
												return diff, dn

						if k > ks[0]:
												while True:
																		# keep doing smaller jumps
																		lowerCamelCase__   , lowerCamelCase__   =next_term(__lowerCAmelCase  ,       k - 1  ,       i + dn  ,       __lowerCAmelCase							)
																		diff += _diff
																		dn += terms_jumped

																		if dn >= max_dn or c + diff >= base[k]:
																								break
						else:
												# would be too small a jump, just compute sequential terms instead
												lowerCamelCase__   , lowerCamelCase__   =compute(__lowerCAmelCase  ,       __lowerCAmelCase  ,       i + dn  ,       __lowerCAmelCase							)
												diff += _diff
												dn += terms_jumped

						lowerCamelCase__   =sub_memo[c]

						# keep jumps sorted by # of terms skipped
						lowerCamelCase__   =0
						while j < len(__lowerCAmelCase							):
												if jumps[j][1] > dn:
																		break
												j += 1

						# cache the jump for this value digitsum(b) and c
						sub_memo[c].insert(__lowerCAmelCase  ,       (diff, dn, k)							)
						return (diff, dn)





def      lowerCamelCase_					(   __lowerCAmelCase  ,       __lowerCAmelCase  ,       __lowerCAmelCase  ,       __lowerCAmelCase							)				->   List[str]:




						'''simple docstring'''



						if i >= n:
												return 0, i
						if k > len(__lowerCAmelCase							):
												a_i.extend([0 for _ in range(k - len(__lowerCAmelCase							)							)]							)

						# note: a_i -> b * 10^k + c
						# ds_b -> digitsum(b)
						# ds_c -> digitsum(c)
						lowerCamelCase__   =i
						lowerCamelCase__   , lowerCamelCase__   , lowerCamelCase__   =0, 0, 0
						for j in range(len(__lowerCAmelCase							)							):
												if j >= k:
																		ds_b += a_i[j]
												else:
																		ds_c += a_i[j]

						while i < n:
												i += 1
												lowerCamelCase__   =ds_c + ds_b
												diff += addend
												lowerCamelCase__   =0
												for j in range(__lowerCAmelCase							):
																		lowerCamelCase__   =a_i[j] + addend
																		lowerCamelCase__   , lowerCamelCase__   =divmod(__lowerCAmelCase  ,       10							)

																		ds_c += a_i[j]

												if addend > 0:
																		break

						if addend > 0:
												add(__lowerCAmelCase  ,       __lowerCAmelCase  ,       __lowerCAmelCase							)
						return diff, i - start_i





def      lowerCamelCase_					(   __lowerCAmelCase  ,       __lowerCAmelCase  ,       __lowerCAmelCase							)				->   int:




						'''simple docstring'''



						for j in range(__lowerCAmelCase  ,       len(__lowerCAmelCase							)							):
												lowerCamelCase__   =digits[j] + addend
												if s >= 10:
																		lowerCamelCase__   , lowerCamelCase__   =divmod(__lowerCAmelCase  ,       10							)
																		lowerCamelCase__   =addend // 10 + quotient
												else:
																		lowerCamelCase__   =s
																		lowerCamelCase__   =addend // 10

												if addend == 0:
																		break

						while addend > 0:
												lowerCamelCase__   , lowerCamelCase__   =divmod(__lowerCAmelCase  ,       10							)
												digits.append(__lowerCAmelCase							)





def      lowerCamelCase_					(   __lowerCAmelCase = 10**15							)				->   int:




						'''simple docstring'''



						lowerCamelCase__   =[1]
						lowerCamelCase__   =1
						lowerCamelCase__   =0
						while True:
												lowerCamelCase__   , lowerCamelCase__   =next_term(__lowerCAmelCase  ,       20  ,       i + dn  ,       __lowerCAmelCase							)
												dn += terms_jumped
												if dn == n - i:
																		break

						lowerCamelCase__   =0
						for j in range(len(__lowerCAmelCase							)							):
												a_n += digits[j] * 10**j
						return a_n


if __name__ == "__main__":
	print(F'''{solution() = }''')


