






'''simple docstring'''





import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class  snake_case__      (					UpperCamelCase     ,				unittest.TestCase):
					a_        =					BertTokenizer
					a_        =					BertTokenizerFast
					a_        =					True
					a_        =					True
					a_        =					filter_non_english




					def 		A      (  self  :		List[str]      )    ->							Union[str, Any]:
								super().setUp()

								UpperCAmelCase_      :       Optional[Any]          =       [
								    '''[UNK]''',
								    '''[CLS]''',
								    '''[SEP]''',
								    '''[PAD]''',
								    '''[MASK]''',
								    '''want''',
								    '''##want''',
								    '''##ed''',
								    '''wa''',
								    '''un''',
								    '''runn''',
								    '''##ing''',
								    ''',''',
								    '''low''',
								    '''lowest''',
								]
								UpperCAmelCase_      :       str          =       os.path.join(self.tmpdirname				, VOCAB_FILES_NAMES['''vocab_file''']      )
								with open(self.vocab_file				, '''w'''				, encoding='''utf-8'''      ) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]      )      )




					def 		A      (  self  :		Union[str, Any]				, _A  :		Optional[int]      )    ->							Any:
								UpperCAmelCase_      :       List[str]          =       '''UNwant\u00E9d,running'''
								UpperCAmelCase_      :       int          =       '''unwanted, running'''
								return input_text, output_text




					def 		A      (  self  :		Union[str, Any]      )    ->							Tuple:
								UpperCAmelCase_      :       Optional[Any]          =       self.tokenizer_class(self.vocab_file      )

								UpperCAmelCase_      :       List[str]          =       tokenizer.tokenize('''UNwant\u00E9d,running'''      )
								self.assertListEqual(_A				, ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']      )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_A      )				, [9, 6, 7, 12, 10, 11]      )




					def 		A      (  self  :		Union[str, Any]      )    ->							Tuple:
								if not self.test_rust_tokenizer:
											return

								UpperCAmelCase_      :       Optional[Any]          =       self.get_tokenizer()
								UpperCAmelCase_      :       Dict          =       self.get_rust_tokenizer()

								UpperCAmelCase_      :       Union[str, Any]          =       '''UNwant\u00E9d,running'''

								UpperCAmelCase_      :       Any          =       tokenizer.tokenize(_A      )
								UpperCAmelCase_      :       Dict          =       rust_tokenizer.tokenize(_A      )
								self.assertListEqual(_A				, _A      )

								UpperCAmelCase_      :       List[str]          =       tokenizer.encode(_A				, add_special_tokens=_A      )
								UpperCAmelCase_      :       Tuple          =       rust_tokenizer.encode(_A				, add_special_tokens=_A      )
								self.assertListEqual(_A				, _A      )

								UpperCAmelCase_      :       Any          =       self.get_rust_tokenizer()
								UpperCAmelCase_      :       str          =       tokenizer.encode(_A      )
								UpperCAmelCase_      :       List[str]          =       rust_tokenizer.encode(_A      )
								self.assertListEqual(_A				, _A      )

								# With lower casing
								UpperCAmelCase_      :       Tuple          =       self.get_tokenizer(do_lower_case=_A      )
								UpperCAmelCase_      :       Dict          =       self.get_rust_tokenizer(do_lower_case=_A      )

								UpperCAmelCase_      :       List[Any]          =       '''UNwant\u00E9d,running'''

								UpperCAmelCase_      :       int          =       tokenizer.tokenize(_A      )
								UpperCAmelCase_      :       List[Any]          =       rust_tokenizer.tokenize(_A      )
								self.assertListEqual(_A				, _A      )

								UpperCAmelCase_      :       Union[str, Any]          =       tokenizer.encode(_A				, add_special_tokens=_A      )
								UpperCAmelCase_      :       List[str]          =       rust_tokenizer.encode(_A				, add_special_tokens=_A      )
								self.assertListEqual(_A				, _A      )

								UpperCAmelCase_      :       List[Any]          =       self.get_rust_tokenizer()
								UpperCAmelCase_      :       Dict          =       tokenizer.encode(_A      )
								UpperCAmelCase_      :       int          =       rust_tokenizer.encode(_A      )
								self.assertListEqual(_A				, _A      )




					def 		A      (  self  :		Union[str, Any]      )    ->							List[Any]:
								UpperCAmelCase_      :       Optional[Any]          =       BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''      )				, ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']      )




					def 		A      (  self  :		Union[str, Any]      )    ->							Union[str, Any]:
								UpperCAmelCase_      :       Union[str, Any]          =       BasicTokenizer(do_lower_case=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''      )				, ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )				, ['''hello''']      )




					def 		A      (  self  :		int      )    ->							int:
								UpperCAmelCase_      :       Optional[int]          =       BasicTokenizer(do_lower_case=_A				, strip_accents=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )				, ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )				, ['''h\u00E9llo''']      )




					def 		A      (  self  :		Tuple      )    ->							int:
								UpperCAmelCase_      :       List[str]          =       BasicTokenizer(do_lower_case=_A				, strip_accents=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )				, ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )				, ['''hello''']      )




					def 		A      (  self  :		List[str]      )    ->							Optional[Any]:
								UpperCAmelCase_      :       str          =       BasicTokenizer(do_lower_case=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )				, ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )				, ['''hello''']      )




					def 		A      (  self  :		int      )    ->							Union[str, Any]:
								UpperCAmelCase_      :       Optional[int]          =       BasicTokenizer(do_lower_case=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''      )				, ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )




					def 		A      (  self  :		str      )    ->							Optional[Any]:
								UpperCAmelCase_      :       int          =       BasicTokenizer(do_lower_case=_A				, strip_accents=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )				, ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )




					def 		A      (  self  :		Union[str, Any]      )    ->							List[Any]:
								UpperCAmelCase_      :       Optional[Any]          =       BasicTokenizer(do_lower_case=_A				, strip_accents=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )				, ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )




					def 		A      (  self  :		Optional[int]      )    ->							str:
								UpperCAmelCase_      :       Any          =       BasicTokenizer(do_lower_case=_A				, never_split=['''[UNK]''']      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''      )				, ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']      )




					def 		A      (  self  :		List[Any]      )    ->							Optional[Any]:
								UpperCAmelCase_      :       Tuple          =       BasicTokenizer()
								UpperCAmelCase_      :       str          =       '''a\n\'ll !!to?\'d of, can\'t.'''
								UpperCAmelCase_      :       List[str]          =       ['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
								self.assertListEqual(tokenizer.tokenize(_A      )				, _A      )




					def 		A      (  self  :		List[str]      )    ->							Tuple:
								UpperCAmelCase_      :       Optional[Any]          =       ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

								UpperCAmelCase_      :       Optional[int]          =       {}
								for i, token in enumerate(_A      ):
											UpperCAmelCase_      :       List[str]          =       i
								UpperCAmelCase_      :       Any          =       WordpieceTokenizer(vocab=_A				, unk_token='''[UNK]'''      )

								self.assertListEqual(tokenizer.tokenize(''''''      )				, []      )

								self.assertListEqual(tokenizer.tokenize('''unwanted running'''      )				, ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']      )

								self.assertListEqual(tokenizer.tokenize('''unwantedX running'''      )				, ['''[UNK]''', '''runn''', '''##ing''']      )




					def 		A      (  self  :		int      )    ->							List[Any]:
								self.assertTrue(_is_whitespace(''' '''      )      )
								self.assertTrue(_is_whitespace('''\t'''      )      )
								self.assertTrue(_is_whitespace('''\r'''      )      )
								self.assertTrue(_is_whitespace('''\n'''      )      )
								self.assertTrue(_is_whitespace('''\u00A0'''      )      )

								self.assertFalse(_is_whitespace('''A'''      )      )
								self.assertFalse(_is_whitespace('''-'''      )      )




					def 		A      (  self  :		List[Any]      )    ->							Optional[Any]:
								self.assertTrue(_is_control('''\u0005'''      )      )

								self.assertFalse(_is_control('''A'''      )      )
								self.assertFalse(_is_control(''' '''      )      )
								self.assertFalse(_is_control('''\t'''      )      )
								self.assertFalse(_is_control('''\r'''      )      )




					def 		A      (  self  :		Union[str, Any]      )    ->							Union[str, Any]:
								self.assertTrue(_is_punctuation('''-'''      )      )
								self.assertTrue(_is_punctuation('''$'''      )      )
								self.assertTrue(_is_punctuation('''`'''      )      )
								self.assertTrue(_is_punctuation('''.'''      )      )

								self.assertFalse(_is_punctuation('''A'''      )      )
								self.assertFalse(_is_punctuation(''' '''      )      )




					def 		A      (  self  :		Optional[Any]      )    ->							str:
								UpperCAmelCase_      :       Optional[int]          =       self.get_tokenizer()
								UpperCAmelCase_      :       List[Any]          =       self.get_rust_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(_A      ) for t in ['''Test''', '''\xad''', '''test''']]				, [['''[UNK]'''], [], ['''[UNK]''']]      )

								self.assertListEqual(
								    [rust_tokenizer.tokenize(_A      ) for t in ['''Test''', '''\xad''', '''test''']]				, [['''[UNK]'''], [], ['''[UNK]''']]      )




					@slow
					def 		A      (  self  :		List[Any]      )    ->							List[Any]:
								UpperCAmelCase_      :       Optional[Any]          =       self.tokenizer_class.from_pretrained('''bert-base-uncased'''      )

								UpperCAmelCase_      :       Optional[int]          =       tokenizer.encode('''sequence builders'''				, add_special_tokens=_A      )
								UpperCAmelCase_      :       Tuple          =       tokenizer.encode('''multi-sequence build'''				, add_special_tokens=_A      )

								UpperCAmelCase_      :       Optional[Any]          =       tokenizer.build_inputs_with_special_tokens(_A      )
								UpperCAmelCase_      :       str          =       tokenizer.build_inputs_with_special_tokens(_A				, _A      )

								assert encoded_sentence == [1_01] + text + [1_02]
								assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]




					def 		A      (  self  :		Tuple      )    ->							Optional[Any]:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
														UpperCAmelCase_      :       int          =       self.rust_tokenizer_class.from_pretrained(_A				, **_A      )

														UpperCAmelCase_      :       int          =       F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
														UpperCAmelCase_      :       str          =       tokenizer_r.encode_plus(
														    _A				, return_attention_mask=_A				, return_token_type_ids=_A				, return_offsets_mapping=_A				, add_special_tokens=_A				, )

														UpperCAmelCase_      :       Optional[Any]          =       tokenizer_r.do_lower_case if hasattr(_A				, '''do_lower_case'''      ) else False
														UpperCAmelCase_      :       Tuple          =       (
														    [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), '''A'''),
														        ((1, 2), ''','''),
														        ((3, 5), '''na'''),
														        ((5, 6), '''##ï'''),
														        ((6, 8), '''##ve'''),
														        ((9, 15), tokenizer_r.mask_token),
														        ((16, 21), '''Allen'''),
														        ((21, 23), '''##NL'''),
														        ((23, 24), '''##P'''),
														        ((25, 33), '''sentence'''),
														        ((33, 34), '''.'''),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														    if not do_lower_case
														    else [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), '''a'''),
														        ((1, 2), ''','''),
														        ((3, 8), '''naive'''),
														        ((9, 15), tokenizer_r.mask_token),
														        ((16, 21), '''allen'''),
														        ((21, 23), '''##nl'''),
														        ((23, 24), '''##p'''),
														        ((25, 33), '''sentence'''),
														        ((33, 34), '''.'''),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														)

														self.assertEqual(
														    [e[1] for e in expected_results]				, tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']      )      )
														self.assertEqual([e[0] for e in expected_results]				, tokens['''offset_mapping''']      )





					def 		A      (  self  :		Optional[Any]      )    ->							Optional[Any]:
								UpperCAmelCase_      :       List[Any]          =       ['''的''', '''人''', '''有''']
								UpperCAmelCase_      :       str          =       ''''''.join(_A      )
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
														UpperCAmelCase_      :       Optional[int]          =       True
														UpperCAmelCase_      :       Any          =       self.tokenizer_class.from_pretrained(_A				, **_A      )
														UpperCAmelCase_      :       Union[str, Any]          =       self.rust_tokenizer_class.from_pretrained(_A				, **_A      )

														UpperCAmelCase_      :       Tuple          =       tokenizer_p.encode(_A				, add_special_tokens=_A      )
														UpperCAmelCase_      :       Union[str, Any]          =       tokenizer_r.encode(_A				, add_special_tokens=_A      )

														UpperCAmelCase_      :       Union[str, Any]          =       tokenizer_r.convert_ids_to_tokens(_A      )
														UpperCAmelCase_      :       str          =       tokenizer_p.convert_ids_to_tokens(_A      )

														# it is expected that each Chinese character is not preceded by "##"
														self.assertListEqual(_A				, _A      )
														self.assertListEqual(_A				, _A      )

														UpperCAmelCase_      :       int          =       False
														UpperCAmelCase_      :       Tuple          =       self.rust_tokenizer_class.from_pretrained(_A				, **_A      )
														UpperCAmelCase_      :       Tuple          =       self.tokenizer_class.from_pretrained(_A				, **_A      )

														UpperCAmelCase_      :       int          =       tokenizer_r.encode(_A				, add_special_tokens=_A      )
														UpperCAmelCase_      :       Any          =       tokenizer_p.encode(_A				, add_special_tokens=_A      )

														UpperCAmelCase_      :       Dict          =       tokenizer_r.convert_ids_to_tokens(_A      )
														UpperCAmelCase_      :       str          =       tokenizer_p.convert_ids_to_tokens(_A      )

														# it is expected that only the first Chinese character is not preceded by "##".
														UpperCAmelCase_      :       int          =       [
														    F"##{token}" if idx != 0 else token for idx, token in enumerate(_A      )
														]
														self.assertListEqual(_A				, _A      )
														self.assertListEqual(_A				, _A      )










'''simple docstring'''





_UpperCamelCase						:						Dict  =					range(2, 20 + 1)
_UpperCamelCase						:						str  =					[10**k for k in range(ks[-1] + 1)]
_UpperCamelCase						:						dict[int, dict[int, list[list[int]]]]  =					{}
def __UpperCAmelCase   (					A					:			List[Any]			,       A					:			str			,       A					:			Union[str, Any]			,       A					:			Union[str, Any]		)    ->   Optional[Any]:
			UpperCAmelCase_      :       Optional[Any]          =       sum(a_i[j] for j in range(A			,       len(A		)		)		)
			UpperCAmelCase_      :       Union[str, Any]          =       sum(a_i[j] * base[j] for j in range(min(len(A		)			,       A		)		)		)

			UpperCAmelCase_    ,      UpperCAmelCase_      :       Optional[int]          =       0, 0
			UpperCAmelCase_      :       str          =       n - i

			UpperCAmelCase_      :       Union[str, Any]          =       memo.get(A		)

			if sub_memo is not None:
						UpperCAmelCase_      :       Dict          =       sub_memo.get(A		)

						if jumps is not None and len(A		) > 0:
									# find and make the largest jump without going over
									UpperCAmelCase_      :       str          =       -1
									for _k in range(len(A		) - 1			,       -1			,       -1		):
												if jumps[_k][2] <= k and jumps[_k][1] <= max_dn:
															UpperCAmelCase_      :       Any          =       _k
															break

									if max_jump >= 0:
												UpperCAmelCase_    ,      UpperCAmelCase_    ,      UpperCAmelCase_      :       Any          =       jumps[max_jump]
												# since the difference between jumps is cached, add c
												UpperCAmelCase_      :       Union[str, Any]          =       diff + c
												for j in range(min(A			,       len(A		)		)		):
															UpperCAmelCase_    ,      UpperCAmelCase_      :       Tuple          =       divmod(A			,       1_0		)
												if new_c > 0:
															add(A			,       A			,       A		)

						else:
									UpperCAmelCase_      :       Tuple          =       []
			else:
						UpperCAmelCase_      :       List[str]          =       {c: []}
						UpperCAmelCase_      :       List[str]          =       sub_memo

			if dn >= max_dn or c + diff >= base[k]:
						return diff, dn

			if k > ks[0]:
						while True:
									# keep doing smaller jumps
									UpperCAmelCase_    ,      UpperCAmelCase_      :       Optional[Any]          =       next_term(A			,       k - 1			,       i + dn			,       A		)
									diff += _diff
									dn += terms_jumped

									if dn >= max_dn or c + diff >= base[k]:
												break
			else:
						# would be too small a jump, just compute sequential terms instead
						UpperCAmelCase_    ,      UpperCAmelCase_      :       int          =       compute(A			,       A			,       i + dn			,       A		)
						diff += _diff
						dn += terms_jumped

			UpperCAmelCase_      :       List[str]          =       sub_memo[c]

			# keep jumps sorted by # of terms skipped
			UpperCAmelCase_      :       List[Any]          =       0
			while j < len(A		):
						if jumps[j][1] > dn:
									break
						j += 1

			# cache the jump for this value digitsum(b) and c
			sub_memo[c].insert(A			,       (diff, dn, k)		)
			return (diff, dn)
def __UpperCAmelCase   (					A					:			List[str]			,       A					:			Dict			,       A					:			List[str]			,       A					:			Union[str, Any]		)    ->   str:
			if i >= n:
						return 0, i
			if k > len(A		):
						a_i.extend([0 for _ in range(k - len(A		)		)]		)

			# note: a_i -> b * 10^k + c
			# ds_b -> digitsum(b)
			# ds_c -> digitsum(c)
			UpperCAmelCase_      :       int          =       i
			UpperCAmelCase_    ,      UpperCAmelCase_    ,      UpperCAmelCase_      :       List[Any]          =       0, 0, 0
			for j in range(len(A		)		):
						if j >= k:
									ds_b += a_i[j]
						else:
									ds_c += a_i[j]

			while i < n:
						i += 1
						UpperCAmelCase_      :       Optional[int]          =       ds_c + ds_b
						diff += addend
						UpperCAmelCase_      :       Any          =       0
						for j in range(A		):
									UpperCAmelCase_      :       Dict          =       a_i[j] + addend
									UpperCAmelCase_    ,      UpperCAmelCase_      :       Tuple          =       divmod(A			,       1_0		)

									ds_c += a_i[j]

						if addend > 0:
									break

			if addend > 0:
						add(A			,       A			,       A		)
			return diff, i - start_i
def __UpperCAmelCase   (					A					:			Dict			,       A					:			Union[str, Any]			,       A					:			str		)    ->   List[Any]:
			for j in range(A			,       len(A		)		):
						UpperCAmelCase_      :       List[str]          =       digits[j] + addend
						if s >= 1_0:
									UpperCAmelCase_    ,      UpperCAmelCase_      :       Any          =       divmod(A			,       1_0		)
									UpperCAmelCase_      :       str          =       addend // 1_0 + quotient
						else:
									UpperCAmelCase_      :       Any          =       s
									UpperCAmelCase_      :       Any          =       addend // 1_0

						if addend == 0:
									break

			while addend > 0:
						UpperCAmelCase_    ,      UpperCAmelCase_      :       Union[str, Any]          =       divmod(A			,       1_0		)
						digits.append(A		)
def __UpperCAmelCase   (					A					:			int = 1_0**1_5		)    ->   int:
			UpperCAmelCase_      :       Any          =       [1]
			UpperCAmelCase_      :       Union[str, Any]          =       1
			UpperCAmelCase_      :       List[Any]          =       0
			while True:
						UpperCAmelCase_    ,      UpperCAmelCase_      :       str          =       next_term(A			,       2_0			,       i + dn			,       A		)
						dn += terms_jumped
						if dn == n - i:
									break

			UpperCAmelCase_      :       Any          =       0
			for j in range(len(A		)		):
						a_n += digits[j] * 1_0**j
			return a_n


if __name__ == "__main__":
						print(f'''{solution() = }''')


