


'''simple docstring'''
UpperCAmelCase_ :     Tuple								=      range(2, 2_0 + 1)
UpperCAmelCase_ :     Union[str, Any]								=      [1_0**k for k in range(ks[-1] + 1)]
UpperCAmelCase_ :     dict[int, dict[int, list[list[int]]]]								=      {}


def 			_lowercase       (				UpperCamelCase__       :					List[str],				UpperCamelCase__       :					Dict,				UpperCamelCase__       :					List[Any],				UpperCamelCase__       :					str			):
 __A  :		Optional[int]       =			sum(a_i[j] for j in range(UpperCamelCase__,				len(UpperCamelCase__			)			)			)
 __A  :		List[str]       =			sum(a_i[j] * base[j] for j in range(min(len(UpperCamelCase__			),				UpperCamelCase__			)			)			)

 __A     ,__A  :		Dict       =			0, 0
 __A  :		Tuple       =			n - i

 __A  :		Optional[Any]       =			memo.get(UpperCamelCase__			)

 if sub_memo is not None:
  __A  :		Any       =			sub_memo.get(UpperCamelCase__			)

  if jumps is not None and len(UpperCamelCase__			) > 0:
   # find and make the largest jump without going over
   __A  :		List[Any]       =			-1
   for _k in range(len(UpperCamelCase__			) - 1,				-1,				-1			):
    if jumps[_k][2] <= k and jumps[_k][1] <= max_dn:
     __A  :		List[str]       =			_k
     break

   if max_jump >= 0:
    __A     ,__A     ,__A  :		Dict       =			jumps[max_jump]
    # since the difference between jumps is cached, add c
    __A  :		Optional[int]       =			diff + c
    for j in range(min(UpperCamelCase__,				len(UpperCamelCase__			)			)			):
     __A     ,__A  :		List[str]       =			divmod(UpperCamelCase__,				10			)
    if new_c > 0:
     add(UpperCamelCase__,				UpperCamelCase__,				UpperCamelCase__			)

  else:
   __A  :		int       =			[]
 else:
  __A  :		Union[str, Any]       =			{c: []}
  __A  :		Dict       =			sub_memo

 if dn >= max_dn or c + diff >= base[k]:
  return diff, dn

 if k > ks[0]:
  while True:
   # keep doing smaller jumps
   __A     ,__A  :		Dict       =			next_term(UpperCamelCase__,				k - 1,				i + dn,				UpperCamelCase__			)
   diff += _diff
   dn += terms_jumped

   if dn >= max_dn or c + diff >= base[k]:
    break
 else:
  # would be too small a jump, just compute sequential terms instead
  __A     ,__A  :		Dict       =			compute(UpperCamelCase__,				UpperCamelCase__,				i + dn,				UpperCamelCase__			)
  diff += _diff
  dn += terms_jumped

 __A  :		List[Any]       =			sub_memo[c]

 # keep jumps sorted by # of terms skipped
 __A  :		List[Any]       =			0
 while j < len(UpperCamelCase__			):
  if jumps[j][1] > dn:
   break
  j += 1

 # cache the jump for this value digitsum(b) and c
 sub_memo[c].insert(UpperCamelCase__,				(diff, dn, k)			)
 return (diff, dn)


def 			_lowercase       (				UpperCamelCase__       :					Any,				UpperCamelCase__       :					int,				UpperCamelCase__       :					Union[str, Any],				UpperCamelCase__       :					Optional[Any]			):
 if i >= n:
  return 0, i
 if k > len(UpperCamelCase__			):
  a_i.extend([0 for _ in range(k - len(UpperCamelCase__			)			)]			)

 # note: a_i -> b * 10^k + c
 # ds_b -> digitsum(b)
 # ds_c -> digitsum(c)
 __A  :		Dict       =			i
 __A     ,__A     ,__A  :		str       =			0, 0, 0
 for j in range(len(UpperCamelCase__			)			):
  if j >= k:
   ds_b += a_i[j]
  else:
   ds_c += a_i[j]

 while i < n:
  i += 1
  __A  :		Union[str, Any]       =			ds_c + ds_b
  diff += addend
  __A  :		int       =			0
  for j in range(UpperCamelCase__			):
   __A  :		Any       =			a_i[j] + addend
   __A     ,__A  :		Optional[int]       =			divmod(UpperCamelCase__,				10			)

   ds_c += a_i[j]

  if addend > 0:
   break

 if addend > 0:
  add(UpperCamelCase__,				UpperCamelCase__,				UpperCamelCase__			)
 return diff, i - start_i


def 			_lowercase       (				UpperCamelCase__       :					Dict,				UpperCamelCase__       :					Any,				UpperCamelCase__       :					List[Any]			):
 for j in range(UpperCamelCase__,				len(UpperCamelCase__			)			):
  __A  :		Dict       =			digits[j] + addend
  if s >= 10:
   __A     ,__A  :		List[Any]       =			divmod(UpperCamelCase__,				10			)
   __A  :		Any       =			addend // 10 + quotient
  else:
   __A  :		Optional[int]       =			s
   __A  :		Any       =			addend // 10

  if addend == 0:
   break

 while addend > 0:
  __A     ,__A  :		Union[str, Any]       =			divmod(UpperCamelCase__,				10			)
  digits.append(UpperCamelCase__			)


def 			_lowercase       (				UpperCamelCase__       :					int = 10**15			):
 __A  :		Union[str, Any]       =			[1]
 __A  :		Optional[int]       =			1
 __A  :		Dict       =			0
 while True:
  __A     ,__A  :		Dict       =			next_term(UpperCamelCase__,				20,				i + dn,				UpperCamelCase__			)
  dn += terms_jumped
  if dn == n - i:
   break

 __A  :		str       =			0
 for j in range(len(UpperCamelCase__			)			):
  a_n += digits[j] * 10**j
 return a_n


if __name__ == "__main__":
 print(f'''{solution() = }''')










'''simple docstring'''
import argparse
import logging
import pickle
import random
import time

import numpy as np

from transformers import BertTokenizer, GPTaTokenizer, RobertaTokenizer


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO
)
UpperCAmelCase_ :     Any								=      logging.getLogger(__name__)


def 			_lowercase       (				):
 __A  :		Tuple       =			argparse.ArgumentParser(
     description='Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).'			)
 parser.add_argument('--file_path',				type=UpperCamelCase__,				default='data/dump.txt',				help='The path to the data.'			)
 parser.add_argument('--tokenizer_type',				type=UpperCamelCase__,				default='bert',				choices=['bert', 'roberta', 'gpt2']			)
 parser.add_argument('--tokenizer_name',				type=UpperCamelCase__,				default='bert-base-uncased',				help='The tokenizer to use.'			)
 parser.add_argument('--dump_file',				type=UpperCamelCase__,				default='data/dump',				help='The dump file prefix.'			)
 __A  :		Tuple       =			parser.parse_args()

 logger.info(f"""Loading Tokenizer ({args.tokenizer_name})"""			)
 if args.tokenizer_type == "bert":
  __A  :		Any       =			BertTokenizer.from_pretrained(args.tokenizer_name			)
  __A  :		Any       =			tokenizer.special_tokens_map['cls_token']  # `[CLS]`
  __A  :		List[Any]       =			tokenizer.special_tokens_map['sep_token']  # `[SEP]`
 elif args.tokenizer_type == "roberta":
  __A  :		Any       =			RobertaTokenizer.from_pretrained(args.tokenizer_name			)
  __A  :		List[Any]       =			tokenizer.special_tokens_map['cls_token']  # `<s>`
  __A  :		Dict       =			tokenizer.special_tokens_map['sep_token']  # `</s>`
 elif args.tokenizer_type == "gpt2":
  __A  :		List[str]       =			GPTaTokenizer.from_pretrained(args.tokenizer_name			)
  __A  :		Any       =			tokenizer.special_tokens_map['bos_token']  # `<|endoftext|>`
  __A  :		Union[str, Any]       =			tokenizer.special_tokens_map['eos_token']  # `<|endoftext|>`

 logger.info(f"""Loading text from {args.file_path}"""			)
 with open(args.file_path,				'r',				encoding='utf8'			) as fp:
  __A  :		Optional[int]       =			fp.readlines()

 logger.info('Start encoding'			)
 logger.info(f"""{len(UpperCamelCase__			)} examples to process."""			)

 __A  :		int       =			[]
 __A  :		List[str]       =			0
 __A  :		Tuple       =			10000
 __A  :		List[str]       =			time.time()
 for text in data:
  __A  :		int       =			f"""{bos} {text.strip()} {sep}"""
  __A  :		List[Any]       =			tokenizer.encode(UpperCamelCase__,				add_special_tokens=UpperCamelCase__			)
  rslt.append(UpperCamelCase__			)

  iter += 1
  if iter % interval == 0:
   __A  :		Any       =			time.time()
   logger.info(f"""{iter} examples processed. - {(end-start):.2f}s/{interval}expl"""			)
   __A  :		Any       =			time.time()
 logger.info('Finished binarization'			)
 logger.info(f"""{len(UpperCamelCase__			)} examples processed."""			)

 __A  :		List[str]       =			f"""{args.dump_file}.{args.tokenizer_name}.pickle"""
 __A  :		str       =			tokenizer.vocab_size
 if vocab_size < (1 << 16):
  __A  :		Optional[int]       =			[np.uintaa(UpperCamelCase__			) for d in rslt]
 else:
  __A  :		int       =			[np.intaa(UpperCamelCase__			) for d in rslt]
 random.shuffle(rslt_			)
 logger.info(f"""Dump to {dp_file}"""			)
 with open(UpperCamelCase__,				'wb'			) as handle:
  pickle.dump(rslt_,				UpperCamelCase__,				protocol=pickle.HIGHEST_PROTOCOL			)


if __name__ == "__main__":
 main()






