





import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
		_lowercase		:  Optional[Any]			       ="""pt"""
elif is_tf_available():
		_lowercase		:  str			       ="""tf"""
else:
		_lowercase		:  List[Any]			       ="""jax"""






class 						UpperCamelCase_  (  snake_case__  ,     unittest.TestCase							):
					_a      :						int						    =					PerceiverTokenizer
					_a      :						int						    =					False



					def   __a     (		self     :     Tuple						):
							super().setUp()
							lowerCamelCase_	:     Optional[Any] 						=      PerceiverTokenizer()
							tokenizer.save_pretrained(self.tmpdirname						)



					@cached_property
					def   __a     (		self     :     Tuple						):
							return PerceiverTokenizer.from_pretrained('deepmind/language-perceiver'						)



					def   __a     (		self     :     Dict       ,      **lowerCamelCase     :     Optional[Any]						):
							return self.tokenizer_class.from_pretrained(self.tmpdirname       ,      **lowerCamelCase						)



					def   __a     (		self     :     List[Any]       ,      lowerCamelCase     :     Any       ,      lowerCamelCase     :     Any=False       ,      lowerCamelCase     :     List[Any]=20       ,      lowerCamelCase     :     Optional[int]=5						):
							# XXX The default common tokenizer tests assume that every ID is decodable on its own.
							# This assumption is invalid for Perceiver because single bytes might not be
							# valid utf-8 (byte 128 for instance).
							# Here we're overriding the smallest possible method to provide
							# a clean sequence without making the same assumption.

							lowerCamelCase_	:     Optional[Any] 						=      []
							for i in range(len(lowerCamelCase						)						):
									try:
											lowerCamelCase_	:     Optional[Any] 						=      tokenizer.decode([i]       ,      clean_up_tokenization_spaces=lowerCamelCase						)
									except UnicodeDecodeError:
											pass
									toks.append((i, tok)						)

							lowerCamelCase_	:     int 						=      list(filter(lambda lowerCamelCase						: re.match(r'^[ a-zA-Z]+$'       ,      t[1]						)       ,      lowerCamelCase						)						)
							lowerCamelCase_	:     Dict 						=      list(filter(lambda lowerCamelCase						: [t[0]] == tokenizer.encode(t[1]       ,      add_special_tokens=lowerCamelCase						)       ,      lowerCamelCase						)						)
							if max_length is not None and len(lowerCamelCase						) > max_length:
									lowerCamelCase_	:     List[Any] 						=      toks[:max_length]
							if min_length is not None and len(lowerCamelCase						) < min_length and len(lowerCamelCase						) > 0:
									while len(lowerCamelCase						) < min_length:
											lowerCamelCase_	:     List[Any] 						=      toks + toks
        # toks_str = [t[1] for t in toks]
							lowerCamelCase_	:     Optional[int] 						=      [t[0] for t in toks]

							# Ensure consistency
							lowerCamelCase_	:     Union[str, Any] 						=      tokenizer.decode(lowerCamelCase       ,      clean_up_tokenization_spaces=lowerCamelCase						)
							if " " not in output_txt and len(lowerCamelCase						) > 1:
									lowerCamelCase_	:     Optional[Any] 						=      (
									    tokenizer.decode([toks_ids[0]]       ,      clean_up_tokenization_spaces=lowerCamelCase						)
									    + ' '
									    + tokenizer.decode(toks_ids[1:]       ,      clean_up_tokenization_spaces=lowerCamelCase						)
									)
							if with_prefix_space:
									lowerCamelCase_	:     Dict 						=      ' ' + output_txt
							lowerCamelCase_	:     Any 						=      tokenizer.encode(lowerCamelCase       ,      add_special_tokens=lowerCamelCase						)
							return output_txt, output_ids



					def   __a     (		self     :     Tuple						):
							lowerCamelCase_	:     str 						=      self.perceiver_tokenizer
							lowerCamelCase_	:     List[str] 						=      'Unicode €.'
							lowerCamelCase_	:     Union[str, Any] 						=      tokenizer(lowerCamelCase						)
							lowerCamelCase_	:     str 						=      [4, 91, 1_16, 1_11, 1_05, 1_17, 1_06, 1_07, 38, 2_32, 1_36, 1_78, 52, 5]
							self.assertEqual(encoded['input_ids']       ,      lowerCamelCase						)

							# decoding
							lowerCamelCase_	:     int 						=      tokenizer.decode(lowerCamelCase						)
							self.assertEqual(lowerCamelCase       ,      '[CLS]Unicode €.[SEP]'						)

							lowerCamelCase_	:     Dict 						=      tokenizer('e è é ê ë'						)
							lowerCamelCase_	:     Optional[int] 						=      [4, 1_07, 38, 2_01, 1_74, 38, 2_01, 1_75, 38, 2_01, 1_76, 38, 2_01, 1_77, 5]
							self.assertEqual(encoded['input_ids']       ,      lowerCamelCase						)
							# decoding
							lowerCamelCase_	:     Any 						=      tokenizer.decode(lowerCamelCase						)
							self.assertEqual(lowerCamelCase       ,      '[CLS]e è é ê ë[SEP]'						)

							# encode/decode, but with `encode` instead of `__call__`
							self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'						)						)       ,      '[CLS]e è é ê ë[SEP]'						)



					def   __a     (		self     :     Optional[Any]						):
							lowerCamelCase_	:     Dict 						=      self.perceiver_tokenizer
							lowerCamelCase_	:     Union[str, Any] 						=      ['A long paragraph for summarization.', 'Another paragraph for summarization.']
							# fmt: off
							lowerCamelCase_	:     Optional[int] 						=      [4, 71, 38, 1_14, 1_17, 1_16, 1_09, 38, 1_18, 1_03, 1_20, 1_03, 1_09, 1_20, 1_03, 1_18, 1_10, 38, 1_08, 1_17, 1_20, 38, 1_21, 1_23, 1_15, 1_15, 1_03, 1_20, 1_11, 1_28, 1_03, 1_22, 1_11, 1_17, 1_16, 52, 5, 0]
							# fmt: on
							lowerCamelCase_	:     Optional[int] 						=      tokenizer(lowerCamelCase       ,      padding=lowerCamelCase       ,      return_tensors=lowerCamelCase						)
							self.assertIsInstance(lowerCamelCase       ,      lowerCamelCase						)

							if FRAMEWORK != "jax":
									lowerCamelCase_	:     Any 						=      list(batch.input_ids.numpy()[0]						)
							else:
									lowerCamelCase_	:     Tuple 						=      list(batch.input_ids.tolist()[0]						)

							self.assertListEqual(lowerCamelCase       ,      lowerCamelCase						)

							self.assertEqual((2, 38)       ,      batch.input_ids.shape						)
							self.assertEqual((2, 38)       ,      batch.attention_mask.shape						)



					def   __a     (		self     :     int						):
							lowerCamelCase_	:     int 						=      self.perceiver_tokenizer
							lowerCamelCase_	:     int 						=      ['A long paragraph for summarization.', 'Another paragraph for summarization.']
							lowerCamelCase_	:     Union[str, Any] 						=      tokenizer(lowerCamelCase       ,      padding=lowerCamelCase       ,      return_tensors=lowerCamelCase						)
							# check if input_ids are returned and no decoder_input_ids
							self.assertIn('input_ids'       ,      lowerCamelCase						)
							self.assertIn('attention_mask'       ,      lowerCamelCase						)
							self.assertNotIn('decoder_input_ids'       ,      lowerCamelCase						)
							self.assertNotIn('decoder_attention_mask'       ,      lowerCamelCase						)



					def   __a     (		self     :     List[Any]						):
							lowerCamelCase_	:     Optional[Any] 						=      self.perceiver_tokenizer
							lowerCamelCase_	:     List[str] 						=      [
							    'Summary of the text.',
							    'Another summary.',
							]
							lowerCamelCase_	:     Optional[int] 						=      tokenizer(
							    text_target=lowerCamelCase       ,      max_length=32       ,      padding='max_length'       ,      truncation=lowerCamelCase       ,      return_tensors=lowerCamelCase						)
							self.assertEqual(32       ,      targets['input_ids'].shape[1]						)



					def   __a     (		self     :     Any						):
							# safety check on max_len default value so we are sure the test works
							lowerCamelCase_	:     List[Any] 						=      self.get_tokenizers()
							for tokenizer in tokenizers:
									with self.subTest(F"{tokenizer.__class__.__name__}"						):
											self.assertNotEqual(tokenizer.model_max_length       ,      42						)

        # Now let's start the test
							lowerCamelCase_	:     List[Any] 						=      self.get_tokenizers()
							for tokenizer in tokenizers:
									with self.subTest(F"{tokenizer.__class__.__name__}"						):
											# Isolate this from the other tests because we save additional tokens/etc
											lowerCamelCase_	:     Optional[Any] 						=      tempfile.mkdtemp()

											lowerCamelCase_	:     Any 						=      ' He is very happy, UNwant\u00E9d,running'
											lowerCamelCase_	:     List[str] 						=      tokenizer.encode(lowerCamelCase       ,      add_special_tokens=lowerCamelCase						)
											tokenizer.save_pretrained(lowerCamelCase						)

											lowerCamelCase_	:     int 						=      tokenizer.__class__.from_pretrained(lowerCamelCase						)
											lowerCamelCase_	:     Any 						=      after_tokenizer.encode(lowerCamelCase       ,      add_special_tokens=lowerCamelCase						)
											self.assertListEqual(lowerCamelCase       ,      lowerCamelCase						)

											shutil.rmtree(lowerCamelCase						)

							lowerCamelCase_	:     List[Any] 						=      self.get_tokenizers(model_max_length=42						)
							for tokenizer in tokenizers:
									with self.subTest(F"{tokenizer.__class__.__name__}"						):
											# Isolate this from the other tests because we save additional tokens/etc
											lowerCamelCase_	:     int 						=      tempfile.mkdtemp()

											lowerCamelCase_	:     int 						=      ' He is very happy, UNwant\u00E9d,running'
											tokenizer.add_tokens(['bim', 'bambam']						)
											lowerCamelCase_	:     Optional[int] 						=      tokenizer.additional_special_tokens
											additional_special_tokens.append('new_additional_special_token'						)
											tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}						)
											lowerCamelCase_	:     Optional[Any] 						=      tokenizer.encode(lowerCamelCase       ,      add_special_tokens=lowerCamelCase						)
											tokenizer.save_pretrained(lowerCamelCase						)

											lowerCamelCase_	:     List[str] 						=      tokenizer.__class__.from_pretrained(lowerCamelCase						)
											lowerCamelCase_	:     int 						=      after_tokenizer.encode(lowerCamelCase       ,      add_special_tokens=lowerCamelCase						)
											self.assertListEqual(lowerCamelCase       ,      lowerCamelCase						)
											self.assertIn('new_additional_special_token'       ,      after_tokenizer.additional_special_tokens						)
											self.assertEqual(after_tokenizer.model_max_length       ,      42						)

											lowerCamelCase_	:     int 						=      tokenizer.__class__.from_pretrained(lowerCamelCase       ,      model_max_length=43						)
											self.assertEqual(tokenizer.model_max_length       ,      43						)

											shutil.rmtree(lowerCamelCase						)



					def   __a     (		self     :     Union[str, Any]						):
							lowerCamelCase_	:     List[str] 						=      []
							if self.test_slow_tokenizer:
									tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())						)

							if self.test_rust_tokenizer:
									tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())						)

							for tokenizer_class, tokenizer_utils in tokenizer_list:
									with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer_utils.save_pretrained(lowerCamelCase						)

											with open(os.path.join(lowerCamelCase       ,      'special_tokens_map.json'						)       ,      encoding='utf-8'						) as json_file:
													lowerCamelCase_	:     Dict 						=      json.load(lowerCamelCase						)

											with open(os.path.join(lowerCamelCase       ,      'tokenizer_config.json'						)       ,      encoding='utf-8'						) as json_file:
													lowerCamelCase_	:     int 						=      json.load(lowerCamelCase						)

											lowerCamelCase_	:     int 						=      [F"<extra_id_{i}>" for i in range(1_25						)]

											lowerCamelCase_	:     str 						=      added_tokens_extra_ids + [
											    'an_additional_special_token'
											]
											lowerCamelCase_	:     Any 						=      added_tokens_extra_ids + [
											    'an_additional_special_token'
											]

											with open(os.path.join(lowerCamelCase       ,      'special_tokens_map.json'						)       ,      'w'       ,      encoding='utf-8'						) as outfile:
													json.dump(lowerCamelCase       ,      lowerCamelCase						)
											with open(os.path.join(lowerCamelCase       ,      'tokenizer_config.json'						)       ,      'w'       ,      encoding='utf-8'						) as outfile:
													json.dump(lowerCamelCase       ,      lowerCamelCase						)

											# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
											# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
											# "special_tokens_map.json" files
											lowerCamelCase_	:     str 						=      tokenizer_class.from_pretrained(
											    lowerCamelCase       ,      )
											self.assertIn(
											    'an_additional_special_token'       ,      tokenizer_without_change_in_init.additional_special_tokens						)
											self.assertEqual(
											    ['an_additional_special_token']       ,      tokenizer_without_change_in_init.convert_ids_to_tokens(
											        tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']						)						)       ,      )

											# Now we test that we can change the value of additional_special_tokens in the from_pretrained
											lowerCamelCase_	:     Tuple 						=      added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'       ,      lstrip=lowerCamelCase						)]
											lowerCamelCase_	:     Optional[Any] 						=      tokenizer_class.from_pretrained(
											    lowerCamelCase       ,      additional_special_tokens=lowerCamelCase       ,      )

											self.assertIn('a_new_additional_special_token'       ,      tokenizer.additional_special_tokens						)
											self.assertEqual(
											    ['a_new_additional_special_token']       ,      tokenizer.convert_ids_to_tokens(
											        tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']						)						)       ,      )



					def   __a     (		self     :     Tuple						):
							lowerCamelCase_	:     Tuple 						=      self.perceiver_tokenizer
							self.assertEqual(tokenizer.decode([1_78]						)       ,      '�'						)



					def   __a     (		self     :     Union[str, Any]						):
							pass



					def   __a     (		self     :     List[Any]						):
							pass



					def   __a     (		self     :     Optional[int]						):
							pass



					def   __a     (		self     :     Optional[int]						):
							pass





					def   __a     (		self     :     int						):
							# The default common tokenizer tests uses invalid tokens for Perceiver that can only accept one-character
							# strings and special added tokens as tokens
							lowerCamelCase_	:     Tuple 						=      self.get_tokenizers(fast=lowerCamelCase       ,      do_lower_case=lowerCamelCase						)
							for tokenizer in tokenizers:
									with self.subTest(F"{tokenizer.__class__.__name__}"						):
											lowerCamelCase_	:     List[Any] 						=      ['[CLS]', 't', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 's', 't', '[SEP]']
											lowerCamelCase_	:     str 						=      tokenizer.convert_tokens_to_string(lowerCamelCase						)

											self.assertIsInstance(lowerCamelCase       ,      lowerCamelCase						)







def 		_SCREAMING_SNAKE_CASE     ( lowerCAmelCase__=2_81_23						):
		lowerCamelCase_	:     Any 						=      [1] * (limit + 1)

		for i in range(2							,int(limit**0.5						) + 1						):
				sum_divs[i * i] += i
				for k in range(i + 1							,limit // i + 1						):
						sum_divs[k * i] += k + i

		lowerCamelCase_	:     List[str] 						=      set()
		lowerCamelCase_	:     List[str] 						=      0

		for n in range(1							,limit + 1						):
				if sum_divs[n] > n:
						abundants.add(lowerCAmelCase__						)

				if not any((n - a in abundants) for a in abundants						):
						res += n

		return res


if __name__ == "__main__":
		print(solution())
