


'''simple docstring'''






import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 					__magic_name__							(		__SCREAMING_SNAKE_CASE					,unittest.TestCase				):
       UpperCAmelCase						      =RealmTokenizer
       UpperCAmelCase						      =RealmTokenizerFast
       UpperCAmelCase						      =True
       UpperCAmelCase						      =True
       UpperCAmelCase						      =filter_non_english






       def 			lowerCAmelCase		(			self)						->  Optional[Any]:




          '''simple docstring'''
          super().setUp()

          _UpperCAmelCase   :   int							    =[
              '[UNK]',
              '[CLS]',
              '[SEP]',
              '[PAD]',
              '[MASK]',
              'want',
              '##want',
              '##ed',
              'wa',
              'un',
              'runn',
              '##ing',
              ',',
              'low',
              'lowest',
          ]
          _UpperCAmelCase   :   Tuple							    =os.path.join(self.tmpdirname  ,							VOCAB_FILES_NAMES['vocab_file'])
          with open(self.vocab_file  ,							'w'  ,							encoding='utf-8') as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))






       def 			lowerCAmelCase		(			self  ,							snake_case)						->  Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase   :   Union[str, Any]							    ='UNwant\u00E9d,running'
          _UpperCAmelCase   :   Any							    ='unwanted, running'
          return input_text, output_text






       def 			lowerCAmelCase		(			self)						->  Optional[int]:




          '''simple docstring'''
          _UpperCAmelCase   :   Optional[Any]							    =self.tokenizer_class(self.vocab_file)

          _UpperCAmelCase   :   Tuple							    =tokenizer.tokenize('UNwant\u00E9d,running')
          self.assertListEqual(lowerCAmelCase__  ,							['un', '##want', '##ed', ',', 'runn', '##ing'])
          self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__)  ,							[9, 6, 7, 1_2, 1_0, 1_1])






       def 			lowerCAmelCase		(			self)						->  Optional[int]:




          '''simple docstring'''




          if not self.test_rust_tokenizer:
             return

          _UpperCAmelCase   :   List[Any]							    =self.get_tokenizer()
          _UpperCAmelCase   :   Dict							    =self.get_rust_tokenizer()

          _UpperCAmelCase   :   Any							    ='UNwant\u00E9d,running'

          _UpperCAmelCase   :   Any							    =tokenizer.tokenize(lowerCAmelCase__)
          _UpperCAmelCase   :   str							    =rust_tokenizer.tokenize(lowerCAmelCase__)
          self.assertListEqual(lowerCAmelCase__  ,							lowerCAmelCase__)

          _UpperCAmelCase   :   Dict							    =tokenizer.encode(lowerCAmelCase__  ,							add_special_tokens=lowerCAmelCase__)
          _UpperCAmelCase   :   Union[str, Any]							    =rust_tokenizer.encode(lowerCAmelCase__  ,							add_special_tokens=lowerCAmelCase__)
          self.assertListEqual(lowerCAmelCase__  ,							lowerCAmelCase__)

          _UpperCAmelCase   :   Optional[int]							    =self.get_rust_tokenizer()
          _UpperCAmelCase   :   List[Any]							    =tokenizer.encode(lowerCAmelCase__)
          _UpperCAmelCase   :   str							    =rust_tokenizer.encode(lowerCAmelCase__)
          self.assertListEqual(lowerCAmelCase__  ,							lowerCAmelCase__)

          # With lower casing
          _UpperCAmelCase   :   Optional[int]							    =self.get_tokenizer(do_lower_case=lowerCAmelCase__)
          _UpperCAmelCase   :   List[str]							    =self.get_rust_tokenizer(do_lower_case=lowerCAmelCase__)

          _UpperCAmelCase   :   List[Any]							    ='UNwant\u00E9d,running'

          _UpperCAmelCase   :   str							    =tokenizer.tokenize(lowerCAmelCase__)
          _UpperCAmelCase   :   Optional[Any]							    =rust_tokenizer.tokenize(lowerCAmelCase__)
          self.assertListEqual(lowerCAmelCase__  ,							lowerCAmelCase__)

          _UpperCAmelCase   :   Optional[int]							    =tokenizer.encode(lowerCAmelCase__  ,							add_special_tokens=lowerCAmelCase__)
          _UpperCAmelCase   :   Union[str, Any]							    =rust_tokenizer.encode(lowerCAmelCase__  ,							add_special_tokens=lowerCAmelCase__)
          self.assertListEqual(lowerCAmelCase__  ,							lowerCAmelCase__)

          _UpperCAmelCase   :   Any							    =self.get_rust_tokenizer()
          _UpperCAmelCase   :   List[str]							    =tokenizer.encode(lowerCAmelCase__)
          _UpperCAmelCase   :   Any							    =rust_tokenizer.encode(lowerCAmelCase__)
          self.assertListEqual(lowerCAmelCase__  ,							lowerCAmelCase__)






       def 			lowerCAmelCase		(			self)						->  Tuple:




          '''simple docstring'''
          _UpperCAmelCase   :   Dict							    =BasicTokenizer()

          self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz')  ,							['ah', '\u535A', '\u63A8', 'zz'])






       def 			lowerCAmelCase		(			self)						->  Dict:




          '''simple docstring'''
          _UpperCAmelCase   :   Union[str, Any]							    =BasicTokenizer(do_lower_case=lowerCAmelCase__)

          self.assertListEqual(
              tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')  ,							['hello', '!', 'how', 'are', 'you', '?'])
          self.assertListEqual(tokenizer.tokenize('H\u00E9llo')  ,							['hello'])






       def 			lowerCAmelCase		(			self)						->  str:




          '''simple docstring'''
          _UpperCAmelCase   :   Optional[Any]							    =BasicTokenizer(do_lower_case=lowerCAmelCase__  ,							strip_accents=lowerCAmelCase__)

          self.assertListEqual(
              tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')  ,							['hällo', '!', 'how', 'are', 'you', '?'])
          self.assertListEqual(tokenizer.tokenize('H\u00E9llo')  ,							['h\u00E9llo'])






       def 			lowerCAmelCase		(			self)						->  str:




          '''simple docstring'''
          _UpperCAmelCase   :   List[Any]							    =BasicTokenizer(do_lower_case=lowerCAmelCase__  ,							strip_accents=lowerCAmelCase__)

          self.assertListEqual(
              tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')  ,							['hallo', '!', 'how', 'are', 'you', '?'])
          self.assertListEqual(tokenizer.tokenize('H\u00E9llo')  ,							['hello'])






       def 			lowerCAmelCase		(			self)						->  List[str]:




          '''simple docstring'''
          _UpperCAmelCase   :   Tuple							    =BasicTokenizer(do_lower_case=lowerCAmelCase__)

          self.assertListEqual(
              tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')  ,							['hallo', '!', 'how', 'are', 'you', '?'])
          self.assertListEqual(tokenizer.tokenize('H\u00E9llo')  ,							['hello'])






       def 			lowerCAmelCase		(			self)						->  Tuple:




          '''simple docstring'''
          _UpperCAmelCase   :   Any							    =BasicTokenizer(do_lower_case=lowerCAmelCase__)

          self.assertListEqual(
              tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')  ,							['HeLLo', '!', 'how', 'Are', 'yoU', '?'])






       def 			lowerCAmelCase		(			self)						->  Optional[int]:




          '''simple docstring'''
          _UpperCAmelCase   :   Optional[int]							    =BasicTokenizer(do_lower_case=lowerCAmelCase__  ,							strip_accents=lowerCAmelCase__)

          self.assertListEqual(
              tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')  ,							['HäLLo', '!', 'how', 'Are', 'yoU', '?'])






       def 			lowerCAmelCase		(			self)						->  Any:




          '''simple docstring'''
          _UpperCAmelCase   :   List[Any]							    =BasicTokenizer(do_lower_case=lowerCAmelCase__  ,							strip_accents=lowerCAmelCase__)

          self.assertListEqual(
              tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')  ,							['HaLLo', '!', 'how', 'Are', 'yoU', '?'])






       def 			lowerCAmelCase		(			self)						->  List[Any]:




          '''simple docstring'''
          _UpperCAmelCase   :   Any							    =BasicTokenizer(do_lower_case=lowerCAmelCase__  ,							never_split=['[UNK]'])

          self.assertListEqual(
              tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]')  ,							['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'])






       def 			lowerCAmelCase		(			self)						->  List[str]:




          '''simple docstring'''
          _UpperCAmelCase   :   Optional[Any]							    =['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

          _UpperCAmelCase   :   Union[str, Any]							    ={}
          for i, token in enumerate(lowerCAmelCase__):
             _UpperCAmelCase   :   Any							    =i
          _UpperCAmelCase   :   List[Any]							    =WordpieceTokenizer(vocab=lowerCAmelCase__  ,							unk_token='[UNK]')

          self.assertListEqual(tokenizer.tokenize('')  ,							[])

          self.assertListEqual(tokenizer.tokenize('unwanted running')  ,							['un', '##want', '##ed', 'runn', '##ing'])

          self.assertListEqual(tokenizer.tokenize('unwantedX running')  ,							['[UNK]', 'runn', '##ing'])






       def 			lowerCAmelCase		(			self)						->  Any:




          '''simple docstring'''
          self.assertTrue(_is_whitespace(' '))
          self.assertTrue(_is_whitespace('\t'))
          self.assertTrue(_is_whitespace('\r'))
          self.assertTrue(_is_whitespace('\n'))
          self.assertTrue(_is_whitespace('\u00A0'))

          self.assertFalse(_is_whitespace('A'))
          self.assertFalse(_is_whitespace('-'))






       def 			lowerCAmelCase		(			self)						->  List[Any]:




          '''simple docstring'''
          self.assertTrue(_is_control('\u0005'))

          self.assertFalse(_is_control('A'))
          self.assertFalse(_is_control(' '))
          self.assertFalse(_is_control('\t'))
          self.assertFalse(_is_control('\r'))






       def 			lowerCAmelCase		(			self)						->  int:




          '''simple docstring'''
          self.assertTrue(_is_punctuation('-'))
          self.assertTrue(_is_punctuation('$'))
          self.assertTrue(_is_punctuation('`'))
          self.assertTrue(_is_punctuation('.'))

          self.assertFalse(_is_punctuation('A'))
          self.assertFalse(_is_punctuation(' '))






       def 			lowerCAmelCase		(			self)						->  Any:




          '''simple docstring'''
          _UpperCAmelCase   :   List[Any]							    =self.get_tokenizer()

          # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
          self.assertListEqual([tokenizer.tokenize(lowerCAmelCase__) for t in ['Test', '\xad', 'test']]  ,							[['[UNK]'], [], ['[UNK]']])

          if self.test_rust_tokenizer:
             _UpperCAmelCase   :   str							    =self.get_rust_tokenizer()
             self.assertListEqual(
                 [rust_tokenizer.tokenize(lowerCAmelCase__) for t in ['Test', '\xad', 'test']]  ,							[['[UNK]'], [], ['[UNK]']])






       @slow
       def 			lowerCAmelCase		(			self)						->  Union[str, Any]:




          '''simple docstring'''
          _UpperCAmelCase   :   str							    =self.tokenizer_class.from_pretrained('bert-base-uncased')

          _UpperCAmelCase   :   Optional[int]							    =tokenizer.encode('sequence builders'  ,							add_special_tokens=lowerCAmelCase__)
          _UpperCAmelCase   :   Optional[Any]							    =tokenizer.encode('multi-sequence build'  ,							add_special_tokens=lowerCAmelCase__)

          _UpperCAmelCase   :   Optional[int]							    =tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__)
          _UpperCAmelCase   :   Any							    =tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__  ,							lowerCAmelCase__)

          assert encoded_sentence == [1_0_1] + text + [1_0_2]
          assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]






       def 			lowerCAmelCase		(			self)						->  Optional[int]:




          '''simple docstring'''




          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                _UpperCAmelCase   :   Optional[int]							    =self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__  ,							**lowerCAmelCase__)

                _UpperCAmelCase   :   Union[str, Any]							    =f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
                _UpperCAmelCase   :   Optional[Any]							    =tokenizer_r.encode_plus(
                    lowerCAmelCase__  ,							return_attention_mask=lowerCAmelCase__  ,							return_token_type_ids=lowerCAmelCase__  ,							return_offsets_mapping=lowerCAmelCase__  ,							add_special_tokens=lowerCAmelCase__  ,							)

                _UpperCAmelCase   :   List[Any]							    =tokenizer_r.do_lower_case if hasattr(lowerCAmelCase__  ,							'do_lower_case') else False
                _UpperCAmelCase   :   str							    =(
                    [
                        ((0, 0), tokenizer_r.cls_token),
                        ((0, 1), 'A'),
                        ((1, 2), ','),
                        ((3, 5), 'na'),
                        ((5, 6), '##ï'),
                        ((6, 8), '##ve'),
                        ((9, 1_5), tokenizer_r.mask_token),
                        ((1_6, 2_1), 'Allen'),
                        ((2_1, 2_3), '##NL'),
                        ((2_3, 2_4), '##P'),
                        ((2_5, 3_3), 'sentence'),
                        ((3_3, 3_4), '.'),
                        ((0, 0), tokenizer_r.sep_token),
                    ]
                    if not do_lower_case
                    else [
                        ((0, 0), tokenizer_r.cls_token),
                        ((0, 1), 'a'),
                        ((1, 2), ','),
                        ((3, 8), 'naive'),
                        ((9, 1_5), tokenizer_r.mask_token),
                        ((1_6, 2_1), 'allen'),
                        ((2_1, 2_3), '##nl'),
                        ((2_3, 2_4), '##p'),
                        ((2_5, 3_3), 'sentence'),
                        ((3_3, 3_4), '.'),
                        ((0, 0), tokenizer_r.sep_token),
                    ]
                )

                self.assertEqual(
                    [e[1] for e in expected_results]  ,							tokenizer_r.convert_ids_to_tokens(tokens['input_ids']))
                self.assertEqual([e[0] for e in expected_results]  ,							tokens['offset_mapping'])






       @slow
       def 			lowerCAmelCase		(			self)						->  Tuple:




          '''simple docstring'''




          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                _UpperCAmelCase   :   str							    =self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__  ,							**lowerCAmelCase__)
                _UpperCAmelCase   :   List[Any]							    =self.tokenizer_class.from_pretrained(lowerCAmelCase__  ,							**lowerCAmelCase__)
                _UpperCAmelCase   :   Optional[int]							    =[['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

                _UpperCAmelCase   :   List[Any]							    =tokenizer_r.batch_encode_candidates(lowerCAmelCase__  ,							max_length=1_0  ,							return_tensors='np')
                _UpperCAmelCase   :   Any							    =tokenizer_p.batch_encode_candidates(lowerCAmelCase__  ,							max_length=1_0  ,							return_tensors='np')

                _UpperCAmelCase   :   Optional[int]							    =(2, 2, 1_0)

                self.assertEqual(encoded_sentence_r['input_ids'].shape  ,							lowerCAmelCase__)
                self.assertEqual(encoded_sentence_r['attention_mask'].shape  ,							lowerCAmelCase__)
                self.assertEqual(encoded_sentence_r['token_type_ids'].shape  ,							lowerCAmelCase__)

                self.assertEqual(encoded_sentence_p['input_ids'].shape  ,							lowerCAmelCase__)
                self.assertEqual(encoded_sentence_p['attention_mask'].shape  ,							lowerCAmelCase__)
                self.assertEqual(encoded_sentence_p['token_type_ids'].shape  ,							lowerCAmelCase__)





def 		a__       (							A__ = 2_0							):

			try:
						SCREAMING_SNAKE_CASE_						:			Optional[Any]       						= int(A__							)
			except (TypeError, ValueError):
						raise TypeError('Parameter n must be int or castable to int.'							)
			if n <= 0:
						raise ValueError('Parameter n must be greater than or equal to one.'							)
			SCREAMING_SNAKE_CASE_						:			int       						= 0
			while 1:
						i += n * (n - 1)
						SCREAMING_SNAKE_CASE_						:			List[str]       						= 0
						for j in range(2,	A__							):
									if i % j != 0:
												SCREAMING_SNAKE_CASE_						:			str       						= 1
												break
						if nfound == 0:
									if i == 0:
												SCREAMING_SNAKE_CASE_						:			Tuple       						= 1
									return i
			return None


if __name__ == "__main__":
				print(F"""{solution() = }""")





