






'''simple docstring'''





import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 							A_    (						__A ,   unittest.TestCase			):






 '''simple docstring'''




 _SCREAMING_SNAKE_CASE							:			Tuple =					BertTokenizer
 _SCREAMING_SNAKE_CASE							:			Union[str, Any] =					BertTokenizerFast
 _SCREAMING_SNAKE_CASE							:			Any =					True
 _SCREAMING_SNAKE_CASE							:			List[str] =					True
 _SCREAMING_SNAKE_CASE							:			Tuple =					filter_non_english


 def  snake_case__				(	self) ->		Optional[int]:

     """simple docstring"""



     super().setUp()

     _UpperCAmelCase				:      int			  =				[
         '[UNK]',
         '[CLS]',
         '[SEP]',
         '[PAD]',
         '[MASK]',
         'want',
         '##want',
         '##ed',
         'wa',
         'un',
         'runn',
         '##ing',
         ',',
         'low',
         'lowest',
     ]
     _UpperCAmelCase				:      Optional[int]			  =				os.path.join(self.tmpdirname							,  VOCAB_FILES_NAMES['''vocab_file'''])
     with open(self.vocab_file							,  '''w'''							,  encoding='''utf-8''') as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]))


 def  snake_case__				(	self							,  _A) ->		Optional[int]:

     """simple docstring"""



     _UpperCAmelCase				:      int			  =				'UNwant\u00E9d,running'
     _UpperCAmelCase				:      Optional[int]			  =				'unwanted, running'
     return input_text, output_text


 def  snake_case__				(	self) ->		Dict:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[Any]			  =				self.tokenizer_class(self.vocab_file)

     _UpperCAmelCase				:      int			  =				tokenizer.tokenize('''UNwant\u00E9d,running''')
     self.assertListEqual(_A							,  ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''])
     self.assertListEqual(tokenizer.convert_tokens_to_ids(_A)							,  [9, 6, 7, 12, 10, 11])


 def  snake_case__				(	self) ->		Tuple:

     """simple docstring"""



     if not self.test_rust_tokenizer:
         return

     _UpperCAmelCase				:      List[str]			  =				self.get_tokenizer()
     _UpperCAmelCase				:      Optional[Any]			  =				self.get_rust_tokenizer()

     _UpperCAmelCase				:      Optional[Any]			  =				'UNwant\u00E9d,running'

     _UpperCAmelCase				:      str			  =				tokenizer.tokenize(_A)
     _UpperCAmelCase				:      Dict			  =				rust_tokenizer.tokenize(_A)
     self.assertListEqual(_A							,  _A)

     _UpperCAmelCase				:      int			  =				tokenizer.encode(_A							,  add_special_tokens=_A)
     _UpperCAmelCase				:      Dict			  =				rust_tokenizer.encode(_A							,  add_special_tokens=_A)
     self.assertListEqual(_A							,  _A)

     _UpperCAmelCase				:      Tuple			  =				self.get_rust_tokenizer()
     _UpperCAmelCase				:      Optional[int]			  =				tokenizer.encode(_A)
     _UpperCAmelCase				:      Any			  =				rust_tokenizer.encode(_A)
     self.assertListEqual(_A							,  _A)

     # With lower casing
     _UpperCAmelCase				:      Union[str, Any]			  =				self.get_tokenizer(do_lower_case=_A)
     _UpperCAmelCase				:      Tuple			  =				self.get_rust_tokenizer(do_lower_case=_A)

     _UpperCAmelCase				:      List[Any]			  =				'UNwant\u00E9d,running'

     _UpperCAmelCase				:      Optional[int]			  =				tokenizer.tokenize(_A)
     _UpperCAmelCase				:      Union[str, Any]			  =				rust_tokenizer.tokenize(_A)
     self.assertListEqual(_A							,  _A)

     _UpperCAmelCase				:      Optional[int]			  =				tokenizer.encode(_A							,  add_special_tokens=_A)
     _UpperCAmelCase				:      List[Any]			  =				rust_tokenizer.encode(_A							,  add_special_tokens=_A)
     self.assertListEqual(_A							,  _A)

     _UpperCAmelCase				:      str			  =				self.get_rust_tokenizer()
     _UpperCAmelCase				:      Tuple			  =				tokenizer.encode(_A)
     _UpperCAmelCase				:      Union[str, Any]			  =				rust_tokenizer.encode(_A)
     self.assertListEqual(_A							,  _A)


 def  snake_case__				(	self) ->		str:

     """simple docstring"""



     _UpperCAmelCase				:      Tuple			  =				BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''')							,  ['''ah''', '''\u535A''', '''\u63A8''', '''zz'''])


 def  snake_case__				(	self) ->		Optional[int]:

     """simple docstring"""



     _UpperCAmelCase				:      Any			  =				BasicTokenizer(do_lower_case=_A)

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''')							,  ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''])
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')							,  ['''hello'''])


 def  snake_case__				(	self) ->		Union[str, Any]:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[Any]			  =				BasicTokenizer(do_lower_case=_A							,  strip_accents=_A)

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')							,  ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')							,  ['''h\u00E9llo'''])


 def  snake_case__				(	self) ->		str:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[Any]			  =				BasicTokenizer(do_lower_case=_A							,  strip_accents=_A)

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')							,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')							,  ['''hello'''])


 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[int]			  =				BasicTokenizer(do_lower_case=_A)

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')							,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
     self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')							,  ['''hello'''])


 def  snake_case__				(	self) ->		Tuple:

     """simple docstring"""



     _UpperCAmelCase				:      List[Any]			  =				BasicTokenizer(do_lower_case=_A)

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''')							,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])


 def  snake_case__				(	self) ->		Any:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[Any]			  =				BasicTokenizer(do_lower_case=_A							,  strip_accents=_A)

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')							,  ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])


 def  snake_case__				(	self) ->		Dict:

     """simple docstring"""



     _UpperCAmelCase				:      Dict			  =				BasicTokenizer(do_lower_case=_A							,  strip_accents=_A)

     self.assertListEqual(
         tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')							,  ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])


 def  snake_case__				(	self) ->		Tuple:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[Any]			  =				BasicTokenizer(do_lower_case=_A							,  never_split=['''[UNK]'''])

     self.assertListEqual(
         tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''')							,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''])


 def  snake_case__				(	self) ->		List[str]:

     """simple docstring"""



     _UpperCAmelCase				:      Any			  =				BasicTokenizer()
     _UpperCAmelCase				:      int			  =				'a\n\'ll !!to?\'d of, can\'t.'
     _UpperCAmelCase				:      Any			  =				['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
     self.assertListEqual(tokenizer.tokenize(_A)							,  _A)


 def  snake_case__				(	self) ->		Optional[int]:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[Any]			  =				['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

     _UpperCAmelCase				:      Optional[Any]			  =				{}
     for i, token in enumerate(_A):
         _UpperCAmelCase				:      Dict			  =				i
     _UpperCAmelCase				:      Any			  =				WordpieceTokenizer(vocab=_A							,  unk_token='''[UNK]''')

     self.assertListEqual(tokenizer.tokenize('''''')							,  [])

     self.assertListEqual(tokenizer.tokenize('''unwanted running''')							,  ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''])

     self.assertListEqual(tokenizer.tokenize('''unwantedX running''')							,  ['''[UNK]''', '''runn''', '''##ing'''])


 def  snake_case__				(	self) ->		int:

     """simple docstring"""



     self.assertTrue(_is_whitespace(''' '''))
     self.assertTrue(_is_whitespace('''\t'''))
     self.assertTrue(_is_whitespace('''\r'''))
     self.assertTrue(_is_whitespace('''\n'''))
     self.assertTrue(_is_whitespace('''\u00A0'''))

     self.assertFalse(_is_whitespace('''A'''))
     self.assertFalse(_is_whitespace('''-'''))


 def  snake_case__				(	self) ->		int:

     """simple docstring"""



     self.assertTrue(_is_control('''\u0005'''))

     self.assertFalse(_is_control('''A'''))
     self.assertFalse(_is_control(''' '''))
     self.assertFalse(_is_control('''\t'''))
     self.assertFalse(_is_control('''\r'''))


 def  snake_case__				(	self) ->		Optional[Any]:

     """simple docstring"""



     self.assertTrue(_is_punctuation('''-'''))
     self.assertTrue(_is_punctuation('''$'''))
     self.assertTrue(_is_punctuation('''`'''))
     self.assertTrue(_is_punctuation('''.'''))

     self.assertFalse(_is_punctuation('''A'''))
     self.assertFalse(_is_punctuation(''' '''))


 def  snake_case__				(	self) ->		List[Any]:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[int]			  =				self.get_tokenizer()
     _UpperCAmelCase				:      Any			  =				self.get_rust_tokenizer()

     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(_A) for t in ['''Test''', '''\xad''', '''test''']]							,  [['''[UNK]'''], [], ['''[UNK]''']])

     self.assertListEqual(
         [rust_tokenizer.tokenize(_A) for t in ['''Test''', '''\xad''', '''test''']]							,  [['''[UNK]'''], [], ['''[UNK]''']])


 @slow
 def  snake_case__				(	self) ->		Union[str, Any]:

     """simple docstring"""



     _UpperCAmelCase				:      List[str]			  =				self.tokenizer_class.from_pretrained('''bert-base-uncased''')

     _UpperCAmelCase				:      Tuple			  =				tokenizer.encode('''sequence builders'''							,  add_special_tokens=_A)
     _UpperCAmelCase				:      int			  =				tokenizer.encode('''multi-sequence build'''							,  add_special_tokens=_A)

     _UpperCAmelCase				:      Tuple			  =				tokenizer.build_inputs_with_special_tokens(_A)
     _UpperCAmelCase				:      Union[str, Any]			  =				tokenizer.build_inputs_with_special_tokens(_A							,  _A)

     assert encoded_sentence == [101] + text + [102]
     assert encoded_pair == [101] + text + [102] + text_a + [102]


 def  snake_case__				(	self) ->		List[str]:

     """simple docstring"""



     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''):
             _UpperCAmelCase				:      int			  =				self.rust_tokenizer_class.from_pretrained(_A							,  **_A)

             _UpperCAmelCase				:      Optional[int]			  =				f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
             _UpperCAmelCase				:      Union[str, Any]			  =				tokenizer_r.encode_plus(
                 _A							,  return_attention_mask=_A							,  return_token_type_ids=_A							,  return_offsets_mapping=_A							,  add_special_tokens=_A							,  )

             _UpperCAmelCase				:      List[Any]			  =				tokenizer_r.do_lower_case if hasattr(_A							,  '''do_lower_case''') else False
             _UpperCAmelCase				:      List[Any]			  =				(
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), 'A'),
                     ((1, 2), ','),
                     ((3, 5), 'na'),
                     ((5, 6), '##ï'),
                     ((6, 8), '##ve'),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), 'Allen'),
                     ((21, 23), '##NL'),
                     ((23, 24), '##P'),
                     ((25, 33), 'sentence'),
                     ((33, 34), '.'),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), 'a'),
                     ((1, 2), ','),
                     ((3, 8), 'naive'),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), 'allen'),
                     ((21, 23), '##nl'),
                     ((23, 24), '##p'),
                     ((25, 33), 'sentence'),
                     ((33, 34), '.'),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results]							,  tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']))
             self.assertEqual([e[0] for e in expected_results]							,  tokens['''offset_mapping'''])


 def  snake_case__				(	self) ->		str:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[Any]			  =				['的', '人', '有']
     _UpperCAmelCase				:      int			  =				''.join(_A)
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''):
             _UpperCAmelCase				:      int			  =				True
             _UpperCAmelCase				:      List[str]			  =				self.tokenizer_class.from_pretrained(_A							,  **_A)
             _UpperCAmelCase				:      Tuple			  =				self.rust_tokenizer_class.from_pretrained(_A							,  **_A)

             _UpperCAmelCase				:      Any			  =				tokenizer_p.encode(_A							,  add_special_tokens=_A)
             _UpperCAmelCase				:      int			  =				tokenizer_r.encode(_A							,  add_special_tokens=_A)

             _UpperCAmelCase				:      Optional[Any]			  =				tokenizer_r.convert_ids_to_tokens(_A)
             _UpperCAmelCase				:      Any			  =				tokenizer_p.convert_ids_to_tokens(_A)

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(_A							,  _A)
             self.assertListEqual(_A							,  _A)

             _UpperCAmelCase				:      str			  =				False
             _UpperCAmelCase				:      Union[str, Any]			  =				self.rust_tokenizer_class.from_pretrained(_A							,  **_A)
             _UpperCAmelCase				:      List[Any]			  =				self.tokenizer_class.from_pretrained(_A							,  **_A)

             _UpperCAmelCase				:      Union[str, Any]			  =				tokenizer_r.encode(_A							,  add_special_tokens=_A)
             _UpperCAmelCase				:      Tuple			  =				tokenizer_p.encode(_A							,  add_special_tokens=_A)

             _UpperCAmelCase				:      Any			  =				tokenizer_r.convert_ids_to_tokens(_A)
             _UpperCAmelCase				:      Optional[int]			  =				tokenizer_p.convert_ids_to_tokens(_A)

             # it is expected that only the first Chinese character is not preceded by "##".
             _UpperCAmelCase				:      Dict			  =				[
                 f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_A)
             ]
             self.assertListEqual(_A							,  _A)
             self.assertListEqual(_A							,  _A)












def      _lowerCamelCase	(						__A    :							int )      ->					bool:
    if not isinstance(__A		,		__A ):
        _UpperCAmelCase				:      Tuple			  =				f'''Input value of [number={number}] must be an integer'''
        raise TypeError(__A )
    if number < 0:
        return False
    _UpperCAmelCase				:      List[str]			  =				number * number
    while number > 0:
        if number % 10 != number_square % 10:
            return False
        number //= 10
        number_square //= 10
    return True


if __name__ == "__main__":
  import doctest

  doctest.testmod()




