





import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class  __A(						__UpperCAmelCase						,			unittest.TestCase ):
     __A	  =       BertTokenizer
     __A	  =       BertTokenizerFast
     __A	  =       True
     __A	  =       True
     __A	  =       filter_non_english

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            super().setUp()

            _UpperCamelCase              =						[
                '''[UNK]''',
                '''[CLS]''',
                '''[SEP]''',
                '''[PAD]''',
                '''[MASK]''',
                '''want''',
                '''##want''',
                '''##ed''',
                '''wa''',
                '''un''',
                '''runn''',
                '''##ing''',
                ''',''',
                '''low''',
                '''lowest''',
            ]
            _UpperCamelCase              =						os.path.join(self.tmpdirname,       VOCAB_FILES_NAMES['''vocab_file''']      )
            with open(self.vocab_file,       '''w''',       encoding='''utf-8'''      ) as vocab_writer:
                   vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]      )      )

     def 	_UpperCamelCase     (      self,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						'''UNwant\u00E9d,running'''
            _UpperCamelCase              =						'''unwanted, running'''
            return input_text, output_text

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						self.tokenizer_class(self.vocab_file      )

            _UpperCamelCase              =						tokenizer.tokenize('''UNwant\u00E9d,running'''      )
            self.assertListEqual(A,       ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']      )
            self.assertListEqual(tokenizer.convert_tokens_to_ids(A      ),       [9, 6, 7, 12, 10, 11]      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            if not self.test_rust_tokenizer:
                   return

            _UpperCamelCase              =						self.get_tokenizer()
            _UpperCamelCase              =						self.get_rust_tokenizer()

            _UpperCamelCase              =						'''UNwant\u00E9d,running'''

            _UpperCamelCase              =						tokenizer.tokenize(A      )
            _UpperCamelCase              =						rust_tokenizer.tokenize(A      )
            self.assertListEqual(A,       A      )

            _UpperCamelCase              =						tokenizer.encode(A,       add_special_tokens=A      )
            _UpperCamelCase              =						rust_tokenizer.encode(A,       add_special_tokens=A      )
            self.assertListEqual(A,       A      )

            _UpperCamelCase              =						self.get_rust_tokenizer()
            _UpperCamelCase              =						tokenizer.encode(A      )
            _UpperCamelCase              =						rust_tokenizer.encode(A      )
            self.assertListEqual(A,       A      )

            # With lower casing
            _UpperCamelCase              =						self.get_tokenizer(do_lower_case=A      )
            _UpperCamelCase              =						self.get_rust_tokenizer(do_lower_case=A      )

            _UpperCamelCase              =						'''UNwant\u00E9d,running'''

            _UpperCamelCase              =						tokenizer.tokenize(A      )
            _UpperCamelCase              =						rust_tokenizer.tokenize(A      )
            self.assertListEqual(A,       A      )

            _UpperCamelCase              =						tokenizer.encode(A,       add_special_tokens=A      )
            _UpperCamelCase              =						rust_tokenizer.encode(A,       add_special_tokens=A      )
            self.assertListEqual(A,       A      )

            _UpperCamelCase              =						self.get_rust_tokenizer()
            _UpperCamelCase              =						tokenizer.encode(A      )
            _UpperCamelCase              =						rust_tokenizer.encode(A      )
            self.assertListEqual(A,       A      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						BasicTokenizer()

            self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''      ),       ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						BasicTokenizer(do_lower_case=A      )

            self.assertListEqual(
                tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''      ),       ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
            self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      ),       ['''hello''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						BasicTokenizer(do_lower_case=A,       strip_accents=A      )

            self.assertListEqual(
                tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      ),       ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
            self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      ),       ['''h\u00E9llo''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						BasicTokenizer(do_lower_case=A,       strip_accents=A      )

            self.assertListEqual(
                tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      ),       ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
            self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      ),       ['''hello''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						BasicTokenizer(do_lower_case=A      )

            self.assertListEqual(
                tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      ),       ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
            self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      ),       ['''hello''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						BasicTokenizer(do_lower_case=A      )

            self.assertListEqual(
                tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''      ),       ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						BasicTokenizer(do_lower_case=A,       strip_accents=A      )

            self.assertListEqual(
                tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      ),       ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						BasicTokenizer(do_lower_case=A,       strip_accents=A      )

            self.assertListEqual(
                tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      ),       ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						BasicTokenizer(do_lower_case=A,       never_split=['''[UNK]''']      )

            self.assertListEqual(
                tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''      ),       ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						BasicTokenizer()
            _UpperCamelCase              =						'''a\n\'ll !!to?\'d of, can\'t.'''
            _UpperCamelCase              =						['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
            self.assertListEqual(tokenizer.tokenize(A      ),       A      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

            _UpperCamelCase              =						{}
            for i, token in enumerate(A      ):
                   _UpperCamelCase              =						i
            _UpperCamelCase              =						WordpieceTokenizer(vocab=A,       unk_token='''[UNK]'''      )

            self.assertListEqual(tokenizer.tokenize(''''''      ),       []      )

            self.assertListEqual(tokenizer.tokenize('''unwanted running'''      ),       ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']      )

            self.assertListEqual(tokenizer.tokenize('''unwantedX running'''      ),       ['''[UNK]''', '''runn''', '''##ing''']      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            self.assertTrue(_is_whitespace(''' '''      )      )
            self.assertTrue(_is_whitespace('''\t'''      )      )
            self.assertTrue(_is_whitespace('''\r'''      )      )
            self.assertTrue(_is_whitespace('''\n'''      )      )
            self.assertTrue(_is_whitespace('''\u00A0'''      )      )

            self.assertFalse(_is_whitespace('''A'''      )      )
            self.assertFalse(_is_whitespace('''-'''      )      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            self.assertTrue(_is_control('''\u0005'''      )      )

            self.assertFalse(_is_control('''A'''      )      )
            self.assertFalse(_is_control(''' '''      )      )
            self.assertFalse(_is_control('''\t'''      )      )
            self.assertFalse(_is_control('''\r'''      )      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            self.assertTrue(_is_punctuation('''-'''      )      )
            self.assertTrue(_is_punctuation('''$'''      )      )
            self.assertTrue(_is_punctuation('''`'''      )      )
            self.assertTrue(_is_punctuation('''.'''      )      )

            self.assertFalse(_is_punctuation('''A'''      )      )
            self.assertFalse(_is_punctuation(''' '''      )      )

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						self.get_tokenizer()
            _UpperCamelCase              =						self.get_rust_tokenizer()

            # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
            self.assertListEqual([tokenizer.tokenize(A      ) for t in ['''Test''', '''\xad''', '''test''']],       [['''[UNK]'''], [], ['''[UNK]''']]      )

            self.assertListEqual(
                [rust_tokenizer.tokenize(A      ) for t in ['''Test''', '''\xad''', '''test''']],       [['''[UNK]'''], [], ['''[UNK]''']]      )

     @slow
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						self.tokenizer_class.from_pretrained('''bert-base-uncased'''      )

            _UpperCamelCase              =						tokenizer.encode('''sequence builders''',       add_special_tokens=A      )
            _UpperCamelCase              =						tokenizer.encode('''multi-sequence build''',       add_special_tokens=A      )

            _UpperCamelCase              =						tokenizer.build_inputs_with_special_tokens(A      )
            _UpperCamelCase              =						tokenizer.build_inputs_with_special_tokens(A,       A      )

            assert encoded_sentence == [101] + text + [102]
            assert encoded_pair == [101] + text + [102] + text_a + [102]

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''      ):
                          _UpperCamelCase              =						self.rust_tokenizer_class.from_pretrained(A,       **A      )

                          _UpperCamelCase              =						F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                          _UpperCamelCase              =						tokenizer_r.encode_plus(
                              A,       return_attention_mask=A,       return_token_type_ids=A,       return_offsets_mapping=A,       add_special_tokens=A,       )

                          _UpperCamelCase              =						tokenizer_r.do_lower_case if hasattr(A,       '''do_lower_case'''      ) else False
                          _UpperCamelCase              =						(
                              [
                                  ((0, 0), tokenizer_r.cls_token),
                                  ((0, 1), '''A'''),
                                  ((1, 2), ''','''),
                                  ((3, 5), '''na'''),
                                  ((5, 6), '''##ï'''),
                                  ((6, 8), '''##ve'''),
                                  ((9, 15), tokenizer_r.mask_token),
                                  ((16, 21), '''Allen'''),
                                  ((21, 23), '''##NL'''),
                                  ((23, 24), '''##P'''),
                                  ((25, 33), '''sentence'''),
                                  ((33, 34), '''.'''),
                                  ((0, 0), tokenizer_r.sep_token),
                              ]
                              if not do_lower_case
                              else [
                                  ((0, 0), tokenizer_r.cls_token),
                                  ((0, 1), '''a'''),
                                  ((1, 2), ''','''),
                                  ((3, 8), '''naive'''),
                                  ((9, 15), tokenizer_r.mask_token),
                                  ((16, 21), '''allen'''),
                                  ((21, 23), '''##nl'''),
                                  ((23, 24), '''##p'''),
                                  ((25, 33), '''sentence'''),
                                  ((33, 34), '''.'''),
                                  ((0, 0), tokenizer_r.sep_token),
                              ]
                          )

                          self.assertEqual(
                              [e[1] for e in expected_results],       tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']      )      )
                          self.assertEqual([e[0] for e in expected_results],       tokens['''offset_mapping''']      )



     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						['''的''', '''人''', '''有''']
            _UpperCamelCase              =						''''''.join(A      )
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''      ):
                          _UpperCamelCase              =						True
                          _UpperCamelCase              =						self.tokenizer_class.from_pretrained(A,       **A      )
                          _UpperCamelCase              =						self.rust_tokenizer_class.from_pretrained(A,       **A      )

                          _UpperCamelCase              =						tokenizer_p.encode(A,       add_special_tokens=A      )
                          _UpperCamelCase              =						tokenizer_r.encode(A,       add_special_tokens=A      )

                          _UpperCamelCase              =						tokenizer_r.convert_ids_to_tokens(A      )
                          _UpperCamelCase              =						tokenizer_p.convert_ids_to_tokens(A      )

                          # it is expected that each Chinese character is not preceded by "##"
                          self.assertListEqual(A,       A      )
                          self.assertListEqual(A,       A      )

                          _UpperCamelCase              =						False
                          _UpperCamelCase              =						self.rust_tokenizer_class.from_pretrained(A,       **A      )
                          _UpperCamelCase              =						self.tokenizer_class.from_pretrained(A,       **A      )

                          _UpperCamelCase              =						tokenizer_r.encode(A,       add_special_tokens=A      )
                          _UpperCamelCase              =						tokenizer_p.encode(A,       add_special_tokens=A      )

                          _UpperCamelCase              =						tokenizer_r.convert_ids_to_tokens(A      )
                          _UpperCamelCase              =						tokenizer_p.convert_ids_to_tokens(A      )

                          # it is expected that only the first Chinese character is not preceded by "##".
                          _UpperCamelCase              =						[
                              F'''##{token}''' if idx != 0 else token for idx, token in enumerate(A      )
                          ]
                          self.assertListEqual(A,       A      )
                          self.assertListEqual(A,       A      )













from __future__ import annotations

from collections.abc import Iterator

class  __A:

     def __init__(      self,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						value
            _UpperCamelCase              =						None
            _UpperCamelCase              =						None






class  __A:

     def __init__(      self,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						tree

     def 	_UpperCamelCase     (      self,       A      ):





            """simple docstring"""

            if node is None:
                   return 0
            return node.value + (
                self.depth_first_search(node.left      ) + self.depth_first_search(node.right      )
            )



     def __iter__(      self      ):





            """simple docstring"""

            yield self.depth_first_search(self.tree      )


if __name__ == "__main__":
       import doctest

       doctest.testmod()






