



import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class     _UpperCAmelCase			(       A__			,      unittest.TestCase     ):
     UpperCamelCase__		    =				MobileBertTokenizer
     UpperCamelCase__		    =				MobileBertTokenizerFast
     UpperCamelCase__		    =				True
     UpperCamelCase__		    =				True
     UpperCamelCase__		    =				filter_non_english
     UpperCamelCase__		    =				'''google/mobilebert-uncased'''




     def 		snake_case_		(				self):
         super().setUp()

         A__		 =					[
             '''[UNK]''',
             '''[CLS]''',
             '''[SEP]''',
             '''[PAD]''',
             '''[MASK]''',
             '''want''',
             '''##want''',
             '''##ed''',
             '''wa''',
             '''un''',
             '''runn''',
             '''##ing''',
             ''',''',
             '''low''',
             '''lowest''',
         ]
         A__		 =					os.path.join(self.tmpdirname ,						VOCAB_FILES_NAMES['''vocab_file'''])
         with open(self.vocab_file ,						'''w''' ,						encoding='''utf-8''') as vocab_writer:
             vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]))

         A__		 =					[
             (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
             for tokenizer_def in self.tokenizers_list
         ]




     def 		snake_case_		(				self ,						a__):
         A__		 =					'''UNwant\u00E9d,running'''
         A__		 =					'''unwanted, running'''
         return input_text, output_text




     def 		snake_case_		(				self):
         A__		 =					self.tokenizer_class(self.vocab_file)

         A__		 =					tokenizer.tokenize('''UNwant\u00E9d,running''')
         self.assertListEqual(a__ ,						['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''])
         self.assertListEqual(tokenizer.convert_tokens_to_ids(a__) ,						[9, 6, 7, 1_2, 1_0, 1_1])




     def 		snake_case_		(				self):
         if not self.test_rust_tokenizer:
             return

         A__		 =					self.get_tokenizer()
         A__		 =					self.get_rust_tokenizer()

         A__		 =					'''UNwant\u00E9d,running'''

         A__		 =					tokenizer.tokenize(a__)
         A__		 =					rust_tokenizer.tokenize(a__)
         self.assertListEqual(a__ ,						a__)

         A__		 =					tokenizer.encode(a__ ,						add_special_tokens=a__)
         A__		 =					rust_tokenizer.encode(a__ ,						add_special_tokens=a__)
         self.assertListEqual(a__ ,						a__)

         A__		 =					self.get_rust_tokenizer()
         A__		 =					tokenizer.encode(a__)
         A__		 =					rust_tokenizer.encode(a__)
         self.assertListEqual(a__ ,						a__)

         # With lower casing
         A__		 =					self.get_tokenizer(do_lower_case=a__)
         A__		 =					self.get_rust_tokenizer(do_lower_case=a__)

         A__		 =					'''UNwant\u00E9d,running'''

         A__		 =					tokenizer.tokenize(a__)
         A__		 =					rust_tokenizer.tokenize(a__)
         self.assertListEqual(a__ ,						a__)

         A__		 =					tokenizer.encode(a__ ,						add_special_tokens=a__)
         A__		 =					rust_tokenizer.encode(a__ ,						add_special_tokens=a__)
         self.assertListEqual(a__ ,						a__)

         A__		 =					self.get_rust_tokenizer()
         A__		 =					tokenizer.encode(a__)
         A__		 =					rust_tokenizer.encode(a__)
         self.assertListEqual(a__ ,						a__)




     def 		snake_case_		(				self):
         A__		 =					BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''') ,						['''ah''', '''\u535A''', '''\u63A8''', '''zz'''])




     def 		snake_case_		(				self):
         A__		 =					BasicTokenizer(do_lower_case=a__)

         self.assertListEqual(
             tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''') ,						['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''])
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''') ,						['''hello'''])




     def 		snake_case_		(				self):
         A__		 =					BasicTokenizer(do_lower_case=a__ ,						strip_accents=a__)

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''') ,						['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''') ,						['''h\u00E9llo'''])




     def 		snake_case_		(				self):
         A__		 =					BasicTokenizer(do_lower_case=a__ ,						strip_accents=a__)

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''') ,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''') ,						['''hello'''])




     def 		snake_case_		(				self):
         A__		 =					BasicTokenizer(do_lower_case=a__)

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''') ,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''') ,						['''hello'''])




     def 		snake_case_		(				self):
         A__		 =					BasicTokenizer(do_lower_case=a__)

         self.assertListEqual(
             tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''') ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])




     def 		snake_case_		(				self):
         A__		 =					BasicTokenizer(do_lower_case=a__ ,						strip_accents=a__)

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''') ,						['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])




     def 		snake_case_		(				self):
         A__		 =					BasicTokenizer(do_lower_case=a__ ,						strip_accents=a__)

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''') ,						['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])




     def 		snake_case_		(				self):
         A__		 =					BasicTokenizer(do_lower_case=a__ ,						never_split=['''[UNK]'''])

         self.assertListEqual(
             tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''') ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''])




     def 		snake_case_		(				self):
         A__		 =					['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

         A__		 =					{}
         for i, token in enumerate(a__):
             A__		 =					i
         A__		 =					WordpieceTokenizer(vocab=a__ ,						unk_token='''[UNK]''')

         self.assertListEqual(tokenizer.tokenize('''''') ,						[])

         self.assertListEqual(tokenizer.tokenize('''unwanted running''') ,						['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''])

         self.assertListEqual(tokenizer.tokenize('''unwantedX running''') ,						['''[UNK]''', '''runn''', '''##ing'''])




     def 		snake_case_		(				self):
         self.assertTrue(_is_whitespace(''' '''))
         self.assertTrue(_is_whitespace('''\t'''))
         self.assertTrue(_is_whitespace('''\r'''))
         self.assertTrue(_is_whitespace('''\n'''))
         self.assertTrue(_is_whitespace('''\u00A0'''))

         self.assertFalse(_is_whitespace('''A'''))
         self.assertFalse(_is_whitespace('''-'''))




     def 		snake_case_		(				self):
         self.assertTrue(_is_control('''\u0005'''))

         self.assertFalse(_is_control('''A'''))
         self.assertFalse(_is_control(''' '''))
         self.assertFalse(_is_control('''\t'''))
         self.assertFalse(_is_control('''\r'''))




     def 		snake_case_		(				self):
         self.assertTrue(_is_punctuation('''-'''))
         self.assertTrue(_is_punctuation('''$'''))
         self.assertTrue(_is_punctuation('''`'''))
         self.assertTrue(_is_punctuation('''.'''))

         self.assertFalse(_is_punctuation('''A'''))
         self.assertFalse(_is_punctuation(''' '''))




     def 		snake_case_		(				self):
         A__		 =					self.get_tokenizer()
         A__		 =					self.get_rust_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(a__) for t in ['''Test''', '''\xad''', '''test''']] ,						[['''[UNK]'''], [], ['''[UNK]''']])

         self.assertListEqual(
             [rust_tokenizer.tokenize(a__) for t in ['''Test''', '''\xad''', '''test''']] ,						[['''[UNK]'''], [], ['''[UNK]''']])




     @slow
     def 		snake_case_		(				self):
         A__		 =					self.tokenizer_class.from_pretrained('''google/mobilebert-uncased''')

         A__		 =					tokenizer.encode('''sequence builders''' ,						add_special_tokens=a__)
         A__		 =					tokenizer.encode('''multi-sequence build''' ,						add_special_tokens=a__)

         A__		 =					tokenizer.build_inputs_with_special_tokens(a__)
         A__		 =					tokenizer.build_inputs_with_special_tokens(a__ ,						a__)

         assert encoded_sentence == [1_0_1] + text + [1_0_2]
         assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]




     def 		snake_case_		(				self):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"):
                 A__		 =					self.rust_tokenizer_class.from_pretrained(a__ ,						**a__)

                 A__		 =					F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
                 A__		 =					tokenizer_r.encode_plus(
                     a__ ,						return_attention_mask=a__ ,						return_token_type_ids=a__ ,						return_offsets_mapping=a__ ,						add_special_tokens=a__ ,						)

                 A__		 =					tokenizer_r.do_lower_case if hasattr(a__ ,						'''do_lower_case''') else False
                 A__		 =					(
                     [
                         ((0, 0), tokenizer_r.cls_token),
                         ((0, 1), '''A'''),
                         ((1, 2), ''','''),
                         ((3, 5), '''na'''),
                         ((5, 6), '''##ï'''),
                         ((6, 8), '''##ve'''),
                         ((9, 1_5), tokenizer_r.mask_token),
                         ((1_6, 2_1), '''Allen'''),
                         ((2_1, 2_3), '''##NL'''),
                         ((2_3, 2_4), '''##P'''),
                         ((2_5, 3_3), '''sentence'''),
                         ((3_3, 3_4), '''.'''),
                         ((0, 0), tokenizer_r.sep_token),
                     ]
                     if not do_lower_case
                     else [
                         ((0, 0), tokenizer_r.cls_token),
                         ((0, 1), '''a'''),
                         ((1, 2), ''','''),
                         ((3, 8), '''naive'''),
                         ((9, 1_5), tokenizer_r.mask_token),
                         ((1_6, 2_1), '''allen'''),
                         ((2_1, 2_3), '''##nl'''),
                         ((2_3, 2_4), '''##p'''),
                         ((2_5, 3_3), '''sentence'''),
                         ((3_3, 3_4), '''.'''),
                         ((0, 0), tokenizer_r.sep_token),
                     ]
                 )

                 self.assertEqual(
                     [e[1] for e in expected_results] ,						tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']))
                 self.assertEqual([e[0] for e in expected_results] ,						tokens['''offset_mapping'''])




     def 		snake_case_		(				self):
         A__		 =					['''的''', '''人''', '''有''']
         A__		 =					''''''.join(a__)
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"):
                 A__		 =					True
                 A__		 =					self.tokenizer_class.from_pretrained(a__ ,						**a__)
                 A__		 =					self.rust_tokenizer_class.from_pretrained(a__ ,						**a__)

                 A__		 =					tokenizer_p.encode(a__ ,						add_special_tokens=a__)
                 A__		 =					tokenizer_r.encode(a__ ,						add_special_tokens=a__)

                 A__		 =					tokenizer_r.convert_ids_to_tokens(a__)
                 A__		 =					tokenizer_p.convert_ids_to_tokens(a__)

                 # it is expected that each Chinese character is not preceded by "##"
                 self.assertListEqual(a__ ,						a__)
                 self.assertListEqual(a__ ,						a__)

                 A__		 =					False
                 A__		 =					self.rust_tokenizer_class.from_pretrained(a__ ,						**a__)
                 A__		 =					self.tokenizer_class.from_pretrained(a__ ,						**a__)

                 A__		 =					tokenizer_r.encode(a__ ,						add_special_tokens=a__)
                 A__		 =					tokenizer_p.encode(a__ ,						add_special_tokens=a__)

                 A__		 =					tokenizer_r.convert_ids_to_tokens(a__)
                 A__		 =					tokenizer_p.convert_ids_to_tokens(a__)

                 # it is expected that only the first Chinese character is not preceded by "##".
                 A__		 =					[
                     F"##{token}" if idx != 0 else token for idx, token in enumerate(a__)
                 ]
                 self.assertListEqual(a__ ,						a__)
                 self.assertListEqual(a__ ,						a__)







from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowercase						  =		logging.get_logger(__name__)



class     _UpperCAmelCase			(       A__     ):
     UpperCamelCase__		    =				'''timm_backbone'''




     def __init__(				self ,						a__=None ,						a__=3 ,						a__=True ,						a__=True ,						a__=None ,						**a__ ,						):
         super().__init__(**a__)
         A__		 =					backbone
         A__		 =					num_channels
         A__		 =					features_only
         A__		 =					use_pretrained_backbone
         A__		 =					True
         A__		 =					out_indices if out_indices is not None else (-1,)
