
import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class     __UpperCamelCase (							lowerCAmelCase__ ,						unittest.TestCase       ):






     """simple docstring"""







     lowerCAmelCase_							   =			MobileBertTokenizer
     lowerCAmelCase_							   =			MobileBertTokenizerFast
     lowerCAmelCase_							   =			True
     lowerCAmelCase_							   =			True
     lowerCAmelCase_							   =			filter_non_english
     lowerCAmelCase_							   =			'''google/mobilebert-uncased'''



     def  UpperCAmelCase__							(				self							:							Dict ):







       """simple docstring"""




       super().setUp()

       __SCREAMING_SNAKE_CASE      :	List[str]					=		[
           '''[UNK]''',
           '''[CLS]''',
           '''[SEP]''',
           '''[PAD]''',
           '''[MASK]''',
           '''want''',
           '''##want''',
           '''##ed''',
           '''wa''',
           '''un''',
           '''runn''',
           '''##ing''',
           ''',''',
           '''low''',
           '''lowest''',
       ]
       __SCREAMING_SNAKE_CASE      :	str					=		os.path.join(self.tmpdirname     ,						VOCAB_FILES_NAMES['''vocab_file'''] )
       with open(self.vocab_file     ,						'''w'''     ,						encoding='''utf-8''' ) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )

       __SCREAMING_SNAKE_CASE      :	int					=		[
           (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
           for tokenizer_def in self.tokenizers_list
       ]



     def  UpperCAmelCase__							(				self							:							Tuple     ,						_A							:							Optional[int] ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		'''UNwant\u00E9d,running'''
       __SCREAMING_SNAKE_CASE      :	List[str]					=		'''unwanted, running'''
       return input_text, output_text



     def  UpperCAmelCase__							(				self							:							Optional[int] ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Optional[int]					=		self.tokenizer_class(self.vocab_file )

       __SCREAMING_SNAKE_CASE      :	List[str]					=		tokenizer.tokenize('''UNwant\u00E9d,running''' )
       self.assertListEqual(_A     ,						['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_A )     ,						[9, 6, 7, 12, 10, 11] )



     def  UpperCAmelCase__							(				self							:							int ):







       """simple docstring"""




       if not self.test_rust_tokenizer:
         return

       __SCREAMING_SNAKE_CASE      :	List[str]					=		self.get_tokenizer()
       __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		self.get_rust_tokenizer()

       __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		'''UNwant\u00E9d,running'''

       __SCREAMING_SNAKE_CASE      :	Any					=		tokenizer.tokenize(_A )
       __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		rust_tokenizer.tokenize(_A )
       self.assertListEqual(_A     ,						_A )

       __SCREAMING_SNAKE_CASE      :	Dict					=		tokenizer.encode(_A     ,						add_special_tokens=_A )
       __SCREAMING_SNAKE_CASE      :	str					=		rust_tokenizer.encode(_A     ,						add_special_tokens=_A )
       self.assertListEqual(_A     ,						_A )

       __SCREAMING_SNAKE_CASE      :	Any					=		self.get_rust_tokenizer()
       __SCREAMING_SNAKE_CASE      :	str					=		tokenizer.encode(_A )
       __SCREAMING_SNAKE_CASE      :	Any					=		rust_tokenizer.encode(_A )
       self.assertListEqual(_A     ,						_A )

       # With lower casing
       __SCREAMING_SNAKE_CASE      :	Any					=		self.get_tokenizer(do_lower_case=_A )
       __SCREAMING_SNAKE_CASE      :	List[str]					=		self.get_rust_tokenizer(do_lower_case=_A )

       __SCREAMING_SNAKE_CASE      :	List[str]					=		'''UNwant\u00E9d,running'''

       __SCREAMING_SNAKE_CASE      :	Any					=		tokenizer.tokenize(_A )
       __SCREAMING_SNAKE_CASE      :	Optional[int]					=		rust_tokenizer.tokenize(_A )
       self.assertListEqual(_A     ,						_A )

       __SCREAMING_SNAKE_CASE      :	Any					=		tokenizer.encode(_A     ,						add_special_tokens=_A )
       __SCREAMING_SNAKE_CASE      :	List[str]					=		rust_tokenizer.encode(_A     ,						add_special_tokens=_A )
       self.assertListEqual(_A     ,						_A )

       __SCREAMING_SNAKE_CASE      :	int					=		self.get_rust_tokenizer()
       __SCREAMING_SNAKE_CASE      :	Any					=		tokenizer.encode(_A )
       __SCREAMING_SNAKE_CASE      :	Optional[int]					=		rust_tokenizer.encode(_A )
       self.assertListEqual(_A     ,						_A )



     def  UpperCAmelCase__							(				self							:							Optional[int] ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Optional[int]					=		BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''' )     ,						['''ah''', '''\u535A''', '''\u63A8''', '''zz'''] )



     def  UpperCAmelCase__							(				self							:							Optional[Any] ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Dict					=		BasicTokenizer(do_lower_case=_A )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' )     ,						['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )     ,						['''hello'''] )



     def  UpperCAmelCase__							(				self							:							Tuple ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		BasicTokenizer(do_lower_case=_A     ,						strip_accents=_A )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )     ,						['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )     ,						['''h\u00E9llo'''] )



     def  UpperCAmelCase__							(				self							:							List[str] ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Optional[int]					=		BasicTokenizer(do_lower_case=_A     ,						strip_accents=_A )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )     ,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )     ,						['''hello'''] )



     def  UpperCAmelCase__							(				self							:							Tuple ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	List[Any]					=		BasicTokenizer(do_lower_case=_A )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )     ,						['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )     ,						['''hello'''] )



     def  UpperCAmelCase__							(				self							:							Optional[int] ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	int					=		BasicTokenizer(do_lower_case=_A )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' )     ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )



     def  UpperCAmelCase__							(				self							:							Any ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Any					=		BasicTokenizer(do_lower_case=_A     ,						strip_accents=_A )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )     ,						['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )



     def  UpperCAmelCase__							(				self							:							Optional[int] ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Tuple					=		BasicTokenizer(do_lower_case=_A     ,						strip_accents=_A )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )     ,						['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )



     def  UpperCAmelCase__							(				self							:							Union[str, Any] ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Any					=		BasicTokenizer(do_lower_case=_A     ,						never_split=['''[UNK]'''] )

       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''' )     ,						['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''] )



     def  UpperCAmelCase__							(				self							:							Union[str, Any] ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

       __SCREAMING_SNAKE_CASE      :	Dict					=		{}
       for i, token in enumerate(_A ):
         __SCREAMING_SNAKE_CASE      :	List[str]					=		i
       __SCREAMING_SNAKE_CASE      :	str					=		WordpieceTokenizer(vocab=_A     ,						unk_token='''[UNK]''' )

       self.assertListEqual(tokenizer.tokenize('''''' )     ,						[] )

       self.assertListEqual(tokenizer.tokenize('''unwanted running''' )     ,						['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''] )

       self.assertListEqual(tokenizer.tokenize('''unwantedX running''' )     ,						['''[UNK]''', '''runn''', '''##ing'''] )



     def  UpperCAmelCase__							(				self							:							List[str] ):







       """simple docstring"""




       self.assertTrue(_is_whitespace(''' ''' ) )
       self.assertTrue(_is_whitespace('''\t''' ) )
       self.assertTrue(_is_whitespace('''\r''' ) )
       self.assertTrue(_is_whitespace('''\n''' ) )
       self.assertTrue(_is_whitespace('''\u00A0''' ) )

       self.assertFalse(_is_whitespace('''A''' ) )
       self.assertFalse(_is_whitespace('''-''' ) )



     def  UpperCAmelCase__							(				self							:							str ):







       """simple docstring"""




       self.assertTrue(_is_control('''\u0005''' ) )

       self.assertFalse(_is_control('''A''' ) )
       self.assertFalse(_is_control(''' ''' ) )
       self.assertFalse(_is_control('''\t''' ) )
       self.assertFalse(_is_control('''\r''' ) )



     def  UpperCAmelCase__							(				self							:							Any ):







       """simple docstring"""




       self.assertTrue(_is_punctuation('''-''' ) )
       self.assertTrue(_is_punctuation('''$''' ) )
       self.assertTrue(_is_punctuation('''`''' ) )
       self.assertTrue(_is_punctuation('''.''' ) )

       self.assertFalse(_is_punctuation('''A''' ) )
       self.assertFalse(_is_punctuation(''' ''' ) )



     def  UpperCAmelCase__							(				self							:							Dict ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Dict					=		self.get_tokenizer()
       __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(_A ) for t in ['''Test''', '''\xad''', '''test''']]     ,						[['''[UNK]'''], [], ['''[UNK]''']] )

       self.assertListEqual(
           [rust_tokenizer.tokenize(_A ) for t in ['''Test''', '''\xad''', '''test''']]     ,						[['''[UNK]'''], [], ['''[UNK]''']] )



     @slow
     def  UpperCAmelCase__							(				self							:							List[str] ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		self.tokenizer_class.from_pretrained('''google/mobilebert-uncased''' )

       __SCREAMING_SNAKE_CASE      :	Tuple					=		tokenizer.encode('''sequence builders'''     ,						add_special_tokens=_A )
       __SCREAMING_SNAKE_CASE      :	int					=		tokenizer.encode('''multi-sequence build'''     ,						add_special_tokens=_A )

       __SCREAMING_SNAKE_CASE      :	Any					=		tokenizer.build_inputs_with_special_tokens(_A )
       __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		tokenizer.build_inputs_with_special_tokens(_A     ,						_A )

       assert encoded_sentence == [101] + text + [102]
       assert encoded_pair == [101] + text + [102] + text_a + [102]



     def  UpperCAmelCase__							(				self							:							Tuple ):







       """simple docstring"""




       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
           __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		self.rust_tokenizer_class.from_pretrained(_A     ,						**_A )

           __SCREAMING_SNAKE_CASE      :	str					=		F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
           __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		tokenizer_r.encode_plus(
               _A     ,						return_attention_mask=_A     ,						return_token_type_ids=_A     ,						return_offsets_mapping=_A     ,						add_special_tokens=_A     ,						)

           __SCREAMING_SNAKE_CASE      :	Union[str, Any]					=		tokenizer_r.do_lower_case if hasattr(_A     ,						'''do_lower_case''' ) else False
           __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		(
               [
                   ((0, 0), tokenizer_r.cls_token),
                   ((0, 1), '''A'''),
                   ((1, 2), ''','''),
                   ((3, 5), '''na'''),
                   ((5, 6), '''##ï'''),
                   ((6, 8), '''##ve'''),
                   ((9, 15), tokenizer_r.mask_token),
                   ((16, 21), '''Allen'''),
                   ((21, 23), '''##NL'''),
                   ((23, 24), '''##P'''),
                   ((25, 33), '''sentence'''),
                   ((33, 34), '''.'''),
                   ((0, 0), tokenizer_r.sep_token),
               ]
               if not do_lower_case
               else [
                   ((0, 0), tokenizer_r.cls_token),
                   ((0, 1), '''a'''),
                   ((1, 2), ''','''),
                   ((3, 8), '''naive'''),
                   ((9, 15), tokenizer_r.mask_token),
                   ((16, 21), '''allen'''),
                   ((21, 23), '''##nl'''),
                   ((23, 24), '''##p'''),
                   ((25, 33), '''sentence'''),
                   ((33, 34), '''.'''),
                   ((0, 0), tokenizer_r.sep_token),
               ]
           )

           self.assertEqual(
               [e[1] for e in expected_results]     ,						tokenizer_r.convert_ids_to_tokens(tokens['''input_ids'''] ) )
           self.assertEqual([e[0] for e in expected_results]     ,						tokens['''offset_mapping'''] )




     def  UpperCAmelCase__							(				self							:							str ):







       """simple docstring"""




       __SCREAMING_SNAKE_CASE      :	Optional[int]					=		['''的''', '''人''', '''有''']
       __SCREAMING_SNAKE_CASE      :	int					=		''''''.join(_A )
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})''' ):
           __SCREAMING_SNAKE_CASE      :	str					=		True
           __SCREAMING_SNAKE_CASE      :	int					=		self.tokenizer_class.from_pretrained(_A     ,						**_A )
           __SCREAMING_SNAKE_CASE      :	int					=		self.rust_tokenizer_class.from_pretrained(_A     ,						**_A )

           __SCREAMING_SNAKE_CASE      :	List[str]					=		tokenizer_p.encode(_A     ,						add_special_tokens=_A )
           __SCREAMING_SNAKE_CASE      :	Tuple					=		tokenizer_r.encode(_A     ,						add_special_tokens=_A )

           __SCREAMING_SNAKE_CASE      :	Optional[int]					=		tokenizer_r.convert_ids_to_tokens(_A )
           __SCREAMING_SNAKE_CASE      :	int					=		tokenizer_p.convert_ids_to_tokens(_A )

           # it is expected that each Chinese character is not preceded by "##"
           self.assertListEqual(_A     ,						_A )
           self.assertListEqual(_A     ,						_A )

           __SCREAMING_SNAKE_CASE      :	Optional[Any]					=		False
           __SCREAMING_SNAKE_CASE      :	Any					=		self.rust_tokenizer_class.from_pretrained(_A     ,						**_A )
           __SCREAMING_SNAKE_CASE      :	List[str]					=		self.tokenizer_class.from_pretrained(_A     ,						**_A )

           __SCREAMING_SNAKE_CASE      :	List[Any]					=		tokenizer_r.encode(_A     ,						add_special_tokens=_A )
           __SCREAMING_SNAKE_CASE      :	int					=		tokenizer_p.encode(_A     ,						add_special_tokens=_A )

           __SCREAMING_SNAKE_CASE      :	Dict					=		tokenizer_r.convert_ids_to_tokens(_A )
           __SCREAMING_SNAKE_CASE      :	int					=		tokenizer_p.convert_ids_to_tokens(_A )

           # it is expected that only the first Chinese character is not preceded by "##".
           __SCREAMING_SNAKE_CASE      :	List[Any]					=		[
               F'''##{token}''' if idx != 0 else token for idx, token in enumerate(_A )
           ]
           self.assertListEqual(_A     ,						_A )
           self.assertListEqual(_A     ,						_A )




'''simple docstring'''





from ..utils import DummyObject, requires_backends




class 				UpperCAmelCase			(			metaclass=_snake_case						):
 UpperCAmelCase				=					["speech"]


 def __init__(      self							:	List[Any]			,       *__lowerCamelCase							:	List[Any]			,       **__lowerCamelCase							:	List[str] ):
    requires_backends(self			,       ['''speech'''] )




class 				UpperCAmelCase			(			metaclass=_snake_case						):
 UpperCAmelCase				=					["speech"]


 def __init__(      self							:	int			,       *__lowerCamelCase							:	List[Any]			,       **__lowerCamelCase							:	List[str] ):
    requires_backends(self			,       ['''speech'''] )



