



import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class 		_snake_case   (      lowerCamelCase       ,unittest.TestCase					):





 """simple docstring"""



 lowerCamelCase_        =      MobileBertTokenizer
 lowerCamelCase_        =      MobileBertTokenizerFast
 lowerCamelCase_        =      True
 lowerCamelCase_        =      True
 lowerCamelCase_        =      filter_non_english
 lowerCamelCase_        =      '''google/mobilebert-uncased'''
 def lowercase_		(    self       )						->				Any:




   """simple docstring"""







   super().setUp()

   _A							     =     [
       '''[UNK]''',
       '''[CLS]''',
       '''[SEP]''',
       '''[PAD]''',
       '''[MASK]''',
       '''want''',
       '''##want''',
       '''##ed''',
       '''wa''',
       '''un''',
       '''runn''',
       '''##ing''',
       ''',''',
       '''low''',
       '''lowest''',
   ]
   _A							     =     os.path.join(self.tmpdirname						,     VOCAB_FILES_NAMES['''vocab_file''']       )
   with open(self.vocab_file						,     '''w'''						,     encoding='''utf-8'''       ) as vocab_writer:
     vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]       )       )

   _A							     =     [
       (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
       for tokenizer_def in self.tokenizers_list
   ]
 def lowercase_		(    self						,     a       )						->				int:




   """simple docstring"""







   _A							     =     '''UNwant\u00E9d,running'''
   _A							     =     '''unwanted, running'''
   return input_text, output_text
 def lowercase_		(    self       )						->				Tuple:




   """simple docstring"""







   _A							     =     self.tokenizer_class(self.vocab_file       )

   _A							     =     tokenizer.tokenize('''UNwant\u00E9d,running'''       )
   self.assertListEqual(_UpperCamelCase						,     ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']       )
   self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase       )						,     [9, 6, 7, 1_2, 1_0, 1_1]       )
 def lowercase_		(    self       )						->				Any:




   """simple docstring"""







   if not self.test_rust_tokenizer:
     return

   _A							     =     self.get_tokenizer()
   _A							     =     self.get_rust_tokenizer()

   _A							     =     '''UNwant\u00E9d,running'''

   _A							     =     tokenizer.tokenize(_UpperCamelCase       )
   _A							     =     rust_tokenizer.tokenize(_UpperCamelCase       )
   self.assertListEqual(_UpperCamelCase						,     _UpperCamelCase       )

   _A							     =     tokenizer.encode(_UpperCamelCase						,     add_special_tokens=_UpperCamelCase       )
   _A							     =     rust_tokenizer.encode(_UpperCamelCase						,     add_special_tokens=_UpperCamelCase       )
   self.assertListEqual(_UpperCamelCase						,     _UpperCamelCase       )

   _A							     =     self.get_rust_tokenizer()
   _A							     =     tokenizer.encode(_UpperCamelCase       )
   _A							     =     rust_tokenizer.encode(_UpperCamelCase       )
   self.assertListEqual(_UpperCamelCase						,     _UpperCamelCase       )

   # With lower casing
   _A							     =     self.get_tokenizer(do_lower_case=_UpperCamelCase       )
   _A							     =     self.get_rust_tokenizer(do_lower_case=_UpperCamelCase       )

   _A							     =     '''UNwant\u00E9d,running'''

   _A							     =     tokenizer.tokenize(_UpperCamelCase       )
   _A							     =     rust_tokenizer.tokenize(_UpperCamelCase       )
   self.assertListEqual(_UpperCamelCase						,     _UpperCamelCase       )

   _A							     =     tokenizer.encode(_UpperCamelCase						,     add_special_tokens=_UpperCamelCase       )
   _A							     =     rust_tokenizer.encode(_UpperCamelCase						,     add_special_tokens=_UpperCamelCase       )
   self.assertListEqual(_UpperCamelCase						,     _UpperCamelCase       )

   _A							     =     self.get_rust_tokenizer()
   _A							     =     tokenizer.encode(_UpperCamelCase       )
   _A							     =     rust_tokenizer.encode(_UpperCamelCase       )
   self.assertListEqual(_UpperCamelCase						,     _UpperCamelCase       )
 def lowercase_		(    self       )						->				str:




   """simple docstring"""







   _A							     =     BasicTokenizer()

   self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''       )						,     ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']       )
 def lowercase_		(    self       )						->				List[str]:




   """simple docstring"""







   _A							     =     BasicTokenizer(do_lower_case=_UpperCamelCase       )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''       )						,     ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )						,     ['''hello''']       )
 def lowercase_		(    self       )						->				List[Any]:




   """simple docstring"""







   _A							     =     BasicTokenizer(do_lower_case=_UpperCamelCase						,     strip_accents=_UpperCamelCase       )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )						,     ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )						,     ['''h\u00E9llo''']       )
 def lowercase_		(    self       )						->				List[Any]:




   """simple docstring"""







   _A							     =     BasicTokenizer(do_lower_case=_UpperCamelCase						,     strip_accents=_UpperCamelCase       )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )						,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )						,     ['''hello''']       )
 def lowercase_		(    self       )						->				Optional[int]:




   """simple docstring"""







   _A							     =     BasicTokenizer(do_lower_case=_UpperCamelCase       )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )						,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       )						,     ['''hello''']       )
 def lowercase_		(    self       )						->				Tuple:




   """simple docstring"""







   _A							     =     BasicTokenizer(do_lower_case=_UpperCamelCase       )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''       )						,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )
 def lowercase_		(    self       )						->				Union[str, Any]:




   """simple docstring"""







   _A							     =     BasicTokenizer(do_lower_case=_UpperCamelCase						,     strip_accents=_UpperCamelCase       )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )						,     ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )
 def lowercase_		(    self       )						->				Tuple:




   """simple docstring"""







   _A							     =     BasicTokenizer(do_lower_case=_UpperCamelCase						,     strip_accents=_UpperCamelCase       )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       )						,     ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )
 def lowercase_		(    self       )						->				Tuple:




   """simple docstring"""







   _A							     =     BasicTokenizer(do_lower_case=_UpperCamelCase						,     never_split=['''[UNK]''']       )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''       )						,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']       )
 def lowercase_		(    self       )						->				Optional[Any]:




   """simple docstring"""







   _A							     =     ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

   _A							     =     {}
   for i, token in enumerate(_UpperCamelCase       ):
     _A							     =     i
   _A							     =     WordpieceTokenizer(vocab=_UpperCamelCase						,     unk_token='''[UNK]'''       )

   self.assertListEqual(tokenizer.tokenize(''''''       )						,     []       )

   self.assertListEqual(tokenizer.tokenize('''unwanted running'''       )						,     ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']       )

   self.assertListEqual(tokenizer.tokenize('''unwantedX running'''       )						,     ['''[UNK]''', '''runn''', '''##ing''']       )
 def lowercase_		(    self       )						->				Optional[int]:




   """simple docstring"""







   self.assertTrue(_is_whitespace(''' '''       )       )
   self.assertTrue(_is_whitespace('''\t'''       )       )
   self.assertTrue(_is_whitespace('''\r'''       )       )
   self.assertTrue(_is_whitespace('''\n'''       )       )
   self.assertTrue(_is_whitespace('''\u00A0'''       )       )

   self.assertFalse(_is_whitespace('''A'''       )       )
   self.assertFalse(_is_whitespace('''-'''       )       )
 def lowercase_		(    self       )						->				List[Any]:




   """simple docstring"""







   self.assertTrue(_is_control('''\u0005'''       )       )

   self.assertFalse(_is_control('''A'''       )       )
   self.assertFalse(_is_control(''' '''       )       )
   self.assertFalse(_is_control('''\t'''       )       )
   self.assertFalse(_is_control('''\r'''       )       )
 def lowercase_		(    self       )						->				List[str]:




   """simple docstring"""







   self.assertTrue(_is_punctuation('''-'''       )       )
   self.assertTrue(_is_punctuation('''$'''       )       )
   self.assertTrue(_is_punctuation('''`'''       )       )
   self.assertTrue(_is_punctuation('''.'''       )       )

   self.assertFalse(_is_punctuation('''A'''       )       )
   self.assertFalse(_is_punctuation(''' '''       )       )
 def lowercase_		(    self       )						->				Dict:




   """simple docstring"""







   _A							     =     self.get_tokenizer()
   _A							     =     self.get_rust_tokenizer()

   # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
   self.assertListEqual([tokenizer.tokenize(_UpperCamelCase       ) for t in ['''Test''', '''\xad''', '''test''']]						,     [['''[UNK]'''], [], ['''[UNK]''']]       )

   self.assertListEqual(
       [rust_tokenizer.tokenize(_UpperCamelCase       ) for t in ['''Test''', '''\xad''', '''test''']]						,     [['''[UNK]'''], [], ['''[UNK]''']]       )
 @slow
 def lowercase_		(    self       )						->				Tuple:




   """simple docstring"""







   _A							     =     self.tokenizer_class.from_pretrained('''google/mobilebert-uncased'''       )

   _A							     =     tokenizer.encode('''sequence builders'''						,     add_special_tokens=_UpperCamelCase       )
   _A							     =     tokenizer.encode('''multi-sequence build'''						,     add_special_tokens=_UpperCamelCase       )

   _A							     =     tokenizer.build_inputs_with_special_tokens(_UpperCamelCase       )
   _A							     =     tokenizer.build_inputs_with_special_tokens(_UpperCamelCase						,     _UpperCamelCase       )

   assert encoded_sentence == [1_0_1] + text + [1_0_2]
   assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]
 def lowercase_		(    self       )						->				str:




   """simple docstring"""







   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
       _A							     =     self.rust_tokenizer_class.from_pretrained(_UpperCamelCase						,     **_UpperCamelCase       )

       _A							     =     f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
       _A							     =     tokenizer_r.encode_plus(
           _UpperCamelCase						,     return_attention_mask=_UpperCamelCase						,     return_token_type_ids=_UpperCamelCase						,     return_offsets_mapping=_UpperCamelCase						,     add_special_tokens=_UpperCamelCase						,     )

       _A							     =     tokenizer_r.do_lower_case if hasattr(_UpperCamelCase						,     '''do_lower_case'''       ) else False
       _A							     =     (
           [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), '''A'''),
               ((1, 2), ''','''),
               ((3, 5), '''na'''),
               ((5, 6), '''##ï'''),
               ((6, 8), '''##ve'''),
               ((9, 1_5), tokenizer_r.mask_token),
               ((1_6, 2_1), '''Allen'''),
               ((2_1, 2_3), '''##NL'''),
               ((2_3, 2_4), '''##P'''),
               ((2_5, 3_3), '''sentence'''),
               ((3_3, 3_4), '''.'''),
               ((0, 0), tokenizer_r.sep_token),
           ]
           if not do_lower_case
           else [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), '''a'''),
               ((1, 2), ''','''),
               ((3, 8), '''naive'''),
               ((9, 1_5), tokenizer_r.mask_token),
               ((1_6, 2_1), '''allen'''),
               ((2_1, 2_3), '''##nl'''),
               ((2_3, 2_4), '''##p'''),
               ((2_5, 3_3), '''sentence'''),
               ((3_3, 3_4), '''.'''),
               ((0, 0), tokenizer_r.sep_token),
           ]
       )

       self.assertEqual(
           [e[1] for e in expected_results]						,     tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']       )       )
       self.assertEqual([e[0] for e in expected_results]						,     tokens['''offset_mapping''']       )



 def lowercase_		(    self       )						->				Tuple:




   """simple docstring"""







   _A							     =     ['''的''', '''人''', '''有''']
   _A							     =     ''''''.join(_UpperCamelCase       )
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
       _A							     =     True
       _A							     =     self.tokenizer_class.from_pretrained(_UpperCamelCase						,     **_UpperCamelCase       )
       _A							     =     self.rust_tokenizer_class.from_pretrained(_UpperCamelCase						,     **_UpperCamelCase       )

       _A							     =     tokenizer_p.encode(_UpperCamelCase						,     add_special_tokens=_UpperCamelCase       )
       _A							     =     tokenizer_r.encode(_UpperCamelCase						,     add_special_tokens=_UpperCamelCase       )

       _A							     =     tokenizer_r.convert_ids_to_tokens(_UpperCamelCase       )
       _A							     =     tokenizer_p.convert_ids_to_tokens(_UpperCamelCase       )

       # it is expected that each Chinese character is not preceded by "##"
       self.assertListEqual(_UpperCamelCase						,     _UpperCamelCase       )
       self.assertListEqual(_UpperCamelCase						,     _UpperCamelCase       )

       _A							     =     False
       _A							     =     self.rust_tokenizer_class.from_pretrained(_UpperCamelCase						,     **_UpperCamelCase       )
       _A							     =     self.tokenizer_class.from_pretrained(_UpperCamelCase						,     **_UpperCamelCase       )

       _A							     =     tokenizer_r.encode(_UpperCamelCase						,     add_special_tokens=_UpperCamelCase       )
       _A							     =     tokenizer_p.encode(_UpperCamelCase						,     add_special_tokens=_UpperCamelCase       )

       _A							     =     tokenizer_r.convert_ids_to_tokens(_UpperCamelCase       )
       _A							     =     tokenizer_p.convert_ids_to_tokens(_UpperCamelCase       )

       # it is expected that only the first Chinese character is not preceded by "##".
       _A							     =     [
           f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_UpperCamelCase       )
       ]
       self.assertListEqual(_UpperCamelCase						,     _UpperCamelCase       )
       self.assertListEqual(_UpperCamelCase						,     _UpperCamelCase       )



'''simple docstring'''
class        lowerCAmelCase_  :






       def __init__( self			,     _UpperCamelCase   )->      None:
            _A				=							size
            _A				=							[0] * size
            _A				=							[0] * size






       @staticmethod
       def   UpperCamelCase						( _UpperCamelCase   )->      int:
            return index | (index + 1)






       @staticmethod
       def   UpperCamelCase						( _UpperCamelCase   )->      int:
            return (index & (index + 1)) - 1






       def   UpperCamelCase						( self			,     _UpperCamelCase			,     _UpperCamelCase   )->      None:
            _A				=							value
            while index < self.size:
                 _A				=							self.get_prev(_UpperCamelCase   ) + 1
                 if current_left_border == index:
                      _A				=							value
                 else:
                      _A				=							max(_UpperCamelCase			,     _UpperCamelCase			,     _UpperCamelCase   )
                 _A				=							self.get_next(_UpperCamelCase   )






       def   UpperCamelCase						( self			,     _UpperCamelCase			,     _UpperCamelCase   )->      int:
            right -= 1  # Because of right is exclusive
            _A				=							0
            while left <= right:
                 _A				=							self.get_prev(_UpperCamelCase   )
                 if left <= current_left:
                      _A				=							max(_UpperCamelCase			,     self.tree[right]   )
                      _A				=							current_left
                 else:
                      _A				=							max(_UpperCamelCase			,     self.arr[right]   )
                      right -= 1
            return result


if __name__ == "__main__":
    import doctest

    doctest.testmod()





