

"""simple docstring"""




import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 			__magic_name__       (       lowercase_   ,					unittest.TestCase ):
       UpperCamelCase_   =		MobileBertTokenizer
       UpperCamelCase_   =		MobileBertTokenizerFast
       UpperCamelCase_   =		True
       UpperCamelCase_   =		True
       UpperCamelCase_   =		filter_non_english
       UpperCamelCase_   =		'''google/mobilebert-uncased'''
       def 							lowercase_							(     self       )     ->	Optional[int]:





         """simple docstring"""




         super().setUp()

         _lowercase:			Optional[int]       =     [
             """[UNK]""",
             """[CLS]""",
             """[SEP]""",
             """[PAD]""",
             """[MASK]""",
             """want""",
             """##want""",
             """##ed""",
             """wa""",
             """un""",
             """runn""",
             """##ing""",
             """,""",
             """low""",
             """lowest""",
         ]
         _lowercase:			int       =     os.path.join(self.tmpdirname ,				VOCAB_FILES_NAMES['''vocab_file''']       )
         with open(self.vocab_file ,				'''w''' ,				encoding='''utf-8'''       ) as vocab_writer:
           vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]       )       )

         _lowercase:			str       =     [
             (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
             for tokenizer_def in self.tokenizers_list
         ]
       def 							lowercase_							(     self ,				A_       )     ->	List[str]:





         """simple docstring"""




         _lowercase:			Optional[Any]       =     """UNwant\u00E9d,running"""
         _lowercase:			Any       =     """unwanted, running"""
         return input_text, output_text
       def 							lowercase_							(     self       )     ->	Dict:





         """simple docstring"""




         _lowercase:			Tuple       =     self.tokenizer_class(self.vocab_file       )

         _lowercase:			List[str]       =     tokenizer.tokenize('''UNwant\u00E9d,running'''       )
         self.assertListEqual(lowerCamelCase_ ,				['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']       )
         self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_       ) ,				[9, 6, 7, 12, 10, 11]       )
       def 							lowercase_							(     self       )     ->	Any:





         """simple docstring"""




         if not self.test_rust_tokenizer:
           return

         _lowercase:			int       =     self.get_tokenizer()
         _lowercase:			Optional[Any]       =     self.get_rust_tokenizer()

         _lowercase:			Optional[int]       =     """UNwant\u00E9d,running"""

         _lowercase:			Optional[int]       =     tokenizer.tokenize(lowerCamelCase_       )
         _lowercase:			List[Any]       =     rust_tokenizer.tokenize(lowerCamelCase_       )
         self.assertListEqual(lowerCamelCase_ ,				lowerCamelCase_       )

         _lowercase:			List[str]       =     tokenizer.encode(lowerCamelCase_ ,				add_special_tokens=lowerCamelCase_       )
         _lowercase:			Optional[int]       =     rust_tokenizer.encode(lowerCamelCase_ ,				add_special_tokens=lowerCamelCase_       )
         self.assertListEqual(lowerCamelCase_ ,				lowerCamelCase_       )

         _lowercase:			Optional[int]       =     self.get_rust_tokenizer()
         _lowercase:			List[Any]       =     tokenizer.encode(lowerCamelCase_       )
         _lowercase:			Union[str, Any]       =     rust_tokenizer.encode(lowerCamelCase_       )
         self.assertListEqual(lowerCamelCase_ ,				lowerCamelCase_       )

         # With lower casing
         _lowercase:			List[str]       =     self.get_tokenizer(do_lower_case=lowerCamelCase_       )
         _lowercase:			Optional[Any]       =     self.get_rust_tokenizer(do_lower_case=lowerCamelCase_       )

         _lowercase:			List[Any]       =     """UNwant\u00E9d,running"""

         _lowercase:			Tuple       =     tokenizer.tokenize(lowerCamelCase_       )
         _lowercase:			Any       =     rust_tokenizer.tokenize(lowerCamelCase_       )
         self.assertListEqual(lowerCamelCase_ ,				lowerCamelCase_       )

         _lowercase:			Tuple       =     tokenizer.encode(lowerCamelCase_ ,				add_special_tokens=lowerCamelCase_       )
         _lowercase:			Union[str, Any]       =     rust_tokenizer.encode(lowerCamelCase_ ,				add_special_tokens=lowerCamelCase_       )
         self.assertListEqual(lowerCamelCase_ ,				lowerCamelCase_       )

         _lowercase:			int       =     self.get_rust_tokenizer()
         _lowercase:			Dict       =     tokenizer.encode(lowerCamelCase_       )
         _lowercase:			int       =     rust_tokenizer.encode(lowerCamelCase_       )
         self.assertListEqual(lowerCamelCase_ ,				lowerCamelCase_       )
       def 							lowercase_							(     self       )     ->	Optional[int]:





         """simple docstring"""




         _lowercase:			Optional[int]       =     BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''       ) ,				['''ah''', '''\u535A''', '''\u63A8''', '''zz''']       )
       def 							lowercase_							(     self       )     ->	Any:





         """simple docstring"""




         _lowercase:			List[Any]       =     BasicTokenizer(do_lower_case=lowerCamelCase_       )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''       ) ,				['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       ) ,				['''hello''']       )
       def 							lowercase_							(     self       )     ->	List[str]:





         """simple docstring"""




         _lowercase:			List[str]       =     BasicTokenizer(do_lower_case=lowerCamelCase_ ,				strip_accents=lowerCamelCase_       )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       ) ,				['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       ) ,				['''h\u00E9llo''']       )
       def 							lowercase_							(     self       )     ->	Dict:





         """simple docstring"""




         _lowercase:			Tuple       =     BasicTokenizer(do_lower_case=lowerCamelCase_ ,				strip_accents=lowerCamelCase_       )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       ) ,				['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       ) ,				['''hello''']       )
       def 							lowercase_							(     self       )     ->	Optional[int]:





         """simple docstring"""




         _lowercase:			int       =     BasicTokenizer(do_lower_case=lowerCamelCase_       )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       ) ,				['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']       )
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''       ) ,				['''hello''']       )
       def 							lowercase_							(     self       )     ->	str:





         """simple docstring"""




         _lowercase:			List[str]       =     BasicTokenizer(do_lower_case=lowerCamelCase_       )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''       ) ,				['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )
       def 							lowercase_							(     self       )     ->	Optional[Any]:





         """simple docstring"""




         _lowercase:			int       =     BasicTokenizer(do_lower_case=lowerCamelCase_ ,				strip_accents=lowerCamelCase_       )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       ) ,				['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )
       def 							lowercase_							(     self       )     ->	int:





         """simple docstring"""




         _lowercase:			Dict       =     BasicTokenizer(do_lower_case=lowerCamelCase_ ,				strip_accents=lowerCamelCase_       )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''       ) ,				['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']       )
       def 							lowercase_							(     self       )     ->	List[str]:





         """simple docstring"""




         _lowercase:			List[str]       =     BasicTokenizer(do_lower_case=lowerCamelCase_ ,				never_split=['''[UNK]''']       )

         self.assertListEqual(
             tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''       ) ,				['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']       )
       def 							lowercase_							(     self       )     ->	Tuple:





         """simple docstring"""




         _lowercase:			Tuple       =     ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

         _lowercase:			List[str]       =     {}
         for i, token in enumerate(lowerCamelCase_       ):
           _lowercase:			List[str]       =     i
         _lowercase:			Union[str, Any]       =     WordpieceTokenizer(vocab=lowerCamelCase_ ,				unk_token='''[UNK]'''       )

         self.assertListEqual(tokenizer.tokenize(''''''       ) ,				[]       )

         self.assertListEqual(tokenizer.tokenize('''unwanted running'''       ) ,				['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']       )

         self.assertListEqual(tokenizer.tokenize('''unwantedX running'''       ) ,				['''[UNK]''', '''runn''', '''##ing''']       )
       def 							lowercase_							(     self       )     ->	Union[str, Any]:





         """simple docstring"""




         self.assertTrue(_is_whitespace(''' '''       )       )
         self.assertTrue(_is_whitespace('''\t'''       )       )
         self.assertTrue(_is_whitespace('''\r'''       )       )
         self.assertTrue(_is_whitespace('''\n'''       )       )
         self.assertTrue(_is_whitespace('''\u00A0'''       )       )

         self.assertFalse(_is_whitespace('''A'''       )       )
         self.assertFalse(_is_whitespace('''-'''       )       )
       def 							lowercase_							(     self       )     ->	Union[str, Any]:





         """simple docstring"""




         self.assertTrue(_is_control('''\u0005'''       )       )

         self.assertFalse(_is_control('''A'''       )       )
         self.assertFalse(_is_control(''' '''       )       )
         self.assertFalse(_is_control('''\t'''       )       )
         self.assertFalse(_is_control('''\r'''       )       )
       def 							lowercase_							(     self       )     ->	Tuple:





         """simple docstring"""




         self.assertTrue(_is_punctuation('''-'''       )       )
         self.assertTrue(_is_punctuation('''$'''       )       )
         self.assertTrue(_is_punctuation('''`'''       )       )
         self.assertTrue(_is_punctuation('''.'''       )       )

         self.assertFalse(_is_punctuation('''A'''       )       )
         self.assertFalse(_is_punctuation(''' '''       )       )
       def 							lowercase_							(     self       )     ->	Union[str, Any]:





         """simple docstring"""




         _lowercase:			int       =     self.get_tokenizer()
         _lowercase:			List[str]       =     self.get_rust_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(lowerCamelCase_       ) for t in ['''Test''', '''\xad''', '''test''']] ,				[['''[UNK]'''], [], ['''[UNK]''']]       )

         self.assertListEqual(
             [rust_tokenizer.tokenize(lowerCamelCase_       ) for t in ['''Test''', '''\xad''', '''test''']] ,				[['''[UNK]'''], [], ['''[UNK]''']]       )
       @slow
       def 							lowercase_							(     self       )     ->	Dict:





         """simple docstring"""




         _lowercase:			List[Any]       =     self.tokenizer_class.from_pretrained('''google/mobilebert-uncased'''       )

         _lowercase:			List[Any]       =     tokenizer.encode('''sequence builders''' ,				add_special_tokens=lowerCamelCase_       )
         _lowercase:			Optional[int]       =     tokenizer.encode('''multi-sequence build''' ,				add_special_tokens=lowerCamelCase_       )

         _lowercase:			Optional[Any]       =     tokenizer.build_inputs_with_special_tokens(lowerCamelCase_       )
         _lowercase:			Union[str, Any]       =     tokenizer.build_inputs_with_special_tokens(lowerCamelCase_ ,				lowerCamelCase_       )

         assert encoded_sentence == [101] + text + [102]
         assert encoded_pair == [101] + text + [102] + text_a + [102]
       def 							lowercase_							(     self       )     ->	Dict:





         """simple docstring"""




         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
             _lowercase:			Optional[Any]       =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_ ,				**lowerCamelCase_       )

             _lowercase:			Tuple       =     f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
             _lowercase:			List[Any]       =     tokenizer_r.encode_plus(
                 lowerCamelCase_ ,				return_attention_mask=lowerCamelCase_ ,				return_token_type_ids=lowerCamelCase_ ,				return_offsets_mapping=lowerCamelCase_ ,				add_special_tokens=lowerCamelCase_ ,				)

             _lowercase:			int       =     tokenizer_r.do_lower_case if hasattr(lowerCamelCase_ ,				'''do_lower_case'''       ) else False
             _lowercase:			Dict       =     (
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), """A"""),
                     ((1, 2), ""","""),
                     ((3, 5), """na"""),
                     ((5, 6), """##ï"""),
                     ((6, 8), """##ve"""),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), """Allen"""),
                     ((21, 23), """##NL"""),
                     ((23, 24), """##P"""),
                     ((25, 33), """sentence"""),
                     ((33, 34), """."""),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), """a"""),
                     ((1, 2), ""","""),
                     ((3, 8), """naive"""),
                     ((9, 15), tokenizer_r.mask_token),
                     ((16, 21), """allen"""),
                     ((21, 23), """##nl"""),
                     ((23, 24), """##p"""),
                     ((25, 33), """sentence"""),
                     ((33, 34), """."""),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results] ,				tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']       )       )
             self.assertEqual([e[0] for e in expected_results] ,				tokens['''offset_mapping''']       )



       def 							lowercase_							(     self       )     ->	List[str]:





         """simple docstring"""




         _lowercase:			List[str]       =     ["""的""", """人""", """有"""]
         _lowercase:			Tuple       =     """""".join(lowerCamelCase_       )
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
             _lowercase:			str       =     True
             _lowercase:			Optional[Any]       =     self.tokenizer_class.from_pretrained(lowerCamelCase_ ,				**lowerCamelCase_       )
             _lowercase:			str       =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_ ,				**lowerCamelCase_       )

             _lowercase:			List[Any]       =     tokenizer_p.encode(lowerCamelCase_ ,				add_special_tokens=lowerCamelCase_       )
             _lowercase:			Optional[Any]       =     tokenizer_r.encode(lowerCamelCase_ ,				add_special_tokens=lowerCamelCase_       )

             _lowercase:			List[Any]       =     tokenizer_r.convert_ids_to_tokens(lowerCamelCase_       )
             _lowercase:			Optional[Any]       =     tokenizer_p.convert_ids_to_tokens(lowerCamelCase_       )

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(lowerCamelCase_ ,				lowerCamelCase_       )
             self.assertListEqual(lowerCamelCase_ ,				lowerCamelCase_       )

             _lowercase:			Optional[int]       =     False
             _lowercase:			int       =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_ ,				**lowerCamelCase_       )
             _lowercase:			List[str]       =     self.tokenizer_class.from_pretrained(lowerCamelCase_ ,				**lowerCamelCase_       )

             _lowercase:			Any       =     tokenizer_r.encode(lowerCamelCase_ ,				add_special_tokens=lowerCamelCase_       )
             _lowercase:			List[str]       =     tokenizer_p.encode(lowerCamelCase_ ,				add_special_tokens=lowerCamelCase_       )

             _lowercase:			str       =     tokenizer_r.convert_ids_to_tokens(lowerCamelCase_       )
             _lowercase:			Any       =     tokenizer_p.convert_ids_to_tokens(lowerCamelCase_       )

             # it is expected that only the first Chinese character is not preceded by "##".
             _lowercase:			List[str]       =     [
                 f'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCamelCase_       )
             ]
             self.assertListEqual(lowerCamelCase_ ,				lowerCamelCase_       )
             self.assertListEqual(lowerCamelCase_ ,				lowerCamelCase_       )


"""simple docstring"""




from __future__ import annotations

import numpy as np




def  _lowerCAmelCase		(		_UpperCamelCase						):






  """simple docstring"""



  _lowercase ,				_lowercase:			List[str]       =     np.shape(_UpperCamelCase						)
  if rows != columns:
    _lowercase:			int       =     (
        '''\'table\' has to be of square shaped array but got a '''
        f'''{rows}x{columns} array:\n{table}'''
    )
    raise ValueError(_UpperCamelCase						)

  _lowercase:			Tuple       =     np.zeros((rows, columns)						)
  _lowercase:			Tuple       =     np.zeros((rows, columns)						)
  for i in range(_UpperCamelCase						):
    for j in range(_UpperCamelCase						):
      _lowercase:			List[Any]       =     sum(lower[i][k] * upper[k][j] for k in range(_UpperCamelCase						)						)
      if upper[j][j] == 0:
        raise ArithmeticError('''No LU decomposition exists'''						)
      _lowercase:			Any       =     (table[i][j] - total) / upper[j][j]
    _lowercase:			Any       =     1
    for j in range(_UpperCamelCase ,				_UpperCamelCase						):
      _lowercase:			Optional[int]       =     sum(lower[i][k] * upper[k][j] for k in range(_UpperCamelCase						)						)
      _lowercase:			List[str]       =     table[i][j] - total
  return lower, upper


if __name__ == "__main__":
  import doctest

  doctest.testmod()
