


'''simple docstring'''

import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class 						__SCREAMING_SNAKE_CASE					(      lowerCamelCase__ ,			unittest.TestCase ):
     UpperCAmelCase 					=				MobileBertTokenizer
     UpperCAmelCase 					=				MobileBertTokenizerFast
     UpperCAmelCase 					=				True
     UpperCAmelCase 					=				True
     UpperCAmelCase 					=				filter_non_english
     UpperCAmelCase 					=				"""google/mobilebert-uncased"""






     def  a_					(			self		)       ->    Optional[Any]:
          super().setUp()

          _a						=		[
              "[UNK]",
              "[CLS]",
              "[SEP]",
              "[PAD]",
              "[MASK]",
              "want",
              "##want",
              "##ed",
              "wa",
              "un",
              "runn",
              "##ing",
              ",",
              "low",
              "lowest",
          ]
          _a						=		os.path.join(self.tmpdirname     ,							VOCAB_FILES_NAMES["vocab_file"]		)
          with open(self.vocab_file     ,							"w"     ,							encoding="utf-8"		) as vocab_writer:
               vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)

          _a						=		[
              (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
              for tokenizer_def in self.tokenizers_list
          ]






     def  a_					(			self     ,							__UpperCamelCase		)       ->    Dict:
          _a						=		"UNwant\u00E9d,running"
          _a						=		"unwanted, running"
          return input_text, output_text






     def  a_					(			self		)       ->    List[Any]:
          _a						=		self.tokenizer_class(self.vocab_file		)

          _a						=		tokenizer.tokenize("UNwant\u00E9d,running"		)
          self.assertListEqual(__UpperCamelCase     ,							["un", "##want", "##ed", ",", "runn", "##ing"]		)
          self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase		)     ,							[9, 6, 7, 12, 10, 11]		)






     def  a_					(			self		)       ->    Any:

          if not self.test_rust_tokenizer:
               return

          _a						=		self.get_tokenizer()
          _a						=		self.get_rust_tokenizer()

          _a						=		"UNwant\u00E9d,running"

          _a						=		tokenizer.tokenize(__UpperCamelCase		)
          _a						=		rust_tokenizer.tokenize(__UpperCamelCase		)
          self.assertListEqual(__UpperCamelCase     ,							__UpperCamelCase		)

          _a						=		tokenizer.encode(__UpperCamelCase     ,							add_special_tokens=__UpperCamelCase		)
          _a						=		rust_tokenizer.encode(__UpperCamelCase     ,							add_special_tokens=__UpperCamelCase		)
          self.assertListEqual(__UpperCamelCase     ,							__UpperCamelCase		)

          _a						=		self.get_rust_tokenizer()
          _a						=		tokenizer.encode(__UpperCamelCase		)
          _a						=		rust_tokenizer.encode(__UpperCamelCase		)
          self.assertListEqual(__UpperCamelCase     ,							__UpperCamelCase		)

          # With lower casing
          _a						=		self.get_tokenizer(do_lower_case=__UpperCamelCase		)
          _a						=		self.get_rust_tokenizer(do_lower_case=__UpperCamelCase		)

          _a						=		"UNwant\u00E9d,running"

          _a						=		tokenizer.tokenize(__UpperCamelCase		)
          _a						=		rust_tokenizer.tokenize(__UpperCamelCase		)
          self.assertListEqual(__UpperCamelCase     ,							__UpperCamelCase		)

          _a						=		tokenizer.encode(__UpperCamelCase     ,							add_special_tokens=__UpperCamelCase		)
          _a						=		rust_tokenizer.encode(__UpperCamelCase     ,							add_special_tokens=__UpperCamelCase		)
          self.assertListEqual(__UpperCamelCase     ,							__UpperCamelCase		)

          _a						=		self.get_rust_tokenizer()
          _a						=		tokenizer.encode(__UpperCamelCase		)
          _a						=		rust_tokenizer.encode(__UpperCamelCase		)
          self.assertListEqual(__UpperCamelCase     ,							__UpperCamelCase		)






     def  a_					(			self		)       ->    List[Any]:
          _a						=		BasicTokenizer()

          self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"		)     ,							["ah", "\u535A", "\u63A8", "zz"]		)






     def  a_					(			self		)       ->    Optional[int]:
          _a						=		BasicTokenizer(do_lower_case=__UpperCamelCase		)

          self.assertListEqual(
              tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)     ,							["hello", "!", "how", "are", "you", "?"]		)
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)     ,							["hello"]		)






     def  a_					(			self		)       ->    Dict:
          _a						=		BasicTokenizer(do_lower_case=__UpperCamelCase     ,							strip_accents=__UpperCamelCase		)

          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)     ,							["hällo", "!", "how", "are", "you", "?"]		)
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)     ,							["h\u00E9llo"]		)






     def  a_					(			self		)       ->    int:
          _a						=		BasicTokenizer(do_lower_case=__UpperCamelCase     ,							strip_accents=__UpperCamelCase		)

          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)     ,							["hallo", "!", "how", "are", "you", "?"]		)
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)     ,							["hello"]		)






     def  a_					(			self		)       ->    Tuple:
          _a						=		BasicTokenizer(do_lower_case=__UpperCamelCase		)

          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)     ,							["hallo", "!", "how", "are", "you", "?"]		)
          self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)     ,							["hello"]		)






     def  a_					(			self		)       ->    Union[str, Any]:
          _a						=		BasicTokenizer(do_lower_case=__UpperCamelCase		)

          self.assertListEqual(
              tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)     ,							["HeLLo", "!", "how", "Are", "yoU", "?"]		)






     def  a_					(			self		)       ->    List[Any]:
          _a						=		BasicTokenizer(do_lower_case=__UpperCamelCase     ,							strip_accents=__UpperCamelCase		)

          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)     ,							["HäLLo", "!", "how", "Are", "yoU", "?"]		)






     def  a_					(			self		)       ->    Any:
          _a						=		BasicTokenizer(do_lower_case=__UpperCamelCase     ,							strip_accents=__UpperCamelCase		)

          self.assertListEqual(
              tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)     ,							["HaLLo", "!", "how", "Are", "yoU", "?"]		)






     def  a_					(			self		)       ->    Optional[int]:
          _a						=		BasicTokenizer(do_lower_case=__UpperCamelCase     ,							never_split=["[UNK]"]		)

          self.assertListEqual(
              tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"		)     ,							["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]		)






     def  a_					(			self		)       ->    Optional[Any]:
          _a						=		["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

          _a						=		{}
          for i, token in enumerate(__UpperCamelCase		):
               _a						=		i
          _a						=		WordpieceTokenizer(vocab=__UpperCamelCase     ,							unk_token="[UNK]"		)

          self.assertListEqual(tokenizer.tokenize(""		)     ,							[]		)

          self.assertListEqual(tokenizer.tokenize("unwanted running"		)     ,							["un", "##want", "##ed", "runn", "##ing"]		)

          self.assertListEqual(tokenizer.tokenize("unwantedX running"		)     ,							["[UNK]", "runn", "##ing"]		)






     def  a_					(			self		)       ->    int:
          self.assertTrue(_is_whitespace(" "		)		)
          self.assertTrue(_is_whitespace("\t"		)		)
          self.assertTrue(_is_whitespace("\r"		)		)
          self.assertTrue(_is_whitespace("\n"		)		)
          self.assertTrue(_is_whitespace("\u00A0"		)		)

          self.assertFalse(_is_whitespace("A"		)		)
          self.assertFalse(_is_whitespace("-"		)		)






     def  a_					(			self		)       ->    Union[str, Any]:
          self.assertTrue(_is_control("\u0005"		)		)

          self.assertFalse(_is_control("A"		)		)
          self.assertFalse(_is_control(" "		)		)
          self.assertFalse(_is_control("\t"		)		)
          self.assertFalse(_is_control("\r"		)		)






     def  a_					(			self		)       ->    Dict:
          self.assertTrue(_is_punctuation("-"		)		)
          self.assertTrue(_is_punctuation("$"		)		)
          self.assertTrue(_is_punctuation("`"		)		)
          self.assertTrue(_is_punctuation("."		)		)

          self.assertFalse(_is_punctuation("A"		)		)
          self.assertFalse(_is_punctuation(" "		)		)






     def  a_					(			self		)       ->    Optional[Any]:
          _a						=		self.get_tokenizer()
          _a						=		self.get_rust_tokenizer()

          # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
          self.assertListEqual([tokenizer.tokenize(__UpperCamelCase		) for t in ["Test", "\xad", "test"]]     ,							[["[UNK]"], [], ["[UNK]"]]		)

          self.assertListEqual(
              [rust_tokenizer.tokenize(__UpperCamelCase		) for t in ["Test", "\xad", "test"]]     ,							[["[UNK]"], [], ["[UNK]"]]		)






     @slow
     def  a_					(			self		)       ->    Any:
          _a						=		self.tokenizer_class.from_pretrained("google/mobilebert-uncased"		)

          _a						=		tokenizer.encode("sequence builders"     ,							add_special_tokens=__UpperCamelCase		)
          _a						=		tokenizer.encode("multi-sequence build"     ,							add_special_tokens=__UpperCamelCase		)

          _a						=		tokenizer.build_inputs_with_special_tokens(__UpperCamelCase		)
          _a						=		tokenizer.build_inputs_with_special_tokens(__UpperCamelCase     ,							__UpperCamelCase		)

          assert encoded_sentence == [101] + text + [102]
          assert encoded_pair == [101] + text + [102] + text_a + [102]






     def  a_					(			self		)       ->    List[Any]:

          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                    _a						=		self.rust_tokenizer_class.from_pretrained(__UpperCamelCase     ,							**__UpperCamelCase		)

                    _a						=		f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
                    _a						=		tokenizer_r.encode_plus(
                        __UpperCamelCase     ,							return_attention_mask=__UpperCamelCase     ,							return_token_type_ids=__UpperCamelCase     ,							return_offsets_mapping=__UpperCamelCase     ,							add_special_tokens=__UpperCamelCase     ,							)

                    _a						=		tokenizer_r.do_lower_case if hasattr(__UpperCamelCase     ,							"do_lower_case"		) else False
                    _a						=		(
                        [
                            ((0, 0), tokenizer_r.cls_token),
                            ((0, 1), "A"),
                            ((1, 2), ","),
                            ((3, 5), "na"),
                            ((5, 6), "##ï"),
                            ((6, 8), "##ve"),
                            ((9, 15), tokenizer_r.mask_token),
                            ((16, 21), "Allen"),
                            ((21, 23), "##NL"),
                            ((23, 24), "##P"),
                            ((25, 33), "sentence"),
                            ((33, 34), "."),
                            ((0, 0), tokenizer_r.sep_token),
                        ]
                        if not do_lower_case
                        else [
                            ((0, 0), tokenizer_r.cls_token),
                            ((0, 1), "a"),
                            ((1, 2), ","),
                            ((3, 8), "naive"),
                            ((9, 15), tokenizer_r.mask_token),
                            ((16, 21), "allen"),
                            ((21, 23), "##nl"),
                            ((23, 24), "##p"),
                            ((25, 33), "sentence"),
                            ((33, 34), "."),
                            ((0, 0), tokenizer_r.sep_token),
                        ]
                    )

                    self.assertEqual(
                        [e[1] for e in expected_results]     ,							tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]		)		)
                    self.assertEqual([e[0] for e in expected_results]     ,							tokens["offset_mapping"]		)






     def  a_					(			self		)       ->    int:
          _a						=		["的", "人", "有"]
          _a						=		"".join(__UpperCamelCase		)
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                    _a						=		True
                    _a						=		self.tokenizer_class.from_pretrained(__UpperCamelCase     ,							**__UpperCamelCase		)
                    _a						=		self.rust_tokenizer_class.from_pretrained(__UpperCamelCase     ,							**__UpperCamelCase		)

                    _a						=		tokenizer_p.encode(__UpperCamelCase     ,							add_special_tokens=__UpperCamelCase		)
                    _a						=		tokenizer_r.encode(__UpperCamelCase     ,							add_special_tokens=__UpperCamelCase		)

                    _a						=		tokenizer_r.convert_ids_to_tokens(__UpperCamelCase		)
                    _a						=		tokenizer_p.convert_ids_to_tokens(__UpperCamelCase		)

                    # it is expected that each Chinese character is not preceded by "##"
                    self.assertListEqual(__UpperCamelCase     ,							__UpperCamelCase		)
                    self.assertListEqual(__UpperCamelCase     ,							__UpperCamelCase		)

                    _a						=		False
                    _a						=		self.rust_tokenizer_class.from_pretrained(__UpperCamelCase     ,							**__UpperCamelCase		)
                    _a						=		self.tokenizer_class.from_pretrained(__UpperCamelCase     ,							**__UpperCamelCase		)

                    _a						=		tokenizer_r.encode(__UpperCamelCase     ,							add_special_tokens=__UpperCamelCase		)
                    _a						=		tokenizer_p.encode(__UpperCamelCase     ,							add_special_tokens=__UpperCamelCase		)

                    _a						=		tokenizer_r.convert_ids_to_tokens(__UpperCamelCase		)
                    _a						=		tokenizer_p.convert_ids_to_tokens(__UpperCamelCase		)

                    # it is expected that only the first Chinese character is not preceded by "##".
                    _a						=		[
                        f"##{token}" if idx != 0 else token for idx, token in enumerate(__UpperCamelCase		)
                    ]
                    self.assertListEqual(__UpperCamelCase     ,							__UpperCamelCase		)
                    self.assertListEqual(__UpperCamelCase     ,							__UpperCamelCase		)





'''simple docstring'''

lowercase__				     =		256
# Modulus to hash a string
lowercase__				     =		1_000_003






def __UpperCamelCase   (			__lowerCamelCase		: str      ,  __lowerCamelCase		: str				)      ->     bool:







     '''simple docstring'''






     _a						=		len(__lowerCamelCase				)
     _a						=		len(__lowerCamelCase				)
     if p_len > t_len:
          return False

     _a						=		0
     _a						=		0
     _a						=		1

     # Calculating the hash of pattern and substring of text
     for i in range(__lowerCamelCase				):
          _a						=		(ord(pattern[i]				) + p_hash * alphabet_size) % modulus
          _a						=		(ord(text[i]				) + text_hash * alphabet_size) % modulus
          if i == p_len - 1:
               continue
          _a						=		(modulus_power * alphabet_size) % modulus

     for i in range(0      ,  t_len - p_len + 1				):
          if text_hash == p_hash and text[i : i + p_len] == pattern:
               return True
          if i == t_len - p_len:
               continue
          # Calculate the https://en.wikipedia.org/wiki/Rolling_hash
          _a						=		(
              (text_hash - ord(text[i]				) * modulus_power) * alphabet_size
              + ord(text[i + p_len]				)
          ) % modulus
     return False






def __UpperCamelCase   (			)      ->     None:







     '''simple docstring'''






     _a						=		"abc1abc12"
     _a						=		"alskfjaldsabc1abc1abc12k23adsfabcabc"
     _a						=		"alskfjaldsk23adsfabcabc"
     assert rabin_karp(__lowerCamelCase      ,  __lowerCamelCase				) and not rabin_karp(__lowerCamelCase      ,  __lowerCamelCase				)

     # Test 2)
     _a						=		"ABABX"
     _a						=		"ABABZABABYABABX"
     assert rabin_karp(__lowerCamelCase      ,  __lowerCamelCase				)

     # Test 3)
     _a						=		"AAAB"
     _a						=		"ABAAAAAB"
     assert rabin_karp(__lowerCamelCase      ,  __lowerCamelCase				)

     # Test 4)
     _a						=		"abcdabcy"
     _a						=		"abcxabcdabxabcdabcdabcy"
     assert rabin_karp(__lowerCamelCase      ,  __lowerCamelCase				)

     # Test 5)
     _a						=		"Lü"
     _a						=		"Lüsai"
     assert rabin_karp(__lowerCamelCase      ,  __lowerCamelCase				)
     _a						=		"Lue"
     assert not rabin_karp(__lowerCamelCase      ,  __lowerCamelCase				)
     print("Success."				)


if __name__ == "__main__":
   test_rabin_karp()

