




import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class 					__lowerCamelCase			(       UpperCamelCase__      ,							unittest.TestCase       ):




    """simple docstring"""

    snake_case__   = MobileBertTokenizer
    snake_case__   = MobileBertTokenizerFast
    snake_case__   = True
    snake_case__   = True
    snake_case__   = filter_non_english
    snake_case__   = "google/mobilebert-uncased"


    def   a (			self	:	Optional[Any]  ) ->			List[str]:
         super().setUp()

         lowerCAmelCase__            =					[
             "[UNK]",
             "[CLS]",
             "[SEP]",
             "[PAD]",
             "[MASK]",
             "want",
             "##want",
             "##ed",
             "wa",
             "un",
             "runn",
             "##ing",
             ",",
             "low",
             "lowest",
         ]
         lowerCAmelCase__            =					os.path.join(self.tmpdirname  ,      VOCAB_FILES_NAMES["vocab_file"]  )
         with open(self.vocab_file  ,      "w"  ,      encoding="utf-8"  ) as vocab_writer:
              vocab_writer.write("".join([x + "\n" for x in vocab_tokens]  )  )

         lowerCAmelCase__            =					[
             (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
             for tokenizer_def in self.tokenizers_list
         ]


    def   a (			self	:	Optional[int]  ,      SCREAMING_SNAKE_CASE__	:	Optional[int]  ) ->			str:
         lowerCAmelCase__            =					"UNwant\u00E9d,running"
         lowerCAmelCase__            =					"unwanted, running"
         return input_text, output_text


    def   a (			self	:	Union[str, Any]  ) ->			Optional[int]:
         lowerCAmelCase__            =					self.tokenizer_class(self.vocab_file  )

         lowerCAmelCase__            =					tokenizer.tokenize("UNwant\u00E9d,running"  )
         self.assertListEqual(SCREAMING_SNAKE_CASE__  ,      ["un", "##want", "##ed", ",", "runn", "##ing"]  )
         self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__  )  ,      [9, 6, 7, 12, 10, 11]  )


    def   a (			self	:	List[str]  ) ->			Union[str, Any]:
         if not self.test_rust_tokenizer:
              return

         lowerCAmelCase__            =					self.get_tokenizer()
         lowerCAmelCase__            =					self.get_rust_tokenizer()

         lowerCAmelCase__            =					"UNwant\u00E9d,running"

         lowerCAmelCase__            =					tokenizer.tokenize(SCREAMING_SNAKE_CASE__  )
         lowerCAmelCase__            =					rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE__  )
         self.assertListEqual(SCREAMING_SNAKE_CASE__  ,      SCREAMING_SNAKE_CASE__  )

         lowerCAmelCase__            =					tokenizer.encode(SCREAMING_SNAKE_CASE__  ,      add_special_tokens=SCREAMING_SNAKE_CASE__  )
         lowerCAmelCase__            =					rust_tokenizer.encode(SCREAMING_SNAKE_CASE__  ,      add_special_tokens=SCREAMING_SNAKE_CASE__  )
         self.assertListEqual(SCREAMING_SNAKE_CASE__  ,      SCREAMING_SNAKE_CASE__  )

         lowerCAmelCase__            =					self.get_rust_tokenizer()
         lowerCAmelCase__            =					tokenizer.encode(SCREAMING_SNAKE_CASE__  )
         lowerCAmelCase__            =					rust_tokenizer.encode(SCREAMING_SNAKE_CASE__  )
         self.assertListEqual(SCREAMING_SNAKE_CASE__  ,      SCREAMING_SNAKE_CASE__  )

         # With lower casing
         lowerCAmelCase__            =					self.get_tokenizer(do_lower_case=SCREAMING_SNAKE_CASE__  )
         lowerCAmelCase__            =					self.get_rust_tokenizer(do_lower_case=SCREAMING_SNAKE_CASE__  )

         lowerCAmelCase__            =					"UNwant\u00E9d,running"

         lowerCAmelCase__            =					tokenizer.tokenize(SCREAMING_SNAKE_CASE__  )
         lowerCAmelCase__            =					rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE__  )
         self.assertListEqual(SCREAMING_SNAKE_CASE__  ,      SCREAMING_SNAKE_CASE__  )

         lowerCAmelCase__            =					tokenizer.encode(SCREAMING_SNAKE_CASE__  ,      add_special_tokens=SCREAMING_SNAKE_CASE__  )
         lowerCAmelCase__            =					rust_tokenizer.encode(SCREAMING_SNAKE_CASE__  ,      add_special_tokens=SCREAMING_SNAKE_CASE__  )
         self.assertListEqual(SCREAMING_SNAKE_CASE__  ,      SCREAMING_SNAKE_CASE__  )

         lowerCAmelCase__            =					self.get_rust_tokenizer()
         lowerCAmelCase__            =					tokenizer.encode(SCREAMING_SNAKE_CASE__  )
         lowerCAmelCase__            =					rust_tokenizer.encode(SCREAMING_SNAKE_CASE__  )
         self.assertListEqual(SCREAMING_SNAKE_CASE__  ,      SCREAMING_SNAKE_CASE__  )


    def   a (			self	:	List[str]  ) ->			Any:
         lowerCAmelCase__            =					BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"  )  ,      ["ah", "\u535A", "\u63A8", "zz"]  )


    def   a (			self	:	Tuple  ) ->			Union[str, Any]:
         lowerCAmelCase__            =					BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__  )

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "  )  ,      ["hello", "!", "how", "are", "you", "?"]  )
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  )  ,      ["hello"]  )


    def   a (			self	:	Optional[int]  ) ->			Union[str, Any]:
         lowerCAmelCase__            =					BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__  ,      strip_accents=SCREAMING_SNAKE_CASE__  )

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  )  ,      ["hällo", "!", "how", "are", "you", "?"]  )
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  )  ,      ["h\u00E9llo"]  )


    def   a (			self	:	int  ) ->			List[str]:
         lowerCAmelCase__            =					BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__  ,      strip_accents=SCREAMING_SNAKE_CASE__  )

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  )  ,      ["hallo", "!", "how", "are", "you", "?"]  )
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  )  ,      ["hello"]  )


    def   a (			self	:	List[Any]  ) ->			Optional[Any]:
         lowerCAmelCase__            =					BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__  )

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  )  ,      ["hallo", "!", "how", "are", "you", "?"]  )
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  )  ,      ["hello"]  )


    def   a (			self	:	int  ) ->			Union[str, Any]:
         lowerCAmelCase__            =					BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__  )

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "  )  ,      ["HeLLo", "!", "how", "Are", "yoU", "?"]  )


    def   a (			self	:	List[Any]  ) ->			Dict:
         lowerCAmelCase__            =					BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__  ,      strip_accents=SCREAMING_SNAKE_CASE__  )

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  )  ,      ["HäLLo", "!", "how", "Are", "yoU", "?"]  )


    def   a (			self	:	Dict  ) ->			Any:
         lowerCAmelCase__            =					BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__  ,      strip_accents=SCREAMING_SNAKE_CASE__  )

         self.assertListEqual(
             tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  )  ,      ["HaLLo", "!", "how", "Are", "yoU", "?"]  )


    def   a (			self	:	Dict  ) ->			List[str]:
         lowerCAmelCase__            =					BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__  ,      never_split=["[UNK]"]  )

         self.assertListEqual(
             tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"  )  ,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]  )


    def   a (			self	:	Optional[int]  ) ->			int:
         lowerCAmelCase__            =					["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

         lowerCAmelCase__            =					{}
         for i, token in enumerate(SCREAMING_SNAKE_CASE__  ):
              lowerCAmelCase__            =					i
         lowerCAmelCase__            =					WordpieceTokenizer(vocab=SCREAMING_SNAKE_CASE__  ,      unk_token="[UNK]"  )

         self.assertListEqual(tokenizer.tokenize(""  )  ,      []  )

         self.assertListEqual(tokenizer.tokenize("unwanted running"  )  ,      ["un", "##want", "##ed", "runn", "##ing"]  )

         self.assertListEqual(tokenizer.tokenize("unwantedX running"  )  ,      ["[UNK]", "runn", "##ing"]  )


    def   a (			self	:	str  ) ->			Dict:
         self.assertTrue(_is_whitespace(" "  )  )
         self.assertTrue(_is_whitespace("\t"  )  )
         self.assertTrue(_is_whitespace("\r"  )  )
         self.assertTrue(_is_whitespace("\n"  )  )
         self.assertTrue(_is_whitespace("\u00A0"  )  )

         self.assertFalse(_is_whitespace("A"  )  )
         self.assertFalse(_is_whitespace("-"  )  )


    def   a (			self	:	str  ) ->			List[str]:
         self.assertTrue(_is_control("\u0005"  )  )

         self.assertFalse(_is_control("A"  )  )
         self.assertFalse(_is_control(" "  )  )
         self.assertFalse(_is_control("\t"  )  )
         self.assertFalse(_is_control("\r"  )  )


    def   a (			self	:	Optional[Any]  ) ->			Dict:
         self.assertTrue(_is_punctuation("-"  )  )
         self.assertTrue(_is_punctuation("$"  )  )
         self.assertTrue(_is_punctuation("`"  )  )
         self.assertTrue(_is_punctuation("."  )  )

         self.assertFalse(_is_punctuation("A"  )  )
         self.assertFalse(_is_punctuation(" "  )  )


    def   a (			self	:	Union[str, Any]  ) ->			Tuple:
         lowerCAmelCase__            =					self.get_tokenizer()
         lowerCAmelCase__            =					self.get_rust_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(SCREAMING_SNAKE_CASE__  ) for t in ["Test", "\xad", "test"]]  ,      [["[UNK]"], [], ["[UNK]"]]  )

         self.assertListEqual(
             [rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE__  ) for t in ["Test", "\xad", "test"]]  ,      [["[UNK]"], [], ["[UNK]"]]  )


    @slow
    def   a (			self	:	List[Any]  ) ->			Tuple:
         lowerCAmelCase__            =					self.tokenizer_class.from_pretrained("google/mobilebert-uncased"  )

         lowerCAmelCase__            =					tokenizer.encode("sequence builders"  ,      add_special_tokens=SCREAMING_SNAKE_CASE__  )
         lowerCAmelCase__            =					tokenizer.encode("multi-sequence build"  ,      add_special_tokens=SCREAMING_SNAKE_CASE__  )

         lowerCAmelCase__            =					tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE__  )
         lowerCAmelCase__            =					tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE__  ,      SCREAMING_SNAKE_CASE__  )

         assert encoded_sentence == [101] + text + [102]
         assert encoded_pair == [101] + text + [102] + text_a + [102]


    def   a (			self	:	Dict  ) ->			List[str]:
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
                   lowerCAmelCase__            =					self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__  ,      **SCREAMING_SNAKE_CASE__  )

                   lowerCAmelCase__            =					f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
                   lowerCAmelCase__            =					tokenizer_r.encode_plus(
                       SCREAMING_SNAKE_CASE__  ,      return_attention_mask=SCREAMING_SNAKE_CASE__  ,      return_token_type_ids=SCREAMING_SNAKE_CASE__  ,      return_offsets_mapping=SCREAMING_SNAKE_CASE__  ,      add_special_tokens=SCREAMING_SNAKE_CASE__  ,      )

                   lowerCAmelCase__            =					tokenizer_r.do_lower_case if hasattr(SCREAMING_SNAKE_CASE__  ,      "do_lower_case"  ) else False
                   lowerCAmelCase__            =					(
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), "A"),
                           ((1, 2), ","),
                           ((3, 5), "na"),
                           ((5, 6), "##ï"),
                           ((6, 8), "##ve"),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), "Allen"),
                           ((21, 23), "##NL"),
                           ((23, 24), "##P"),
                           ((25, 33), "sentence"),
                           ((33, 34), "."),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), "a"),
                           ((1, 2), ","),
                           ((3, 8), "naive"),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), "allen"),
                           ((21, 23), "##nl"),
                           ((23, 24), "##p"),
                           ((25, 33), "sentence"),
                           ((33, 34), "."),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]  ,      tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]  )  )
                   self.assertEqual([e[0] for e in expected_results]  ,      tokens["offset_mapping"]  )


    def   a (			self	:	Tuple  ) ->			Dict:
         lowerCAmelCase__            =					["的", "人", "有"]
         lowerCAmelCase__            =					"".join(SCREAMING_SNAKE_CASE__  )
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
                   lowerCAmelCase__            =					True
                   lowerCAmelCase__            =					self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__  ,      **SCREAMING_SNAKE_CASE__  )
                   lowerCAmelCase__            =					self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__  ,      **SCREAMING_SNAKE_CASE__  )

                   lowerCAmelCase__            =					tokenizer_p.encode(SCREAMING_SNAKE_CASE__  ,      add_special_tokens=SCREAMING_SNAKE_CASE__  )
                   lowerCAmelCase__            =					tokenizer_r.encode(SCREAMING_SNAKE_CASE__  ,      add_special_tokens=SCREAMING_SNAKE_CASE__  )

                   lowerCAmelCase__            =					tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__  )
                   lowerCAmelCase__            =					tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__  )

                   # it is expected that each Chinese character is not preceded by "##"
                   self.assertListEqual(SCREAMING_SNAKE_CASE__  ,      SCREAMING_SNAKE_CASE__  )
                   self.assertListEqual(SCREAMING_SNAKE_CASE__  ,      SCREAMING_SNAKE_CASE__  )

                   lowerCAmelCase__            =					False
                   lowerCAmelCase__            =					self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__  ,      **SCREAMING_SNAKE_CASE__  )
                   lowerCAmelCase__            =					self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__  ,      **SCREAMING_SNAKE_CASE__  )

                   lowerCAmelCase__            =					tokenizer_r.encode(SCREAMING_SNAKE_CASE__  ,      add_special_tokens=SCREAMING_SNAKE_CASE__  )
                   lowerCAmelCase__            =					tokenizer_p.encode(SCREAMING_SNAKE_CASE__  ,      add_special_tokens=SCREAMING_SNAKE_CASE__  )

                   lowerCAmelCase__            =					tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__  )
                   lowerCAmelCase__            =					tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__  )

                   # it is expected that only the first Chinese character is not preceded by "##".
                   lowerCAmelCase__            =					[
                       f'##{token}' if idx != 0 else token for idx, token in enumerate(SCREAMING_SNAKE_CASE__  )
                   ]
                   self.assertListEqual(SCREAMING_SNAKE_CASE__  ,      SCREAMING_SNAKE_CASE__  )
                   self.assertListEqual(SCREAMING_SNAKE_CASE__  ,      SCREAMING_SNAKE_CASE__  )


lowerCamelCase__     :		List[str]        = """
# Installazione di Transformers
! pip install transformers datasets
# Per installare dalla fonte invece dell'ultima versione rilasciata, commenta il comando sopra e
# rimuovi la modalità commento al comando seguente.
# ! pip install git+https://github.com/huggingface/transformers.git
"""

lowerCamelCase__     :		List[Any]        = [{"""type""": """code""", """content""": INSTALL_CONTENT}]
lowerCamelCase__     :		int        = {
    """{processor_class}""": """FakeProcessorClass""",
    """{model_class}""": """FakeModelClass""",
    """{object_class}""": """FakeObjectClass""",
}






