

"""simple docstring"""






import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class       snake_case__  ( lowerCAmelCase_					,  unittest.TestCase     ):
			SCREAMING_SNAKE_CASE__     				=						MobileBertTokenizer
			SCREAMING_SNAKE_CASE__     				=						MobileBertTokenizerFast
			SCREAMING_SNAKE_CASE__     				=						True
			SCREAMING_SNAKE_CASE__     				=						True
			SCREAMING_SNAKE_CASE__     				=						filter_non_english
			SCREAMING_SNAKE_CASE__     				=						'''google/mobilebert-uncased'''







			def 			__lowerCAmelCase				(			self			: Tuple  ):





								'''simple docstring'''



								super().setUp()

								UpperCAmelCase			:   List[str]      =				[
								    "[UNK]",
								    "[CLS]",
								    "[SEP]",
								    "[PAD]",
								    "[MASK]",
								    "want",
								    "##want",
								    "##ed",
								    "wa",
								    "un",
								    "runn",
								    "##ing",
								    ",",
								    "low",
								    "lowest",
								]
								UpperCAmelCase			:   Optional[Any]      =				os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES["vocab_file"]  )
								with open(self.vocab_file			,  "w"			,  encoding="utf-8"  ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in vocab_tokens]  )  )

								UpperCAmelCase			:   List[Any]      =				[
								    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
								    for tokenizer_def in self.tokenizers_list
								]







			def 			__lowerCAmelCase				(			self			: Union[str, Any]			,  lowercase			: Any  ):





								'''simple docstring'''



								UpperCAmelCase			:   Dict      =				"UNwant\u00E9d,running"
								UpperCAmelCase			:   Dict      =				"unwanted, running"
								return input_text, output_text







			def 			__lowerCAmelCase				(			self			: str  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[Any]      =				self.tokenizer_class(self.vocab_file  )

								UpperCAmelCase			:   Dict      =				tokenizer.tokenize("UNwant\u00E9d,running"  )
								self.assertListEqual(lowercase			,  ["un", "##want", "##ed", ",", "runn", "##ing"]  )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase  )			,  [9, 6, 7, 12, 10, 11]  )







			def 			__lowerCAmelCase				(			self			: Dict  ):





								'''simple docstring'''



								if not self.test_rust_tokenizer:
													return

								UpperCAmelCase			:   Dict      =				self.get_tokenizer()
								UpperCAmelCase			:   Union[str, Any]      =				self.get_rust_tokenizer()

								UpperCAmelCase			:   Dict      =				"UNwant\u00E9d,running"

								UpperCAmelCase			:   int      =				tokenizer.tokenize(lowercase  )
								UpperCAmelCase			:   List[Any]      =				rust_tokenizer.tokenize(lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )

								UpperCAmelCase			:   List[str]      =				tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )
								UpperCAmelCase			:   Optional[Any]      =				rust_tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )

								UpperCAmelCase			:   Optional[Any]      =				self.get_rust_tokenizer()
								UpperCAmelCase			:   Union[str, Any]      =				tokenizer.encode(lowercase  )
								UpperCAmelCase			:   Tuple      =				rust_tokenizer.encode(lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )

								# With lower casing
								UpperCAmelCase			:   Dict      =				self.get_tokenizer(do_lower_case=lowercase  )
								UpperCAmelCase			:   Dict      =				self.get_rust_tokenizer(do_lower_case=lowercase  )

								UpperCAmelCase			:   int      =				"UNwant\u00E9d,running"

								UpperCAmelCase			:   Any      =				tokenizer.tokenize(lowercase  )
								UpperCAmelCase			:   List[str]      =				rust_tokenizer.tokenize(lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )

								UpperCAmelCase			:   str      =				tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )
								UpperCAmelCase			:   Optional[int]      =				rust_tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )

								UpperCAmelCase			:   Tuple      =				self.get_rust_tokenizer()
								UpperCAmelCase			:   str      =				tokenizer.encode(lowercase  )
								UpperCAmelCase			:   Optional[Any]      =				rust_tokenizer.encode(lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )







			def 			__lowerCAmelCase				(			self			: Union[str, Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[str]      =				BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"  )			,  ["ah", "\u535A", "\u63A8", "zz"]  )







			def 			__lowerCAmelCase				(			self			: Optional[int]  ):





								'''simple docstring'''



								UpperCAmelCase			:   int      =				BasicTokenizer(do_lower_case=lowercase  )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "  )			,  ["hello", "!", "how", "are", "you", "?"]  )
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  )			,  ["hello"]  )







			def 			__lowerCAmelCase				(			self			: List[str]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Tuple      =				BasicTokenizer(do_lower_case=lowercase			,  strip_accents=lowercase  )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  )			,  ["hällo", "!", "how", "are", "you", "?"]  )
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  )			,  ["h\u00E9llo"]  )







			def 			__lowerCAmelCase				(			self			: Union[str, Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   int      =				BasicTokenizer(do_lower_case=lowercase			,  strip_accents=lowercase  )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  )			,  ["hallo", "!", "how", "are", "you", "?"]  )
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  )			,  ["hello"]  )







			def 			__lowerCAmelCase				(			self			: str  ):





								'''simple docstring'''



								UpperCAmelCase			:   Any      =				BasicTokenizer(do_lower_case=lowercase  )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  )			,  ["hallo", "!", "how", "are", "you", "?"]  )
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  )			,  ["hello"]  )







			def 			__lowerCAmelCase				(			self			: Optional[Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[Any]      =				BasicTokenizer(do_lower_case=lowercase  )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "  )			,  ["HeLLo", "!", "how", "Are", "yoU", "?"]  )







			def 			__lowerCAmelCase				(			self			: int  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[int]      =				BasicTokenizer(do_lower_case=lowercase			,  strip_accents=lowercase  )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  )			,  ["HäLLo", "!", "how", "Are", "yoU", "?"]  )







			def 			__lowerCAmelCase				(			self			: Union[str, Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[Any]      =				BasicTokenizer(do_lower_case=lowercase			,  strip_accents=lowercase  )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "  )			,  ["HaLLo", "!", "how", "Are", "yoU", "?"]  )







			def 			__lowerCAmelCase				(			self			: List[str]  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[Any]      =				BasicTokenizer(do_lower_case=lowercase			,  never_split=["[UNK]"]  )

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"  )			,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]  )







			def 			__lowerCAmelCase				(			self			: List[Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[Any]      =				["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

								UpperCAmelCase			:   Any      =				{}
								for i, token in enumerate(lowercase  ):
													UpperCAmelCase			:   int      =				i
								UpperCAmelCase			:   int      =				WordpieceTokenizer(vocab=lowercase			,  unk_token="[UNK]"  )

								self.assertListEqual(tokenizer.tokenize(""  )			,  []  )

								self.assertListEqual(tokenizer.tokenize("unwanted running"  )			,  ["un", "##want", "##ed", "runn", "##ing"]  )

								self.assertListEqual(tokenizer.tokenize("unwantedX running"  )			,  ["[UNK]", "runn", "##ing"]  )







			def 			__lowerCAmelCase				(			self			: Tuple  ):





								'''simple docstring'''



								self.assertTrue(_is_whitespace(" "  )  )
								self.assertTrue(_is_whitespace("\t"  )  )
								self.assertTrue(_is_whitespace("\r"  )  )
								self.assertTrue(_is_whitespace("\n"  )  )
								self.assertTrue(_is_whitespace("\u00A0"  )  )

								self.assertFalse(_is_whitespace("A"  )  )
								self.assertFalse(_is_whitespace("-"  )  )







			def 			__lowerCAmelCase				(			self			: Dict  ):





								'''simple docstring'''



								self.assertTrue(_is_control("\u0005"  )  )

								self.assertFalse(_is_control("A"  )  )
								self.assertFalse(_is_control(" "  )  )
								self.assertFalse(_is_control("\t"  )  )
								self.assertFalse(_is_control("\r"  )  )







			def 			__lowerCAmelCase				(			self			: int  ):





								'''simple docstring'''



								self.assertTrue(_is_punctuation("-"  )  )
								self.assertTrue(_is_punctuation("$"  )  )
								self.assertTrue(_is_punctuation("`"  )  )
								self.assertTrue(_is_punctuation("."  )  )

								self.assertFalse(_is_punctuation("A"  )  )
								self.assertFalse(_is_punctuation(" "  )  )







			def 			__lowerCAmelCase				(			self			: List[str]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[Any]      =				self.get_tokenizer()
								UpperCAmelCase			:   Dict      =				self.get_rust_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(lowercase  ) for t in ["Test", "\xad", "test"]]			,  [["[UNK]"], [], ["[UNK]"]]  )

								self.assertListEqual(
								    [rust_tokenizer.tokenize(lowercase  ) for t in ["Test", "\xad", "test"]]			,  [["[UNK]"], [], ["[UNK]"]]  )







			@slow
			def 			__lowerCAmelCase				(			self			: str  ):





								'''simple docstring'''



								UpperCAmelCase			:   int      =				self.tokenizer_class.from_pretrained("google/mobilebert-uncased"  )

								UpperCAmelCase			:   Optional[int]      =				tokenizer.encode("sequence builders"			,  add_special_tokens=lowercase  )
								UpperCAmelCase			:   str      =				tokenizer.encode("multi-sequence build"			,  add_special_tokens=lowercase  )

								UpperCAmelCase			:   List[Any]      =				tokenizer.build_inputs_with_special_tokens(lowercase  )
								UpperCAmelCase			:   Any      =				tokenizer.build_inputs_with_special_tokens(lowercase			,  lowercase  )

								assert encoded_sentence == [1_01] + text + [1_02]
								assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]







			def 			__lowerCAmelCase				(			self			: List[str]  ):





								'''simple docstring'''



								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
																		UpperCAmelCase			:   Optional[Any]      =				self.rust_tokenizer_class.from_pretrained(lowercase			,  **lowercase  )

																		UpperCAmelCase			:   Optional[int]      =				f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																		UpperCAmelCase			:   Dict      =				tokenizer_r.encode_plus(
																		    lowercase			,  return_attention_mask=lowercase			,  return_token_type_ids=lowercase			,  return_offsets_mapping=lowercase			,  add_special_tokens=lowercase			,  )

																		UpperCAmelCase			:   Dict      =				tokenizer_r.do_lower_case if hasattr(lowercase			,  "do_lower_case"  ) else False
																		UpperCAmelCase			:   int      =				(
																		    [
																		        ((0, 0), tokenizer_r.cls_token),
																		        ((0, 1), "A"),
																		        ((1, 2), ","),
																		        ((3, 5), "na"),
																		        ((5, 6), "##ï"),
																		        ((6, 8), "##ve"),
																		        ((9, 15), tokenizer_r.mask_token),
																		        ((16, 21), "Allen"),
																		        ((21, 23), "##NL"),
																		        ((23, 24), "##P"),
																		        ((25, 33), "sentence"),
																		        ((33, 34), "."),
																		        ((0, 0), tokenizer_r.sep_token),
																		    ]
																		    if not do_lower_case
																		    else [
																		        ((0, 0), tokenizer_r.cls_token),
																		        ((0, 1), "a"),
																		        ((1, 2), ","),
																		        ((3, 8), "naive"),
																		        ((9, 15), tokenizer_r.mask_token),
																		        ((16, 21), "allen"),
																		        ((21, 23), "##nl"),
																		        ((23, 24), "##p"),
																		        ((25, 33), "sentence"),
																		        ((33, 34), "."),
																		        ((0, 0), tokenizer_r.sep_token),
																		    ]
																		)

																		self.assertEqual(
																		    [e[1] for e in expected_results]			,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]  )  )
																		self.assertEqual([e[0] for e in expected_results]			,  tokens["offset_mapping"]  )







			def 			__lowerCAmelCase				(			self			: Dict  ):





								'''simple docstring'''



								UpperCAmelCase			:   Dict      =				["的", "人", "有"]
								UpperCAmelCase			:   List[str]      =				"".join(lowercase  )
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
																		UpperCAmelCase			:   Tuple      =				True
																		UpperCAmelCase			:   List[Any]      =				self.tokenizer_class.from_pretrained(lowercase			,  **lowercase  )
																		UpperCAmelCase			:   Tuple      =				self.rust_tokenizer_class.from_pretrained(lowercase			,  **lowercase  )

																		UpperCAmelCase			:   int      =				tokenizer_p.encode(lowercase			,  add_special_tokens=lowercase  )
																		UpperCAmelCase			:   List[Any]      =				tokenizer_r.encode(lowercase			,  add_special_tokens=lowercase  )

																		UpperCAmelCase			:   Dict      =				tokenizer_r.convert_ids_to_tokens(lowercase  )
																		UpperCAmelCase			:   int      =				tokenizer_p.convert_ids_to_tokens(lowercase  )

																		# it is expected that each Chinese character is not preceded by "##"
																		self.assertListEqual(lowercase			,  lowercase  )
																		self.assertListEqual(lowercase			,  lowercase  )

																		UpperCAmelCase			:   List[Any]      =				False
																		UpperCAmelCase			:   Any      =				self.rust_tokenizer_class.from_pretrained(lowercase			,  **lowercase  )
																		UpperCAmelCase			:   Union[str, Any]      =				self.tokenizer_class.from_pretrained(lowercase			,  **lowercase  )

																		UpperCAmelCase			:   Tuple      =				tokenizer_r.encode(lowercase			,  add_special_tokens=lowercase  )
																		UpperCAmelCase			:   List[str]      =				tokenizer_p.encode(lowercase			,  add_special_tokens=lowercase  )

																		UpperCAmelCase			:   Tuple      =				tokenizer_r.convert_ids_to_tokens(lowercase  )
																		UpperCAmelCase			:   List[str]      =				tokenizer_p.convert_ids_to_tokens(lowercase  )

																		# it is expected that only the first Chinese character is not preceded by "##".
																		UpperCAmelCase			:   Tuple      =				[
																		    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowercase  )
																		]
																		self.assertListEqual(lowercase			,  lowercase  )
																		self.assertListEqual(lowercase			,  lowercase  )





"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


snake_case_   :   Dict        =       {
    """configuration_mvp""": ["""MVP_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MvpConfig""", """MvpOnnxConfig"""],
    """tokenization_mvp""": ["""MvpTokenizer"""],
}

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	snake_case_   :   List[Any]        =       ["""MvpTokenizerFast"""]

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	snake_case_   :   Any        =       [
	    """MVP_PRETRAINED_MODEL_ARCHIVE_LIST""",
	    """MvpForCausalLM""",
	    """MvpForConditionalGeneration""",
	    """MvpForQuestionAnswering""",
	    """MvpForSequenceClassification""",
	    """MvpModel""",
	    """MvpPreTrainedModel""",
	]

if TYPE_CHECKING:
	from .configuration_mvp import MVP_PRETRAINED_CONFIG_ARCHIVE_MAP, MvpConfig, MvpOnnxConfig
	from .tokenization_mvp import MvpTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_mvp_fast import MvpTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_mvp import (
		    MVP_PRETRAINED_MODEL_ARCHIVE_LIST,
		    MvpForCausalLM,
		    MvpForConditionalGeneration,
		    MvpForQuestionAnswering,
		    MvpForSequenceClassification,
		    MvpModel,
		    MvpPreTrainedModel,
		)

else:
	import sys

	snake_case_   :   List[str]        =       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


