






"""simple docstring"""





import os
import unittest

from transformers import MobileBertTokenizer, MobileBertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class       __A		(   SCREAMING_SNAKE_CASE_     ,unittest.TestCase			):
					UpperCAmelCase__								=				MobileBertTokenizer
					UpperCAmelCase__								=				MobileBertTokenizerFast
					UpperCAmelCase__								=				True
					UpperCAmelCase__								=				True
					UpperCAmelCase__								=				filter_non_english
					UpperCAmelCase__								=				'''google/mobilebert-uncased'''

					def   lowerCamelCase__   (			self  :      int					)			->							List[Any]:
											super().setUp()

											__magic_name__:	Tuple							=       [
											    """[UNK]""",
											    """[CLS]""",
											    """[SEP]""",
											    """[PAD]""",
											    """[MASK]""",
											    """want""",
											    """##want""",
											    """##ed""",
											    """wa""",
											    """un""",
											    """runn""",
											    """##ing""",
											    """,""",
											    """low""",
											    """lowest""",
											]
											__magic_name__:	int							=       os.path.join(self.tmpdirname     ,		VOCAB_FILES_NAMES["""vocab_file"""]					)
											with open(self.vocab_file     ,		"""w"""     ,		encoding="""utf-8"""					) as vocab_writer:
																	vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]					)					)

											__magic_name__:	Tuple							=       [
											    (tokenizer_def[0], self.pre_trained_model_path, tokenizer_def[2])  # else the 'google/' prefix is stripped
											    for tokenizer_def in self.tokenizers_list
											]

					def   lowerCamelCase__   (			self  :      List[str]     ,		__snake_case  :      Union[str, Any]					)			->							str:
											__magic_name__:	Dict							=       """UNwant\u00E9d,running"""
											__magic_name__:	str							=       """unwanted, running"""
											return input_text, output_text

					def   lowerCamelCase__   (			self  :      Dict					)			->							int:
											__magic_name__:	List[str]							=       self.tokenizer_class(self.vocab_file					)

											__magic_name__:	Any							=       tokenizer.tokenize("""UNwant\u00E9d,running"""					)
											self.assertListEqual(A__     ,		["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]					)
											self.assertListEqual(tokenizer.convert_tokens_to_ids(A__					)     ,		[9, 6, 7, 1_2, 1_0, 1_1]					)

					def   lowerCamelCase__   (			self  :      int					)			->							Optional[int]:
											if not self.test_rust_tokenizer:
																	return

											__magic_name__:	Optional[Any]							=       self.get_tokenizer()
											__magic_name__:	Dict							=       self.get_rust_tokenizer()

											__magic_name__:	Tuple							=       """UNwant\u00E9d,running"""

											__magic_name__:	int							=       tokenizer.tokenize(A__					)
											__magic_name__:	int							=       rust_tokenizer.tokenize(A__					)
											self.assertListEqual(A__     ,		A__					)

											__magic_name__:	Optional[Any]							=       tokenizer.encode(A__     ,		add_special_tokens=A__					)
											__magic_name__:	Tuple							=       rust_tokenizer.encode(A__     ,		add_special_tokens=A__					)
											self.assertListEqual(A__     ,		A__					)

											__magic_name__:	List[str]							=       self.get_rust_tokenizer()
											__magic_name__:	Tuple							=       tokenizer.encode(A__					)
											__magic_name__:	str							=       rust_tokenizer.encode(A__					)
											self.assertListEqual(A__     ,		A__					)

											# With lower casing
											__magic_name__:	int							=       self.get_tokenizer(do_lower_case=A__					)
											__magic_name__:	Dict							=       self.get_rust_tokenizer(do_lower_case=A__					)

											__magic_name__:	Tuple							=       """UNwant\u00E9d,running"""

											__magic_name__:	Union[str, Any]							=       tokenizer.tokenize(A__					)
											__magic_name__:	str							=       rust_tokenizer.tokenize(A__					)
											self.assertListEqual(A__     ,		A__					)

											__magic_name__:	Union[str, Any]							=       tokenizer.encode(A__     ,		add_special_tokens=A__					)
											__magic_name__:	Any							=       rust_tokenizer.encode(A__     ,		add_special_tokens=A__					)
											self.assertListEqual(A__     ,		A__					)

											__magic_name__:	List[Any]							=       self.get_rust_tokenizer()
											__magic_name__:	Optional[Any]							=       tokenizer.encode(A__					)
											__magic_name__:	Union[str, Any]							=       rust_tokenizer.encode(A__					)
											self.assertListEqual(A__     ,		A__					)

					def   lowerCamelCase__   (			self  :      Dict					)			->							str:
											__magic_name__:	Tuple							=       BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""					)     ,		["""ah""", """\u535A""", """\u63A8""", """zz"""]					)

					def   lowerCamelCase__   (			self  :      Optional[int]					)			->							Optional[Any]:
											__magic_name__:	Dict							=       BasicTokenizer(do_lower_case=A__					)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """					)     ,		["""hello""", """!""", """how""", """are""", """you""", """?"""]					)
											self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""					)     ,		["""hello"""]					)

					def   lowerCamelCase__   (			self  :      int					)			->							List[Any]:
											__magic_name__:	Union[str, Any]							=       BasicTokenizer(do_lower_case=A__     ,		strip_accents=A__					)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """					)     ,		["""hällo""", """!""", """how""", """are""", """you""", """?"""]					)
											self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""					)     ,		["""h\u00E9llo"""]					)

					def   lowerCamelCase__   (			self  :      Optional[int]					)			->							List[str]:
											__magic_name__:	List[Any]							=       BasicTokenizer(do_lower_case=A__     ,		strip_accents=A__					)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """					)     ,		["""hallo""", """!""", """how""", """are""", """you""", """?"""]					)
											self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""					)     ,		["""hello"""]					)

					def   lowerCamelCase__   (			self  :      List[str]					)			->							Optional[int]:
											__magic_name__:	Tuple							=       BasicTokenizer(do_lower_case=A__					)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """					)     ,		["""hallo""", """!""", """how""", """are""", """you""", """?"""]					)
											self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""					)     ,		["""hello"""]					)

					def   lowerCamelCase__   (			self  :      str					)			->							Any:
											__magic_name__:	Tuple							=       BasicTokenizer(do_lower_case=A__					)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """					)     ,		["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]					)

					def   lowerCamelCase__   (			self  :      Optional[Any]					)			->							Tuple:
											__magic_name__:	Any							=       BasicTokenizer(do_lower_case=A__     ,		strip_accents=A__					)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """					)     ,		["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]					)

					def   lowerCamelCase__   (			self  :      Optional[int]					)			->							Optional[Any]:
											__magic_name__:	Optional[int]							=       BasicTokenizer(do_lower_case=A__     ,		strip_accents=A__					)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """					)     ,		["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]					)

					def   lowerCamelCase__   (			self  :      List[str]					)			->							Tuple:
											__magic_name__:	Optional[int]							=       BasicTokenizer(do_lower_case=A__     ,		never_split=["""[UNK]"""]					)

											self.assertListEqual(
											    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""					)     ,		["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]					)

					def   lowerCamelCase__   (			self  :      Union[str, Any]					)			->							Dict:
											__magic_name__:	int							=       ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

											__magic_name__:	Optional[Any]							=       {}
											for i, token in enumerate(A__					):
																	__magic_name__:	Tuple							=       i
											__magic_name__:	Tuple							=       WordpieceTokenizer(vocab=A__     ,		unk_token="""[UNK]"""					)

											self.assertListEqual(tokenizer.tokenize(""""""					)     ,		[]					)

											self.assertListEqual(tokenizer.tokenize("""unwanted running"""					)     ,		["""un""", """##want""", """##ed""", """runn""", """##ing"""]					)

											self.assertListEqual(tokenizer.tokenize("""unwantedX running"""					)     ,		["""[UNK]""", """runn""", """##ing"""]					)

					def   lowerCamelCase__   (			self  :      Optional[Any]					)			->							Union[str, Any]:
											self.assertTrue(_is_whitespace(""" """					)					)
											self.assertTrue(_is_whitespace("""\t"""					)					)
											self.assertTrue(_is_whitespace("""\r"""					)					)
											self.assertTrue(_is_whitespace("""\n"""					)					)
											self.assertTrue(_is_whitespace("""\u00A0"""					)					)

											self.assertFalse(_is_whitespace("""A"""					)					)
											self.assertFalse(_is_whitespace("""-"""					)					)

					def   lowerCamelCase__   (			self  :      Dict					)			->							Tuple:
											self.assertTrue(_is_control("""\u0005"""					)					)

											self.assertFalse(_is_control("""A"""					)					)
											self.assertFalse(_is_control(""" """					)					)
											self.assertFalse(_is_control("""\t"""					)					)
											self.assertFalse(_is_control("""\r"""					)					)

					def   lowerCamelCase__   (			self  :      List[Any]					)			->							Union[str, Any]:
											self.assertTrue(_is_punctuation("""-"""					)					)
											self.assertTrue(_is_punctuation("""$"""					)					)
											self.assertTrue(_is_punctuation("""`"""					)					)
											self.assertTrue(_is_punctuation("""."""					)					)

											self.assertFalse(_is_punctuation("""A"""					)					)
											self.assertFalse(_is_punctuation(""" """					)					)

					def   lowerCamelCase__   (			self  :      Any					)			->							Any:
											__magic_name__:	Dict							=       self.get_tokenizer()
											__magic_name__:	Optional[int]							=       self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(A__					) for t in ["""Test""", """\xad""", """test"""]]     ,		[["""[UNK]"""], [], ["""[UNK]"""]]					)

											self.assertListEqual(
											    [rust_tokenizer.tokenize(A__					) for t in ["""Test""", """\xad""", """test"""]]     ,		[["""[UNK]"""], [], ["""[UNK]"""]]					)

					@slow
					def   lowerCamelCase__   (			self  :      List[str]					)			->							Tuple:
											__magic_name__:	Optional[Any]							=       self.tokenizer_class.from_pretrained("""google/mobilebert-uncased"""					)

											__magic_name__:	Tuple							=       tokenizer.encode("""sequence builders"""     ,		add_special_tokens=A__					)
											__magic_name__:	List[Any]							=       tokenizer.encode("""multi-sequence build"""     ,		add_special_tokens=A__					)

											__magic_name__:	Any							=       tokenizer.build_inputs_with_special_tokens(A__					)
											__magic_name__:	List[str]							=       tokenizer.build_inputs_with_special_tokens(A__     ,		A__					)

											assert encoded_sentence == [1_0_1] + text + [1_0_2]
											assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]

					def   lowerCamelCase__   (			self  :      Union[str, Any]					)			->							str:
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'					):
																							__magic_name__:	Dict							=       self.rust_tokenizer_class.from_pretrained(A__     ,		**A__					)

																							__magic_name__:	Optional[Any]							=       F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
																							__magic_name__:	int							=       tokenizer_r.encode_plus(
																							    A__     ,		return_attention_mask=A__     ,		return_token_type_ids=A__     ,		return_offsets_mapping=A__     ,		add_special_tokens=A__     ,		)

																							__magic_name__:	Union[str, Any]							=       tokenizer_r.do_lower_case if hasattr(A__     ,		"""do_lower_case"""					) else False
																							__magic_name__:	Any							=       (
																							    [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), """A"""),
																							        ((1, 2), ""","""),
																							        ((3, 5), """na"""),
																							        ((5, 6), """##ï"""),
																							        ((6, 8), """##ve"""),
																							        ((9, 1_5), tokenizer_r.mask_token),
																							        ((1_6, 2_1), """Allen"""),
																							        ((2_1, 2_3), """##NL"""),
																							        ((2_3, 2_4), """##P"""),
																							        ((2_5, 3_3), """sentence"""),
																							        ((3_3, 3_4), """."""),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							    if not do_lower_case
																							    else [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), """a"""),
																							        ((1, 2), ""","""),
																							        ((3, 8), """naive"""),
																							        ((9, 1_5), tokenizer_r.mask_token),
																							        ((1_6, 2_1), """allen"""),
																							        ((2_1, 2_3), """##nl"""),
																							        ((2_3, 2_4), """##p"""),
																							        ((2_5, 3_3), """sentence"""),
																							        ((3_3, 3_4), """."""),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							)

																							self.assertEqual(
																							    [e[1] for e in expected_results]     ,		tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]					)					)
																							self.assertEqual([e[0] for e in expected_results]     ,		tokens["""offset_mapping"""]					)






					def   lowerCamelCase__   (			self  :      int					)			->							List[Any]:
											__magic_name__:	Tuple							=       ["""的""", """人""", """有"""]
											__magic_name__:	Optional[Any]							=       """""".join(A__					)
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'					):
																							__magic_name__:	Union[str, Any]							=       True
																							__magic_name__:	Optional[Any]							=       self.tokenizer_class.from_pretrained(A__     ,		**A__					)
																							__magic_name__:	str							=       self.rust_tokenizer_class.from_pretrained(A__     ,		**A__					)

																							__magic_name__:	int							=       tokenizer_p.encode(A__     ,		add_special_tokens=A__					)
																							__magic_name__:	List[Any]							=       tokenizer_r.encode(A__     ,		add_special_tokens=A__					)

																							__magic_name__:	str							=       tokenizer_r.convert_ids_to_tokens(A__					)
																							__magic_name__:	Dict							=       tokenizer_p.convert_ids_to_tokens(A__					)

																							# it is expected that each Chinese character is not preceded by "##"
																							self.assertListEqual(A__     ,		A__					)
																							self.assertListEqual(A__     ,		A__					)

																							__magic_name__:	Optional[int]							=       False
																							__magic_name__:	Union[str, Any]							=       self.rust_tokenizer_class.from_pretrained(A__     ,		**A__					)
																							__magic_name__:	Union[str, Any]							=       self.tokenizer_class.from_pretrained(A__     ,		**A__					)

																							__magic_name__:	Dict							=       tokenizer_r.encode(A__     ,		add_special_tokens=A__					)
																							__magic_name__:	Dict							=       tokenizer_p.encode(A__     ,		add_special_tokens=A__					)

																							__magic_name__:	Optional[int]							=       tokenizer_r.convert_ids_to_tokens(A__					)
																							__magic_name__:	Any							=       tokenizer_p.convert_ids_to_tokens(A__					)

																							# it is expected that only the first Chinese character is not preceded by "##".
																							__magic_name__:	Any							=       [
																							    F'##{token}' if idx != 0 else token for idx, token in enumerate(A__					)
																							]
																							self.assertListEqual(A__     ,		A__					)
																							self.assertListEqual(A__     ,		A__					)






A_			:					Union[str, Any]    			=     '\n# Installazione di Transformers\n! pip install transformers datasets\n# Per installare dalla fonte invece dell\'ultima versione rilasciata, commenta il comando sopra e\n# rimuovi la modalità commento al comando seguente.\n# ! pip install git+https://github.com/huggingface/transformers.git\n'

A_			:					str    			=     [{'type': 'code', 'content': INSTALL_CONTENT}]
A_			:					Any    			=     {
    '{processor_class}': 'FakeProcessorClass',
    '{model_class}': 'FakeModelClass',
    '{object_class}': 'FakeObjectClass',
}



