





"""simple docstring"""


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 							lowercase__			(   __SCREAMING_SNAKE_CASE					,     unittest.TestCase	):

			"""simple docstring"""
			__lowerCAmelCase     :				Dict										=  RoCBertTokenizer
			__lowerCAmelCase     :				Dict										=  None
			__lowerCAmelCase     :				Union[str, Any]										=  False
			__lowerCAmelCase     :				Union[str, Any]										=  True
			__lowerCAmelCase     :				List[Any]										=  filter_non_english







			def   _a      (      self    ):






									'''simple docstring'''







									super().setUp()

									UpperCamelCase				:		List[str]				   =			["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
									UpperCamelCase				:		Any				   =			{}
									UpperCamelCase				:		Optional[Any]				   =			{}
									for i, value in enumerate(_A    ):
															UpperCamelCase				:		Optional[int]				   =			i
															UpperCamelCase				:		Optional[int]				   =			i
									UpperCamelCase				:		List[str]				   =			os.path.join(self.tmpdirname						,   VOCAB_FILES_NAMES["""vocab_file"""]    )
									UpperCamelCase				:		Dict				   =			os.path.join(self.tmpdirname						,   VOCAB_FILES_NAMES["""word_shape_file"""]    )
									UpperCamelCase				:		Optional[int]				   =			os.path.join(self.tmpdirname						,   VOCAB_FILES_NAMES["""word_pronunciation_file"""]    )
									with open(self.vocab_file						,   """w"""						,   encoding="""utf-8"""    ) as vocab_writer:
															vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]    )    )
									with open(self.word_shape_file						,   """w"""						,   encoding="""utf-8"""    ) as word_shape_writer:
															json.dump(_A						,   _A						,   ensure_ascii=_A    )
									with open(self.word_pronunciation_file						,   """w"""						,   encoding="""utf-8"""    ) as word_pronunciation_writer:
															json.dump(_A						,   _A						,   ensure_ascii=_A    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Optional[int]				   =			self.tokenizer_class(self.vocab_file						,   self.word_shape_file						,   self.word_pronunciation_file    )

									UpperCamelCase				:		Dict				   =			tokenizer.tokenize("""你好[SEP]你是谁"""    )
									self.assertListEqual(_A						,   ["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]    )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_A    )						,   [5, 6, 2, 5, 7, 8]    )
									self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(_A    )						,   [5, 6, 2, 5, 7, 8]    )
									self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(_A    )						,   [5, 6, 2, 5, 7, 8]    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		List[Any]				   =			RoCBertBasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""    )						,   ["""ah""", """\u535A""", """\u63A8""", """zz"""]    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		int				   =			RoCBertBasicTokenizer(do_lower_case=_A    )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )						,   ["""hello""", """!""", """how""", """are""", """you""", """?"""]    )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )						,   ["""hello"""]    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Union[str, Any]				   =			RoCBertBasicTokenizer(do_lower_case=_A						,   strip_accents=_A    )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )						,   ["""hällo""", """!""", """how""", """are""", """you""", """?"""]    )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )						,   ["""h\u00E9llo"""]    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Optional[int]				   =			RoCBertBasicTokenizer(do_lower_case=_A						,   strip_accents=_A    )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )						,   ["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )						,   ["""hello"""]    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		str				   =			RoCBertBasicTokenizer(do_lower_case=_A    )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )						,   ["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
									self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )						,   ["""hello"""]    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		List[str]				   =			RoCBertBasicTokenizer(do_lower_case=_A    )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )						,   ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		int				   =			RoCBertBasicTokenizer(do_lower_case=_A						,   strip_accents=_A    )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )						,   ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Union[str, Any]				   =			RoCBertBasicTokenizer(do_lower_case=_A						,   strip_accents=_A    )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )						,   ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Dict				   =			RoCBertBasicTokenizer(do_lower_case=_A						,   never_split=["""[UNK]"""]    )

									self.assertListEqual(
									    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""    )						,   ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Tuple				   =			["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

									UpperCamelCase				:		Tuple				   =			{}
									for i, token in enumerate(_A    ):
															UpperCamelCase				:		Union[str, Any]				   =			i
									UpperCamelCase				:		Tuple				   =			RoCBertWordpieceTokenizer(vocab=_A						,   unk_token="""[UNK]"""    )

									self.assertListEqual(tokenizer.tokenize(""""""    )						,   []    )

									self.assertListEqual(tokenizer.tokenize("""unwanted running"""    )						,   ["""un""", """##want""", """##ed""", """runn""", """##ing"""]    )

									self.assertListEqual(tokenizer.tokenize("""unwantedX running"""    )						,   ["""[UNK]""", """runn""", """##ing"""]    )







			def   _a      (      self    ):






									'''simple docstring'''







									self.assertTrue(_is_whitespace(""" """    )    )
									self.assertTrue(_is_whitespace("""\t"""    )    )
									self.assertTrue(_is_whitespace("""\r"""    )    )
									self.assertTrue(_is_whitespace("""\n"""    )    )
									self.assertTrue(_is_whitespace("""\u00A0"""    )    )

									self.assertFalse(_is_whitespace("""A"""    )    )
									self.assertFalse(_is_whitespace("""-"""    )    )







			def   _a      (      self    ):






									'''simple docstring'''







									self.assertTrue(_is_control("""\u0005"""    )    )

									self.assertFalse(_is_control("""A"""    )    )
									self.assertFalse(_is_control(""" """    )    )
									self.assertFalse(_is_control("""\t"""    )    )
									self.assertFalse(_is_control("""\r"""    )    )







			def   _a      (      self    ):






									'''simple docstring'''







									self.assertTrue(_is_punctuation("""-"""    )    )
									self.assertTrue(_is_punctuation("""$"""    )    )
									self.assertTrue(_is_punctuation("""`"""    )    )
									self.assertTrue(_is_punctuation("""."""    )    )

									self.assertFalse(_is_punctuation("""A"""    )    )
									self.assertFalse(_is_punctuation(""" """    )    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Tuple				   =			self.get_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(_A    ) for t in ["""Test""", """\xad""", """test"""]]						,   [["""[UNK]"""], [], ["""[UNK]"""]]    )

									if self.test_rust_tokenizer:
															UpperCamelCase				:		str				   =			self.get_rust_tokenizer()
															self.assertListEqual(
															    [rust_tokenizer.tokenize(_A    ) for t in ["""Test""", """\xad""", """test"""]]						,   [["""[UNK]"""], [], ["""[UNK]"""]]    )







			def   _a      (      self    ):






									'''simple docstring'''







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
																					UpperCamelCase				:		Dict				   =			self.rust_tokenizer_class.from_pretrained(_A						,   **_A    )

																					UpperCamelCase				:		List[str]				   =			f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																					UpperCamelCase				:		List[str]				   =			tokenizer_r.encode_plus(
																					    _A						,   return_attention_mask=_A						,   return_token_type_ids=_A						,   return_offsets_mapping=_A						,   add_special_tokens=_A						,   )

																					UpperCamelCase				:		List[str]				   =			tokenizer_r.do_lower_case if hasattr(_A						,   """do_lower_case"""    ) else False
																					UpperCamelCase				:		Optional[int]				   =			(
																					    [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), """A"""),
																					        ((1, 2), ""","""),
																					        ((3, 5), """na"""),
																					        ((5, 6), """##ï"""),
																					        ((6, 8), """##ve"""),
																					        ((9, 1_5), tokenizer_r.mask_token),
																					        ((1_6, 2_1), """Allen"""),
																					        ((2_1, 2_3), """##NL"""),
																					        ((2_3, 2_4), """##P"""),
																					        ((2_5, 3_3), """sentence"""),
																					        ((3_3, 3_4), """."""),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					    if not do_lower_case
																					    else [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), """a"""),
																					        ((1, 2), ""","""),
																					        ((3, 8), """naive"""),
																					        ((9, 1_5), tokenizer_r.mask_token),
																					        ((1_6, 2_1), """allen"""),
																					        ((2_1, 2_3), """##nl"""),
																					        ((2_3, 2_4), """##p"""),
																					        ((2_5, 3_3), """sentence"""),
																					        ((3_3, 3_4), """."""),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					)

																					self.assertEqual(
																					    [e[1] for e in expected_results]						,   tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]    )    )
																					self.assertEqual([e[0] for e in expected_results]						,   tokens["""offset_mapping"""]    )







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		int				   =			["""的""", """人""", """有"""]
									UpperCamelCase				:		Dict				   =			"""""".join(_A    )
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
																					UpperCamelCase				:		str				   =			True
																					UpperCamelCase				:		Optional[Any]				   =			self.tokenizer_class.from_pretrained(_A						,   **_A    )
																					UpperCamelCase				:		Any				   =			self.rust_tokenizer_class.from_pretrained(_A						,   **_A    )

																					UpperCamelCase				:		List[Any]				   =			tokenizer_p.encode(_A						,   add_special_tokens=_A    )
																					UpperCamelCase				:		Optional[Any]				   =			tokenizer_r.encode(_A						,   add_special_tokens=_A    )

																					UpperCamelCase				:		Dict				   =			tokenizer_r.convert_ids_to_tokens(_A    )
																					UpperCamelCase				:		List[Any]				   =			tokenizer_p.convert_ids_to_tokens(_A    )

																					# it is expected that each Chinese character is not preceded by "##"
																					self.assertListEqual(_A						,   _A    )
																					self.assertListEqual(_A						,   _A    )

																					UpperCamelCase				:		List[Any]				   =			False
																					UpperCamelCase				:		Optional[int]				   =			self.rust_tokenizer_class.from_pretrained(_A						,   **_A    )
																					UpperCamelCase				:		Any				   =			self.tokenizer_class.from_pretrained(_A						,   **_A    )

																					UpperCamelCase				:		int				   =			tokenizer_r.encode(_A						,   add_special_tokens=_A    )
																					UpperCamelCase				:		Tuple				   =			tokenizer_p.encode(_A						,   add_special_tokens=_A    )

																					UpperCamelCase				:		Optional[Any]				   =			tokenizer_r.convert_ids_to_tokens(_A    )
																					UpperCamelCase				:		Tuple				   =			tokenizer_p.convert_ids_to_tokens(_A    )

																					# it is expected that only the first Chinese character is not preceded by "##".
																					UpperCamelCase				:		Dict				   =			[
																					    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(_A    )
																					]
																					self.assertListEqual(_A						,   _A    )
																					self.assertListEqual(_A						,   _A    )







			@slow
			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Optional[int]				   =			self.tokenizer_class(self.vocab_file						,   self.word_shape_file						,   self.word_pronunciation_file    )

									UpperCamelCase				:		List[str]				   =			tokenizer.encode("""你好"""						,   add_special_tokens=_A    )
									UpperCamelCase				:		Tuple				   =			tokenizer.encode("""你是谁"""						,   add_special_tokens=_A    )

									UpperCamelCase				:		Optional[Any]				   =			tokenizer.build_inputs_with_special_tokens(_A    )
									UpperCamelCase				:		Optional[Any]				   =			tokenizer.build_inputs_with_special_tokens(_A						,   _A    )

									assert encoded_sentence == [1] + text + [2]
									assert encoded_pair == [1] + text + [2] + text_a + [2]







			def   _a      (      self    ):






									'''simple docstring'''







									UpperCamelCase				:		Tuple				   =			self.get_tokenizers(do_lower_case=_A    )
									for tokenizer in tokenizers:
															with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
																					UpperCamelCase				:		Union[str, Any]				   =			"""你好，你是谁"""
																					UpperCamelCase				:		int				   =			tokenizer.tokenize(_A    )
																					UpperCamelCase				:		Dict				   =			tokenizer.convert_tokens_to_ids(_A    )
																					UpperCamelCase				:		List[str]				   =			tokenizer.convert_tokens_to_shape_ids(_A    )
																					UpperCamelCase				:		Dict				   =			tokenizer.convert_tokens_to_pronunciation_ids(_A    )
																					UpperCamelCase				:		Union[str, Any]				   =			tokenizer.prepare_for_model(
																					    _A						,   _A						,   _A						,   add_special_tokens=_A    )

																					UpperCamelCase				:		Optional[int]				   =			tokenizer.encode_plus(_A						,   add_special_tokens=_A    )

																					self.assertEqual(_A						,   _A    )





"""simple docstring"""






_lowercase							=				'''
# Installazione di Transformers
! pip install transformers datasets
# Per installare dalla fonte invece dell\'ultima versione rilasciata, commenta il comando sopra e
# rimuovi la modalità commento al comando seguente.
# ! pip install git+https://github.com/huggingface/transformers.git
'''

_lowercase							=				[{'''type''': '''code''', '''content''': INSTALL_CONTENT}]
_lowercase							=				{
    '''{processor_class}''': '''FakeProcessorClass''',
    '''{model_class}''': '''FakeModelClass''',
    '''{object_class}''': '''FakeObjectClass''',
}





