

import os
import tempfile
import unittest

from transformers import DistilBertConfig, is_torch_available
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
	    DistilBertForMaskedLM,
	    DistilBertForMultipleChoice,
	    DistilBertForQuestionAnswering,
	    DistilBertForSequenceClassification,
	    DistilBertForTokenClassification,
	    DistilBertModel,
	)



class        _a		(__magic_name__							):


					'''simple docstring'''






					def __init__(					self       ,			A__       ,			A__=13       ,			A__=7       ,			A__=True       ,			A__=True       ,			A__=False       ,			A__=True       ,			A__=99       ,			A__=32       ,			A__=5       ,			A__=4       ,			A__=37       ,			A__="gelu"       ,			A__=0.1       ,			A__=0.1       ,			A__=512       ,			A__=16       ,			A__=2       ,			A__=0.0_2       ,			A__=3       ,			A__=4       ,			A__=None       ,			):
						A__				:       List[Any]	  =						parent
						A__				:       List[str]	  =						batch_size
						A__				:       Optional[int]	  =						seq_length
						A__				:       Tuple	  =						is_training
						A__				:       str	  =						use_input_mask
						A__				:       List[Any]	  =						use_token_type_ids
						A__				:       Optional[int]	  =						use_labels
						A__				:       Union[str, Any]	  =						vocab_size
						A__				:       List[str]	  =						hidden_size
						A__				:       str	  =						num_hidden_layers
						A__				:       int	  =						num_attention_heads
						A__				:       str	  =						intermediate_size
						A__				:       List[Any]	  =						hidden_act
						A__				:       Any	  =						hidden_dropout_prob
						A__				:       Any	  =						attention_probs_dropout_prob
						A__				:       Dict	  =						max_position_embeddings
						A__				:       Dict	  =						type_vocab_size
						A__				:       str	  =						type_sequence_label_size
						A__				:       Tuple	  =						initializer_range
						A__				:       str	  =						num_labels
						A__				:       str	  =						num_choices
						A__				:       Dict	  =						scope






					def     __A     (					self     ):
						A__				:       Optional[int]	  =						ids_tensor([self.batch_size, self.seq_length]       ,			self.vocab_size     )

						A__				:       Tuple	  =						None
						if self.use_input_mask:
							A__				:       Optional[int]	  =						random_attention_mask([self.batch_size, self.seq_length]     )

						A__				:       Optional[int]	  =						None
						A__				:       Union[str, Any]	  =						None
						A__				:       Any	  =						None
						if self.use_labels:
							A__				:       Optional[Any]	  =						ids_tensor([self.batch_size]       ,			self.type_sequence_label_size     )
							A__				:       Tuple	  =						ids_tensor([self.batch_size, self.seq_length]       ,			self.num_labels     )
							A__				:       Tuple	  =						ids_tensor([self.batch_size]       ,			self.num_choices     )

						A__				:       List[str]	  =						self.get_config()

						return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels






					def     __A     (					self     ):
						return DistilBertConfig(
						    vocab_size=self.vocab_size       ,			dim=self.hidden_size       ,			n_layers=self.num_hidden_layers       ,			n_heads=self.num_attention_heads       ,			hidden_dim=self.intermediate_size       ,			hidden_act=self.hidden_act       ,			dropout=self.hidden_dropout_prob       ,			attention_dropout=self.attention_probs_dropout_prob       ,			max_position_embeddings=self.max_position_embeddings       ,			initializer_range=self.initializer_range       ,			)






					def     __A     (					self       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__     ):
						A__				:       Optional[int]	  =						DistilBertModel(config=A__     )
						model.to(A__     )
						model.eval()
						A__				:       Dict	  =						model(A__       ,			A__     )
						A__				:       Optional[int]	  =						model(A__     )
						self.parent.assertEqual(result.last_hidden_state.shape       ,			(self.batch_size, self.seq_length, self.hidden_size)     )






					def     __A     (					self       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__     ):
						A__				:       Union[str, Any]	  =						DistilBertForMaskedLM(config=A__     )
						model.to(A__     )
						model.eval()
						A__				:       List[Any]	  =						model(A__       ,			attention_mask=A__       ,			labels=A__     )
						self.parent.assertEqual(result.logits.shape       ,			(self.batch_size, self.seq_length, self.vocab_size)     )






					def     __A     (					self       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__     ):
						A__				:       Tuple	  =						DistilBertForQuestionAnswering(config=A__     )
						model.to(A__     )
						model.eval()
						A__				:       Optional[int]	  =						model(
						    A__       ,			attention_mask=A__       ,			start_positions=A__       ,			end_positions=A__     )
						self.parent.assertEqual(result.start_logits.shape       ,			(self.batch_size, self.seq_length)     )
						self.parent.assertEqual(result.end_logits.shape       ,			(self.batch_size, self.seq_length)     )






					def     __A     (					self       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__     ):
						A__				:       int	  =						self.num_labels
						A__				:       Union[str, Any]	  =						DistilBertForSequenceClassification(A__     )
						model.to(A__     )
						model.eval()
						A__				:       List[Any]	  =						model(A__       ,			attention_mask=A__       ,			labels=A__     )
						self.parent.assertEqual(result.logits.shape       ,			(self.batch_size, self.num_labels)     )






					def     __A     (					self       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__     ):
						A__				:       Optional[Any]	  =						self.num_labels
						A__				:       int	  =						DistilBertForTokenClassification(config=A__     )
						model.to(A__     )
						model.eval()

						A__				:       List[str]	  =						model(A__       ,			attention_mask=A__       ,			labels=A__     )
						self.parent.assertEqual(result.logits.shape       ,			(self.batch_size, self.seq_length, self.num_labels)     )






					def     __A     (					self       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__       ,			A__     ):
						A__				:       Dict	  =						self.num_choices
						A__				:       List[Any]	  =						DistilBertForMultipleChoice(config=A__     )
						model.to(A__     )
						model.eval()
						A__				:       Any	  =						input_ids.unsqueeze(1     ).expand(-1       ,			self.num_choices       ,			-1     ).contiguous()
						A__				:       str	  =						input_mask.unsqueeze(1     ).expand(-1       ,			self.num_choices       ,			-1     ).contiguous()
						A__				:       Optional[int]	  =						model(
						    A__       ,			attention_mask=A__       ,			labels=A__       ,			)
						self.parent.assertEqual(result.logits.shape       ,			(self.batch_size, self.num_choices)     )






					def     __A     (					self     ):
						A__				:       int	  =						self.prepare_config_and_inputs()
						((A__)  ,  (A__)  ,  (A__)  ,  (A__)  ,  (A__)  ,  (A__))				:       Tuple	  =						config_and_inputs
						A__				:       Dict	  =						{"""input_ids""": input_ids, """attention_mask""": input_mask}
						return config, inputs_dict



@require_torch
class        _a		(__magic_name__						,  __magic_name__						,  unittest.TestCase							):


					'''simple docstring'''






					UpperCAmelCase__:  str    				=			(
					    (
					        DistilBertModel,
					        DistilBertForMaskedLM,
					        DistilBertForMultipleChoice,
					        DistilBertForQuestionAnswering,
					        DistilBertForSequenceClassification,
					        DistilBertForTokenClassification,
					    )
					    if is_torch_available()
					    else None
					)
					UpperCAmelCase__:  Any    				=			(
					    {
					        '''feature-extraction''': DistilBertModel,
					        '''fill-mask''': DistilBertForMaskedLM,
					        '''question-answering''': DistilBertForQuestionAnswering,
					        '''text-classification''': DistilBertForSequenceClassification,
					        '''token-classification''': DistilBertForTokenClassification,
					        '''zero-shot''': DistilBertForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					UpperCAmelCase__:  Tuple    				=			True
					UpperCAmelCase__:  Dict    				=			True
					UpperCAmelCase__:  List[Any]    				=			True
					UpperCAmelCase__:  List[Any]    				=			True






					def     __A     (					self     ):
						A__				:       int	  =						DistilBertModelTester(self     )
						A__				:       Optional[int]	  =						ConfigTester(self       ,			config_class=A__       ,			dim=37     )






					def     __A     (					self     ):
						self.config_tester.run_common_tests()






					def     __A     (					self     ):
						A__				:       Union[str, Any]	  =						self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_model(*A__     )






					def     __A     (					self     ):
						A__				:       int	  =						self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_for_masked_lm(*A__     )






					def     __A     (					self     ):
						A__				:       Tuple	  =						self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_for_question_answering(*A__     )






					def     __A     (					self     ):
						A__				:       str	  =						self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_for_sequence_classification(*A__     )






					def     __A     (					self     ):
						A__				:       int	  =						self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_for_token_classification(*A__     )






					def     __A     (					self     ):
						A__				:       Tuple	  =						self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_distilbert_for_multiple_choice(*A__     )






					@slow
					def     __A     (					self     ):
						for model_name in DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							A__				:       List[Any]	  =						DistilBertModel.from_pretrained(A__     )
							self.assertIsNotNone(A__     )






					@slow
					@require_torch_gpu
					def     __A     (					self     ):
						A__  ,  A__				:       Optional[Any]	  =						self.model_tester.prepare_config_and_inputs_for_common()
						for model_class in self.all_model_classes:
							# BertForMultipleChoice behaves incorrectly in JIT environments.
							if model_class == DistilBertForMultipleChoice:
								return

							A__				:       List[str]	  =						True
							A__				:       Tuple	  =						model_class(config=A__     )

							A__				:       List[Any]	  =						self._prepare_for_class(A__       ,			A__     )
							A__				:       Tuple	  =						torch.jit.trace(
							    A__       ,			(inputs_dict["""input_ids"""].to("""cpu"""     ), inputs_dict["""attention_mask"""].to("""cpu"""     ))     )

							with tempfile.TemporaryDirectory() as tmp:
								torch.jit.save(A__       ,			os.path.join(A__       ,			"""traced_model.pt"""     )     )
								A__				:       Union[str, Any]	  =						torch.jit.load(os.path.join(A__       ,			"""traced_model.pt"""     )       ,			map_location=A__     )
								loaded(inputs_dict["""input_ids"""].to(A__     )       ,			inputs_dict["""attention_mask"""].to(A__     )     )



@require_torch
class        _a		(unittest.TestCase							):


					'''simple docstring'''






					@slow
					def     __A     (					self     ):
						A__				:       Union[str, Any]	  =						DistilBertModel.from_pretrained("""distilbert-base-uncased"""     )
						A__				:       Optional[Any]	  =						torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]     )
						A__				:       int	  =						torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]     )
						with torch.no_grad():
							A__				:       List[str]	  =						model(A__       ,			attention_mask=A__     )[0]
						A__				:       Tuple	  =						torch.Size((1, 11, 768)     )
						self.assertEqual(output.shape       ,			A__     )
						A__				:       Any	  =						torch.tensor(
						    [[[-0.1_6_3_9, 0.3_2_9_9, 0.1_6_4_8], [-0.1_7_4_6, 0.3_2_8_9, 0.1_7_1_0], [-0.1_8_8_4, 0.3_3_5_7, 0.1_8_1_0]]]     )

						self.assertTrue(torch.allclose(output[:, 1:4, 1:4]       ,			A__       ,			atol=1e-4     )     )






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class        _a		(__magic_name__						,  unittest.TestCase							):


					'''simple docstring'''






					UpperCAmelCase__:  List[str]    				=			RoCBertTokenizer
					UpperCAmelCase__:  Dict    				=			None
					UpperCAmelCase__:  Optional[Any]    				=			False
					UpperCAmelCase__:  Union[str, Any]    				=			True
					UpperCAmelCase__:  Union[str, Any]    				=			filter_non_english






					def     __A     (					self     ):
						super().setUp()

						A__				:       Tuple	  =						["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
						A__				:       Union[str, Any]	  =						{}
						A__				:       Dict	  =						{}
						for i, value in enumerate(A__     ):
							A__				:       Optional[int]	  =						i
							A__				:       Optional[int]	  =						i
						A__				:       Dict	  =						os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES["""vocab_file"""]     )
						A__				:       Optional[Any]	  =						os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES["""word_shape_file"""]     )
						A__				:       int	  =						os.path.join(self.tmpdirname       ,			VOCAB_FILES_NAMES["""word_pronunciation_file"""]     )
						with open(self.vocab_file       ,			"""w"""       ,			encoding="""utf-8"""     ) as vocab_writer:
							vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]     )     )
						with open(self.word_shape_file       ,			"""w"""       ,			encoding="""utf-8"""     ) as word_shape_writer:
							json.dump(A__       ,			A__       ,			ensure_ascii=A__     )
						with open(self.word_pronunciation_file       ,			"""w"""       ,			encoding="""utf-8"""     ) as word_pronunciation_writer:
							json.dump(A__       ,			A__       ,			ensure_ascii=A__     )






					def     __A     (					self     ):
						A__				:       Dict	  =						self.tokenizer_class(self.vocab_file       ,			self.word_shape_file       ,			self.word_pronunciation_file     )

						A__				:       Optional[Any]	  =						tokenizer.tokenize("""你好[SEP]你是谁"""     )
						self.assertListEqual(A__       ,			["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]     )
						self.assertListEqual(tokenizer.convert_tokens_to_ids(A__     )       ,			[5, 6, 2, 5, 7, 8]     )
						self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(A__     )       ,			[5, 6, 2, 5, 7, 8]     )
						self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(A__     )       ,			[5, 6, 2, 5, 7, 8]     )






					def     __A     (					self     ):
						A__				:       Dict	  =						RoCBertBasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""     )       ,			["""ah""", """\u535A""", """\u63A8""", """zz"""]     )






					def     __A     (					self     ):
						A__				:       List[str]	  =						RoCBertBasicTokenizer(do_lower_case=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """     )       ,			["""hello""", """!""", """how""", """are""", """you""", """?"""]     )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )       ,			["""hello"""]     )






					def     __A     (					self     ):
						A__				:       int	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			strip_accents=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""hällo""", """!""", """how""", """are""", """you""", """?"""]     )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )       ,			["""h\u00E9llo"""]     )






					def     __A     (					self     ):
						A__				:       Optional[Any]	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			strip_accents=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""hallo""", """!""", """how""", """are""", """you""", """?"""]     )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )       ,			["""hello"""]     )






					def     __A     (					self     ):
						A__				:       Union[str, Any]	  =						RoCBertBasicTokenizer(do_lower_case=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""hallo""", """!""", """how""", """are""", """you""", """?"""]     )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )       ,			["""hello"""]     )






					def     __A     (					self     ):
						A__				:       Union[str, Any]	  =						RoCBertBasicTokenizer(do_lower_case=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """     )       ,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )






					def     __A     (					self     ):
						A__				:       Dict	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			strip_accents=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )






					def     __A     (					self     ):
						A__				:       Optional[Any]	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			strip_accents=A__     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """     )       ,			["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]     )






					def     __A     (					self     ):
						A__				:       Dict	  =						RoCBertBasicTokenizer(do_lower_case=A__       ,			never_split=["""[UNK]"""]     )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""     )       ,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]     )






					def     __A     (					self     ):
						A__				:       List[Any]	  =						["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

						A__				:       Any	  =						{}
						for i, token in enumerate(A__     ):
							A__				:       Optional[int]	  =						i
						A__				:       Optional[Any]	  =						RoCBertWordpieceTokenizer(vocab=A__       ,			unk_token="""[UNK]"""     )

						self.assertListEqual(tokenizer.tokenize(""""""     )       ,			[]     )

						self.assertListEqual(tokenizer.tokenize("""unwanted running"""     )       ,			["""un""", """##want""", """##ed""", """runn""", """##ing"""]     )

						self.assertListEqual(tokenizer.tokenize("""unwantedX running"""     )       ,			["""[UNK]""", """runn""", """##ing"""]     )






					def     __A     (					self     ):
						self.assertTrue(_is_whitespace(""" """     )     )
						self.assertTrue(_is_whitespace("""\t"""     )     )
						self.assertTrue(_is_whitespace("""\r"""     )     )
						self.assertTrue(_is_whitespace("""\n"""     )     )
						self.assertTrue(_is_whitespace("""\u00A0"""     )     )

						self.assertFalse(_is_whitespace("""A"""     )     )
						self.assertFalse(_is_whitespace("""-"""     )     )






					def     __A     (					self     ):
						self.assertTrue(_is_control("""\u0005"""     )     )

						self.assertFalse(_is_control("""A"""     )     )
						self.assertFalse(_is_control(""" """     )     )
						self.assertFalse(_is_control("""\t"""     )     )
						self.assertFalse(_is_control("""\r"""     )     )






					def     __A     (					self     ):
						self.assertTrue(_is_punctuation("""-"""     )     )
						self.assertTrue(_is_punctuation("""$"""     )     )
						self.assertTrue(_is_punctuation("""`"""     )     )
						self.assertTrue(_is_punctuation("""."""     )     )

						self.assertFalse(_is_punctuation("""A"""     )     )
						self.assertFalse(_is_punctuation(""" """     )     )






					def     __A     (					self     ):
						A__				:       Optional[int]	  =						self.get_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(A__     ) for t in ["""Test""", """\xad""", """test"""]]       ,			[["""[UNK]"""], [], ["""[UNK]"""]]     )

						if self.test_rust_tokenizer:
							A__				:       Tuple	  =						self.get_rust_tokenizer()
							self.assertListEqual(
							    [rust_tokenizer.tokenize(A__     ) for t in ["""Test""", """\xad""", """test"""]]       ,			[["""[UNK]"""], [], ["""[UNK]"""]]     )






					def     __A     (					self     ):
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
								A__				:       Dict	  =						self.rust_tokenizer_class.from_pretrained(A__       ,			**A__     )

								A__				:       str	  =						F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
								A__				:       Union[str, Any]	  =						tokenizer_r.encode_plus(
								    A__       ,			return_attention_mask=A__       ,			return_token_type_ids=A__       ,			return_offsets_mapping=A__       ,			add_special_tokens=A__       ,			)

								A__				:       Any	  =						tokenizer_r.do_lower_case if hasattr(A__       ,			"""do_lower_case"""     ) else False
								A__				:       List[str]	  =						(
								    [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), """A"""),
								        ((1, 2), ""","""),
								        ((3, 5), """na"""),
								        ((5, 6), """##ï"""),
								        ((6, 8), """##ve"""),
								        ((9, 15), tokenizer_r.mask_token),
								        ((16, 21), """Allen"""),
								        ((21, 23), """##NL"""),
								        ((23, 24), """##P"""),
								        ((25, 33), """sentence"""),
								        ((33, 34), """."""),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								    if not do_lower_case
								    else [
								        ((0, 0), tokenizer_r.cls_token),
								        ((0, 1), """a"""),
								        ((1, 2), ""","""),
								        ((3, 8), """naive"""),
								        ((9, 15), tokenizer_r.mask_token),
								        ((16, 21), """allen"""),
								        ((21, 23), """##nl"""),
								        ((23, 24), """##p"""),
								        ((25, 33), """sentence"""),
								        ((33, 34), """."""),
								        ((0, 0), tokenizer_r.sep_token),
								    ]
								)

								self.assertEqual(
								    [e[1] for e in expected_results]       ,			tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]     )     )
								self.assertEqual([e[0] for e in expected_results]       ,			tokens["""offset_mapping"""]     )






					def     __A     (					self     ):
						A__				:       Union[str, Any]	  =						["""的""", """人""", """有"""]
						A__				:       List[str]	  =						"""""".join(A__     )
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
								A__				:       Any	  =						True
								A__				:       int	  =						self.tokenizer_class.from_pretrained(A__       ,			**A__     )
								A__				:       Dict	  =						self.rust_tokenizer_class.from_pretrained(A__       ,			**A__     )

								A__				:       List[str]	  =						tokenizer_p.encode(A__       ,			add_special_tokens=A__     )
								A__				:       List[Any]	  =						tokenizer_r.encode(A__       ,			add_special_tokens=A__     )

								A__				:       Tuple	  =						tokenizer_r.convert_ids_to_tokens(A__     )
								A__				:       int	  =						tokenizer_p.convert_ids_to_tokens(A__     )

								# it is expected that each Chinese character is not preceded by "##"
								self.assertListEqual(A__       ,			A__     )
								self.assertListEqual(A__       ,			A__     )

								A__				:       Optional[int]	  =						False
								A__				:       Union[str, Any]	  =						self.rust_tokenizer_class.from_pretrained(A__       ,			**A__     )
								A__				:       str	  =						self.tokenizer_class.from_pretrained(A__       ,			**A__     )

								A__				:       Tuple	  =						tokenizer_r.encode(A__       ,			add_special_tokens=A__     )
								A__				:       List[str]	  =						tokenizer_p.encode(A__       ,			add_special_tokens=A__     )

								A__				:       Union[str, Any]	  =						tokenizer_r.convert_ids_to_tokens(A__     )
								A__				:       Tuple	  =						tokenizer_p.convert_ids_to_tokens(A__     )

								# it is expected that only the first Chinese character is not preceded by "##".
								A__				:       str	  =						[
								    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(A__     )
								]
								self.assertListEqual(A__       ,			A__     )
								self.assertListEqual(A__       ,			A__     )






					@slow
					def     __A     (					self     ):
						A__				:       str	  =						self.tokenizer_class(self.vocab_file       ,			self.word_shape_file       ,			self.word_pronunciation_file     )

						A__				:       Optional[Any]	  =						tokenizer.encode("""你好"""       ,			add_special_tokens=A__     )
						A__				:       List[Any]	  =						tokenizer.encode("""你是谁"""       ,			add_special_tokens=A__     )

						A__				:       Any	  =						tokenizer.build_inputs_with_special_tokens(A__     )
						A__				:       Any	  =						tokenizer.build_inputs_with_special_tokens(A__       ,			A__     )

						assert encoded_sentence == [1] + text + [2]
						assert encoded_pair == [1] + text + [2] + text_a + [2]






					def     __A     (					self     ):
						A__				:       List[str]	  =						self.get_tokenizers(do_lower_case=A__     )
						for tokenizer in tokenizers:
							with self.subTest(F"""{tokenizer.__class__.__name__}"""     ):
								A__				:       Optional[int]	  =						"""你好，你是谁"""
								A__				:       List[str]	  =						tokenizer.tokenize(A__     )
								A__				:       Union[str, Any]	  =						tokenizer.convert_tokens_to_ids(A__     )
								A__				:       str	  =						tokenizer.convert_tokens_to_shape_ids(A__     )
								A__				:       Optional[int]	  =						tokenizer.convert_tokens_to_pronunciation_ids(A__     )
								A__				:       Union[str, Any]	  =						tokenizer.prepare_for_model(
								    A__       ,			A__       ,			A__       ,			add_special_tokens=A__     )

								A__				:       int	  =						tokenizer.encode_plus(A__       ,			add_special_tokens=A__     )

								self.assertEqual(A__       ,			A__     )



