






import inspect
import unittest

from transformers import RegNetConfig, is_flax_available
from transformers.testing_utils import require_flax, slow
from transformers.utils import cached_property, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor


if is_flax_available():
							import jax
							import jax.numpy as jnp

							from transformers.models.regnet.modeling_flax_regnet import FlaxRegNetForImageClassification, FlaxRegNetModel

if is_vision_available():
							from PIL import Image

							from transformers import AutoImageProcessor




class 		UpperCAmelCase_						(	unittest.TestCase					):


						'''simple docstring'''




						def __init__(	self		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE=3		,				_SCREAMING_SNAKE_CASE=32		,				_SCREAMING_SNAKE_CASE=3		,				_SCREAMING_SNAKE_CASE=10		,				_SCREAMING_SNAKE_CASE=[10, 20, 30, 40]		,				_SCREAMING_SNAKE_CASE=[1, 1, 2, 1]		,				_SCREAMING_SNAKE_CASE=True		,				_SCREAMING_SNAKE_CASE=True		,				_SCREAMING_SNAKE_CASE="relu"		,				_SCREAMING_SNAKE_CASE=3		,				_SCREAMING_SNAKE_CASE=None		,				) ->				Union[str, Any]:
								snake_case_				:							Optional[int]						 =       parent
								snake_case_				:							int						 =       batch_size
								snake_case_				:							Optional[Any]						 =       image_size
								snake_case_				:							Dict						 =       num_channels
								snake_case_				:							str						 =       embeddings_size
								snake_case_				:							Any						 =       hidden_sizes
								snake_case_				:							int						 =       depths
								snake_case_				:							Tuple						 =       is_training
								snake_case_				:							Dict						 =       use_labels
								snake_case_				:							Optional[int]						 =       hidden_act
								snake_case_				:							Dict						 =       num_labels
								snake_case_				:							Any						 =       scope
								snake_case_				:							List[str]						 =       len(_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self				) ->				Dict:
								snake_case_				:							Optional[Any]						 =       floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]				)

								snake_case_				:							Union[str, Any]						 =       self.get_config()

								return config, pixel_values






						def       _lowerCAmelCase       (	self				) ->				int:
								return RegNetConfig(
								    num_channels=self.num_channels		,				embeddings_size=self.embeddings_size		,				hidden_sizes=self.hidden_sizes		,				depths=self.depths		,				hidden_act=self.hidden_act		,				num_labels=self.num_labels		,				image_size=self.image_size		,				)






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				) ->				List[str]:
								snake_case_				:							Optional[int]						 =       FlaxRegNetModel(config=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Any						 =       model(_SCREAMING_SNAKE_CASE				)

								# Output shape (b, c, h, w)
								self.parent.assertEqual(
								    result.last_hidden_state.shape		,				(self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32)		,				)






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				) ->				int:
								snake_case_				:							Any						 =       self.num_labels
								snake_case_				:							Tuple						 =       FlaxRegNetForImageClassification(config=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							str						 =       model(_SCREAMING_SNAKE_CASE				)
								self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.num_labels)				)






						def       _lowerCAmelCase       (	self				) ->				List[Any]:
								snake_case_				:							str						 =       self.prepare_config_and_inputs()
								snake_case_ ,       snake_case_				:							Dict						 =       config_and_inputs
								snake_case_				:							List[Any]						 =       {"pixel_values": pixel_values}
								return config, inputs_dict




@require_flax
class 		UpperCAmelCase_						(	SCREAMING_SNAKE_CASE__					,  unittest.TestCase					):


						'''simple docstring'''




						A     :   int						=					(FlaxRegNetModel, FlaxRegNetForImageClassification) if is_flax_available() else ()

						A     :   Tuple						=					False
						A     :   Dict						=					False
						A     :   Any						=					False






						def       _lowerCAmelCase       (	self				) ->				None:
								snake_case_				:							Tuple						 =       FlaxRegNetModelTester(self				)
								snake_case_				:							Any						 =       ConfigTester(self		,				config_class=_SCREAMING_SNAKE_CASE		,				has_text_modality=_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self				) ->				int:
								self.create_and_test_config_common_properties()
								self.config_tester.create_and_test_config_to_json_string()
								self.config_tester.create_and_test_config_to_json_file()
								self.config_tester.create_and_test_config_from_and_save_pretrained()
								self.config_tester.create_and_test_config_with_num_labels()
								self.config_tester.check_config_can_be_init_without_params()
								self.config_tester.check_config_arguments_init()






						def       _lowerCAmelCase       (	self				) ->				Optional[int]:
								return






						def       _lowerCAmelCase       (	self				) ->				Optional[Any]:
								snake_case_				:							Tuple						 =       self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self				) ->				Any:
								snake_case_				:							Any						 =       self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_image_classification(*_SCREAMING_SNAKE_CASE				)






						@unittest.skip(reason="RegNet does not use inputs_embeds"				)
						def       _lowerCAmelCase       (	self				) ->				Dict:
								pass






						@unittest.skip(reason="RegNet does not support input and output embeddings"				)
						def       _lowerCAmelCase       (	self				) ->				List[str]:
								pass






						def       _lowerCAmelCase       (	self				) ->				str:
								snake_case_ ,       snake_case_				:							List[str]						 =       self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
										snake_case_				:							str						 =       model_class(_SCREAMING_SNAKE_CASE				)
										snake_case_				:							Union[str, Any]						 =       inspect.signature(model.__call__				)
										# signature.parameters is an OrderedDict => so arg_names order is deterministic
										snake_case_				:							Optional[Any]						 =       [*signature.parameters.keys()]

										snake_case_				:							int						 =       ["pixel_values"]
										self.assertListEqual(arg_names[:1]		,				_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self				) ->				Any:
								def check_hidden_states_output(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				):
										snake_case_				:							int						 =       model_class(_SCREAMING_SNAKE_CASE				)

										snake_case_				:							Dict						 =       model(**self._prepare_for_class(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)				)

										snake_case_				:							Dict						 =       outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

										snake_case_				:							Optional[int]						 =       self.model_tester.num_stages
										self.assertEqual(len(_SCREAMING_SNAKE_CASE				)		,				expected_num_stages + 1				)

								snake_case_ ,       snake_case_				:							Optional[Any]						 =       self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
										snake_case_				:							str						 =       True
										check_hidden_states_output(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

										# check that output_hidden_states also work using config
										del inputs_dict["output_hidden_states"]
										snake_case_				:							Optional[Any]						 =       True

										check_hidden_states_output(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self				) ->				Union[str, Any]:
								snake_case_ ,       snake_case_				:							List[str]						 =       self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
										with self.subTest(model_class.__name__				):
												snake_case_				:							Optional[int]						 =       self._prepare_for_class(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)
												snake_case_				:							List[str]						 =       model_class(_SCREAMING_SNAKE_CASE				)

												@jax.jit
												def model_jitted(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				):
														return model(pixel_values=_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)

												with self.subTest("JIT Enabled"				):
														snake_case_				:							str						 =       model_jitted(**_SCREAMING_SNAKE_CASE				).to_tuple()

												with self.subTest("JIT Disabled"				):
														with jax.disable_jit():
																snake_case_				:							int						 =       model_jitted(**_SCREAMING_SNAKE_CASE				).to_tuple()

												self.assertEqual(len(_SCREAMING_SNAKE_CASE				)		,				len(_SCREAMING_SNAKE_CASE				)				)
												for jitted_output, output in zip(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				):
														self.assertEqual(jitted_output.shape		,				output.shape				)




def      lowerCAmelCase__	(    ):
		snake_case_				:							Tuple						 =       Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"    )
		return image




@require_flax
class 		UpperCAmelCase_						(	unittest.TestCase					):


						'''simple docstring'''




						@cached_property
						def       _lowerCAmelCase       (	self				) ->				List[Any]:
								return AutoImageProcessor.from_pretrained("facebook/regnet-y-040"				) if is_vision_available() else None






						@slow
						def       _lowerCAmelCase       (	self				) ->				str:
								snake_case_				:							Optional[int]						 =       FlaxRegNetForImageClassification.from_pretrained("facebook/regnet-y-040"				)

								snake_case_				:							Optional[Any]						 =       self.default_image_processor
								snake_case_				:							List[str]						 =       prepare_img()
								snake_case_				:							Any						 =       image_processor(images=_SCREAMING_SNAKE_CASE		,				return_tensors="np"				)

								snake_case_				:							str						 =       model(**_SCREAMING_SNAKE_CASE				)

								# verify the logits
								snake_case_				:							Tuple						 =       (1, 1000)
								self.assertEqual(outputs.logits.shape		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							List[str]						 =       jnp.array([-0.4180, -1.5051, -3.4836]				)

								self.assertTrue(jnp.allclose(outputs.logits[0, :3]		,				_SCREAMING_SNAKE_CASE		,				atol=1e-4				)				)








import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 		UpperCAmelCase_						(	SCREAMING_SNAKE_CASE__					,  unittest.TestCase					):


						'''simple docstring'''




						A     :   str						=					BertTokenizer
						A     :   Union[str, Any]						=					BertTokenizerFast
						A     :   int						=					True
						A     :   Union[str, Any]						=					True
						A     :   Union[str, Any]						=					filter_non_english






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								super().setUp()

								snake_case_				:							Optional[int]						 =       [
								    "[UNK]",
								    "[CLS]",
								    "[SEP]",
								    "[PAD]",
								    "[MASK]",
								    "want",
								    "##want",
								    "##ed",
								    "wa",
								    "un",
								    "runn",
								    "##ing",
								    ",",
								    "low",
								    "lowest",
								]
								snake_case_				:							str						 =       os.path.join(self.tmpdirname		,				VOCAB_FILES_NAMES["vocab_file"]				)
								with open(self.vocab_file		,				"w"		,				encoding="utf-8"				) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE				) ->				Tuple:
								snake_case_				:							str						 =       "UNwant\u00E9d,running"
								snake_case_				:							Optional[int]						 =       "unwanted, running"
								return input_text, output_text






						def       _lowerCAmelCase       (	self				) ->				str:
								snake_case_				:							str						 =       self.tokenizer_class(self.vocab_file				)

								snake_case_				:							Tuple						 =       tokenizer.tokenize("UNwant\u00E9d,running"				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				["un", "##want", "##ed", ",", "runn", "##ing"]				)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE				)		,				[9, 6, 7, 12, 10, 11]				)






						def       _lowerCAmelCase       (	self				) ->				Optional[Any]:
								if not self.test_rust_tokenizer:
										return

								snake_case_				:							Any						 =       self.get_tokenizer()
								snake_case_				:							str						 =       self.get_rust_tokenizer()

								snake_case_				:							List[Any]						 =       "UNwant\u00E9d,running"

								snake_case_				:							Union[str, Any]						 =       tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							int						 =       rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							str						 =       tokenizer.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							str						 =       rust_tokenizer.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							int						 =       self.get_rust_tokenizer()
								snake_case_				:							Any						 =       tokenizer.encode(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Any						 =       rust_tokenizer.encode(_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								# With lower casing
								snake_case_				:							Tuple						 =       self.get_tokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Any						 =       self.get_rust_tokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Tuple						 =       "UNwant\u00E9d,running"

								snake_case_				:							Tuple						 =       tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							int						 =       rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							List[Any]						 =       tokenizer.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							int						 =       rust_tokenizer.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Optional[int]						 =       self.get_rust_tokenizer()
								snake_case_				:							Dict						 =       tokenizer.encode(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							str						 =       rust_tokenizer.encode(_SCREAMING_SNAKE_CASE				)
								self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self				) ->				List[Any]:
								snake_case_				:							Union[str, Any]						 =       BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"				)		,				["ah", "\u535A", "\u63A8", "zz"]				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								snake_case_				:							List[Any]						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)		,				["hello", "!", "how", "are", "you", "?"]				)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)		,				["hello"]				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								snake_case_				:							List[str]						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["hällo", "!", "how", "are", "you", "?"]				)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)		,				["h\u00E9llo"]				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								snake_case_				:							Optional[Any]						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["hallo", "!", "how", "are", "you", "?"]				)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)		,				["hello"]				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								snake_case_				:							Dict						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["hallo", "!", "how", "are", "you", "?"]				)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)		,				["hello"]				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								snake_case_				:							List[str]						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)		,				["HeLLo", "!", "how", "Are", "yoU", "?"]				)






						def       _lowerCAmelCase       (	self				) ->				int:
								snake_case_				:							str						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["HäLLo", "!", "how", "Are", "yoU", "?"]				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								snake_case_				:							str						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				strip_accents=_SCREAMING_SNAKE_CASE				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)		,				["HaLLo", "!", "how", "Are", "yoU", "?"]				)






						def       _lowerCAmelCase       (	self				) ->				Union[str, Any]:
								snake_case_				:							Any						 =       BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE		,				never_split=["[UNK]"]				)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"				)		,				["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]				)






						def       _lowerCAmelCase       (	self				) ->				Optional[Any]:
								snake_case_				:							Optional[Any]						 =       BasicTokenizer()
								snake_case_				:							Optional[Any]						 =       "a\n'll !!to?'d of, can't."
								snake_case_				:							Tuple						 =       ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
								self.assertListEqual(tokenizer.tokenize(_SCREAMING_SNAKE_CASE				)		,				_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								snake_case_				:							List[Any]						 =       ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

								snake_case_				:							Optional[int]						 =       {}
								for i, token in enumerate(_SCREAMING_SNAKE_CASE				):
										snake_case_				:							Any						 =       i
								snake_case_				:							Any						 =       WordpieceTokenizer(vocab=_SCREAMING_SNAKE_CASE		,				unk_token="[UNK]"				)

								self.assertListEqual(tokenizer.tokenize(""				)		,				[]				)

								self.assertListEqual(tokenizer.tokenize("unwanted running"				)		,				["un", "##want", "##ed", "runn", "##ing"]				)

								self.assertListEqual(tokenizer.tokenize("unwantedX running"				)		,				["[UNK]", "runn", "##ing"]				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								self.assertTrue(_is_whitespace(" "				)				)
								self.assertTrue(_is_whitespace("\t"				)				)
								self.assertTrue(_is_whitespace("\r"				)				)
								self.assertTrue(_is_whitespace("\n"				)				)
								self.assertTrue(_is_whitespace("\u00A0"				)				)

								self.assertFalse(_is_whitespace("A"				)				)
								self.assertFalse(_is_whitespace("-"				)				)






						def       _lowerCAmelCase       (	self				) ->				List[str]:
								self.assertTrue(_is_control("\u0005"				)				)

								self.assertFalse(_is_control("A"				)				)
								self.assertFalse(_is_control(" "				)				)
								self.assertFalse(_is_control("\t"				)				)
								self.assertFalse(_is_control("\r"				)				)






						def       _lowerCAmelCase       (	self				) ->				Tuple:
								self.assertTrue(_is_punctuation("-"				)				)
								self.assertTrue(_is_punctuation("$"				)				)
								self.assertTrue(_is_punctuation("`"				)				)
								self.assertTrue(_is_punctuation("."				)				)

								self.assertFalse(_is_punctuation("A"				)				)
								self.assertFalse(_is_punctuation(" "				)				)






						def       _lowerCAmelCase       (	self				) ->				Optional[int]:
								snake_case_				:							Dict						 =       self.get_tokenizer()
								snake_case_				:							Any						 =       self.get_rust_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(_SCREAMING_SNAKE_CASE				) for t in ["Test", "\xad", "test"]]		,				[["[UNK]"], [], ["[UNK]"]]				)

								self.assertListEqual(
								    [rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE				) for t in ["Test", "\xad", "test"]]		,				[["[UNK]"], [], ["[UNK]"]]				)






						@slow
						def       _lowerCAmelCase       (	self				) ->				Optional[int]:
								snake_case_				:							Union[str, Any]						 =       self.tokenizer_class.from_pretrained("bert-base-uncased"				)

								snake_case_				:							Tuple						 =       tokenizer.encode("sequence builders"		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							List[Any]						 =       tokenizer.encode("multi-sequence build"		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)

								snake_case_				:							Optional[Any]						 =       tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Union[str, Any]						 =       tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

								assert encoded_sentence == [101] + text + [102]
								assert encoded_pair == [101] + text + [102] + text_a + [102]






						def       _lowerCAmelCase       (	self				) ->				int:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
												snake_case_				:							Union[str, Any]						 =       self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)

												snake_case_				:							List[Any]						 =       f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
												snake_case_				:							List[Any]						 =       tokenizer_r.encode_plus(
												    _SCREAMING_SNAKE_CASE		,				return_attention_mask=_SCREAMING_SNAKE_CASE		,				return_token_type_ids=_SCREAMING_SNAKE_CASE		,				return_offsets_mapping=_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE		,				)

												snake_case_				:							Optional[Any]						 =       tokenizer_r.do_lower_case if hasattr(_SCREAMING_SNAKE_CASE		,				"do_lower_case"				) else False
												snake_case_				:							Union[str, Any]						 =       (
												    [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), "A"),
												        ((1, 2), ","),
												        ((3, 5), "na"),
												        ((5, 6), "##ï"),
												        ((6, 8), "##ve"),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), "Allen"),
												        ((21, 23), "##NL"),
												        ((23, 24), "##P"),
												        ((25, 33), "sentence"),
												        ((33, 34), "."),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												    if not do_lower_case
												    else [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), "a"),
												        ((1, 2), ","),
												        ((3, 8), "naive"),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), "allen"),
												        ((21, 23), "##nl"),
												        ((23, 24), "##p"),
												        ((25, 33), "sentence"),
												        ((33, 34), "."),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												)

												self.assertEqual(
												    [e[1] for e in expected_results]		,				tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]				)				)
												self.assertEqual([e[0] for e in expected_results]		,				tokens["offset_mapping"]				)






						def       _lowerCAmelCase       (	self				) ->				Optional[int]:
								snake_case_				:							Union[str, Any]						 =       ["的", "人", "有"]
								snake_case_				:							int						 =       "".join(_SCREAMING_SNAKE_CASE				)
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
												snake_case_				:							int						 =       True
												snake_case_				:							Union[str, Any]						 =       self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)
												snake_case_				:							int						 =       self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)

												snake_case_				:							List[str]						 =       tokenizer_p.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
												snake_case_				:							Union[str, Any]						 =       tokenizer_r.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)

												snake_case_				:							Tuple						 =       tokenizer_r.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)
												snake_case_				:							int						 =       tokenizer_p.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)

												# it is expected that each Chinese character is not preceded by "##"
												self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)
												self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)

												snake_case_				:							Optional[int]						 =       False
												snake_case_				:							Dict						 =       self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)
												snake_case_				:							List[Any]						 =       self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE		,				**_SCREAMING_SNAKE_CASE				)

												snake_case_				:							str						 =       tokenizer_r.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)
												snake_case_				:							Any						 =       tokenizer_p.encode(_SCREAMING_SNAKE_CASE		,				add_special_tokens=_SCREAMING_SNAKE_CASE				)

												snake_case_				:							Any						 =       tokenizer_r.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)
												snake_case_				:							Any						 =       tokenizer_p.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				)

												# it is expected that only the first Chinese character is not preceded by "##".
												snake_case_				:							int						 =       [
												    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_SCREAMING_SNAKE_CASE				)
												]
												self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)
												self.assertListEqual(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)
