






'''simple docstring'''




import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCamelCase :       str 				=       get_tests_dir('''fixtures/spiece.model''')







@require_sentencepiece
@require_tokenizers
class  A_						(A_    ,       unittest.TestCase				):
							"""simple docstring"""




							a__ 		=						DebertaVaTokenizer
							a__ 		=						DebertaVaTokenizerFast
							a__ 		=						True
							a__ 		=						True

							def 				_A						(		self      :Tuple					)					->	Union[str, Any]:


										'''simple docstring'''




										super().setUp()

										# We have a SentencePiece fixture for testing
										snake_case_					:       Union[str, Any] =							DebertaVaTokenizer(snake_case__							,				unk_token="<unk>"					)
										tokenizer.save_pretrained(self.tmpdirname					)

							def 				_A						(		self      :Optional[int]							,				lowerCAmelCase__      :Optional[Any]					)					->	Any:


										'''simple docstring'''




										snake_case_					:       Any =							"this is a test"
										snake_case_					:       List[Any] =							"this is a test"
										return input_text, output_text

							def 				_A						(		self      :Any					)					->	Optional[Any]:


										'''simple docstring'''




										snake_case_					:       Dict =							"<pad>"
										snake_case_					:       int =							0

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__					)							,				snake_case__					)
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__					)							,				snake_case__					)

							def 				_A						(		self      :Union[str, Any]					)					->	int:


										'''simple docstring'''




										snake_case_					:       Union[str, Any] =							list(self.get_tokenizer().get_vocab().keys()					)
										self.assertEqual(vocab_keys[0]							,				"<pad>"					)
										self.assertEqual(vocab_keys[1]							,				"<unk>"					)
										self.assertEqual(vocab_keys[-1]							,				"[PAD]"					)
										self.assertEqual(len(snake_case__					)							,				30_001					)

							def 				_A						(		self      :Any					)					->	Tuple:


										'''simple docstring'''




										self.assertEqual(self.get_tokenizer().vocab_size							,				30_000					)

							def 				_A						(		self      :int					)					->	Optional[int]:


										'''simple docstring'''




										snake_case_					:       Tuple =							" \tHeLLo!how  \n Are yoU?  "
										snake_case_					:       int =							["▁hello", "!", "how", "▁are", "▁you", "?"]
										# fmt: on

										snake_case_					:       Any =							DebertaVaTokenizer(snake_case__							,				do_lower_case=snake_case__					)
										snake_case_					:       List[Any] =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)					)

										self.assertListEqual(snake_case__							,				snake_case__					)

										snake_case_					:       Optional[Any] =							DebertaVaTokenizerFast(snake_case__							,				do_lower_case=snake_case__					)
										snake_case_					:       Union[str, Any] =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)					)

										self.assertListEqual(snake_case__							,				snake_case__					)

							@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."					)
							def 				_A						(		self      :Optional[Any]					)					->	Optional[Any]:


										'''simple docstring'''




										pass

							@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."					)
							def 				_A						(		self      :int					)					->	List[str]:


										'''simple docstring'''




										pass

							def 				_A						(		self      :Tuple					)					->	str:


										'''simple docstring'''




										snake_case_					:       Optional[Any] =							"I was born in 92000, and this is falsé."
										snake_case_					:       Any =							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
										# fmt: on

										snake_case_					:       str =							DebertaVaTokenizer(snake_case__							,				split_by_punct=snake_case__					)
										snake_case_					:       Union[str, Any] =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)					)

										self.assertListEqual(snake_case__							,				snake_case__					)

										snake_case_					:       Any =							DebertaVaTokenizerFast(snake_case__							,				split_by_punct=snake_case__					)
										snake_case_					:       str =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)					)

										self.assertListEqual(snake_case__							,				snake_case__					)

							def 				_A						(		self      :List[Any]					)					->	List[str]:


										'''simple docstring'''




										snake_case_					:       Union[str, Any] =							"I was born in 92000, and this is falsé."
										snake_case_					:       Any =							["▁i", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
										# fmt: on

										snake_case_					:       Union[str, Any] =							DebertaVaTokenizer(snake_case__							,				do_lower_case=snake_case__							,				split_by_punct=snake_case__					)
										snake_case_					:       Tuple =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)					)
										self.assertListEqual(snake_case__							,				snake_case__					)

										snake_case_					:       Dict =							DebertaVaTokenizerFast(snake_case__							,				do_lower_case=snake_case__							,				split_by_punct=snake_case__					)
										snake_case_					:       List[str] =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)					)
										self.assertListEqual(snake_case__							,				snake_case__					)

							def 				_A						(		self      :Dict					)					->	Dict:


										'''simple docstring'''




										snake_case_					:       List[Any] =							"I was born in 92000, and this is falsé."
										snake_case_					:       Dict =							["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
										# fmt: on

										snake_case_					:       int =							DebertaVaTokenizer(snake_case__							,				do_lower_case=snake_case__							,				split_by_punct=snake_case__					)
										snake_case_					:       Optional[Any] =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)					)

										self.assertListEqual(snake_case__							,				snake_case__					)

										snake_case_					:       Optional[int] =							DebertaVaTokenizerFast(snake_case__							,				do_lower_case=snake_case__							,				split_by_punct=snake_case__					)
										snake_case_					:       List[str] =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)					)

										self.assertListEqual(snake_case__							,				snake_case__					)

							def 				_A						(		self      :List[str]					)					->	List[Any]:


										'''simple docstring'''




										snake_case_					:       Any =							"I was born in 92000, and this is falsé."
										snake_case_					:       Optional[int] =							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
										# fmt: on

										snake_case_					:       Tuple =							DebertaVaTokenizer(snake_case__							,				do_lower_case=snake_case__							,				split_by_punct=snake_case__					)
										snake_case_					:       Tuple =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)					)

										self.assertListEqual(snake_case__							,				snake_case__					)

										snake_case_					:       str =							DebertaVaTokenizerFast(snake_case__							,				do_lower_case=snake_case__							,				split_by_punct=snake_case__					)
										snake_case_					:       Any =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)					)

										self.assertListEqual(snake_case__							,				snake_case__					)

							def 				_A						(		self      :Union[str, Any]					)					->	Optional[Any]:


										'''simple docstring'''




										snake_case_					:       Union[str, Any] =							" \tHeLLo!how  \n Are yoU?  "
										snake_case_					:       Dict =							["▁", "<unk>", "e", "<unk>", "o", "!", "how", "▁", "<unk>", "re", "▁yo", "<unk>", "?"]
										# fmt: on

										snake_case_					:       Tuple =							DebertaVaTokenizer(snake_case__							,				do_lower_case=snake_case__							,				split_by_punct=snake_case__					)
										snake_case_					:       Dict =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)					)

										self.assertListEqual(snake_case__							,				snake_case__					)

										snake_case_					:       Any =							DebertaVaTokenizerFast(snake_case__							,				do_lower_case=snake_case__							,				split_by_punct=snake_case__					)
										snake_case_					:       Any =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)					)

										self.assertListEqual(snake_case__							,				snake_case__					)

							def 				_A						(		self      :Any					)					->	Tuple:


										'''simple docstring'''




										snake_case_					:       Optional[Any] =							self.get_tokenizer()
										snake_case_					:       Union[str, Any] =							self.get_rust_tokenizer()

										snake_case_					:       Any =							"I was born in 92000, and this is falsé."

										snake_case_					:       Optional[int] =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)					)
										snake_case_					:       List[Any] =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)					)
										self.assertListEqual(snake_case__							,				snake_case__					)

										snake_case_					:       List[str] =							tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)
										snake_case_					:       List[str] =							rust_tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)
										self.assertListEqual(snake_case__							,				snake_case__					)

										snake_case_					:       Optional[Any] =							self.get_rust_tokenizer()
										snake_case_					:       List[str] =							tokenizer.encode(snake_case__					)
										snake_case_					:       Optional[Any] =							rust_tokenizer.encode(snake_case__					)
										self.assertListEqual(snake_case__							,				snake_case__					)

							def 				_A						(		self      :str					)					->	Optional[Any]:


										'''simple docstring'''




										snake_case_					:       Any =							"This is a test"
										snake_case_					:       Optional[int] =							[13, 1, 4_398, 25, 21, 1_289]
										snake_case_					:       int =							["▁", "T", "his", "▁is", "▁a", "▁test"]
										snake_case_					:       List[str] =							["▁", "<unk>", "his", "▁is", "▁a", "▁test"]

										snake_case_					:       Optional[int] =							DebertaVaTokenizer(snake_case__							,				keep_accents=snake_case__					)
										snake_case_					:       Dict =							DebertaVaTokenizerFast(snake_case__							,				keep_accents=snake_case__					)

										snake_case_					:       List[Any] =							tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)
										self.assertListEqual(snake_case__							,				snake_case__					)
										snake_case_					:       Optional[int] =							tokenizer.tokenize(snake_case__					)
										self.assertListEqual(snake_case__							,				snake_case__					)
										snake_case_					:       Optional[Any] =							tokenizer.convert_ids_to_tokens(snake_case__					)
										self.assertListEqual(snake_case__							,				snake_case__					)

										snake_case_					:       Optional[int] =							rust_tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)
										self.assertListEqual(snake_case__							,				snake_case__					)
										snake_case_					:       Optional[int] =							rust_tokenizer.tokenize(snake_case__					)
										self.assertListEqual(snake_case__							,				snake_case__					)
										snake_case_					:       Dict =							rust_tokenizer.convert_ids_to_tokens(snake_case__					)
										self.assertListEqual(snake_case__							,				snake_case__					)

										# fmt: off
										snake_case_					:       Optional[int] =							"I was born in 92000, and this is falsé."
										snake_case_					:       Tuple =							[13, 1, 23, 386, 19, 561, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]
										snake_case_					:       List[Any] =							["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", ".", ]
										snake_case_					:       int =							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
										# fmt: on

										snake_case_					:       int =							tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)
										self.assertListEqual(snake_case__							,				snake_case__					)
										snake_case_					:       Optional[int] =							tokenizer.tokenize(snake_case__					)
										self.assertListEqual(snake_case__							,				snake_case__					)
										snake_case_					:       List[str] =							tokenizer.convert_ids_to_tokens(snake_case__					)
										self.assertListEqual(snake_case__							,				snake_case__					)

										snake_case_					:       int =							rust_tokenizer.encode(snake_case__							,				add_special_tokens=snake_case__					)
										self.assertListEqual(snake_case__							,				snake_case__					)
										snake_case_					:       int =							rust_tokenizer.tokenize(snake_case__					)
										self.assertListEqual(snake_case__							,				snake_case__					)
										snake_case_					:       int =							rust_tokenizer.convert_ids_to_tokens(snake_case__					)
										self.assertListEqual(snake_case__							,				snake_case__					)

							def 				_A						(		self      :Optional[int]					)					->	Tuple:


										'''simple docstring'''




										snake_case_					:       Dict =							DebertaVaTokenizer(snake_case__					)

										snake_case_					:       str =							tokenizer.encode("sequence builders"					)
										snake_case_					:       List[str] =							tokenizer.encode("multi-sequence build"					)

										snake_case_					:       str =							tokenizer.build_inputs_with_special_tokens(snake_case__					)
										snake_case_					:       Dict =							tokenizer.build_inputs_with_special_tokens(snake_case__							,				snake_case__					)

										self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]							,				snake_case__					)
										self.assertEqual(
										    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]							,				snake_case__							,				)






							@slow
							def 				_A						(		self      :str					)					->	Dict:


										'''simple docstring'''




										snake_case_					:       Any =							{"input_ids": [[1, 39_867, 36, 19_390, 486, 27, 35_052, 81_436, 18, 60_685, 1_225, 7, 35_052, 81_436, 18, 9_367, 16_899, 18, 15_937, 53, 594, 773, 18, 16_287, 30_465, 36, 15_937, 6, 41_139, 38, 36_979, 60_763, 191, 6, 34_132, 99, 6, 50_538, 390, 43_230, 6, 34_132, 2_779, 20_850, 14, 699, 1_072, 1_194, 36, 382, 10_901, 53, 7, 699, 1_072, 2_084, 36, 20_422, 630, 53, 19, 105, 3_049, 1_896, 1_053, 16_899, 1_506, 11, 37_978, 4_243, 7, 1_237, 31_869, 200, 16_566, 654, 6, 35_052, 81_436, 7, 55_630, 13_593, 4, 2], [1, 26, 15_011, 13, 667, 8, 1_053, 18, 23_611, 1_237, 72_356, 12_820, 34, 104_134, 1_209, 35, 13_313, 6_627, 21, 202, 347, 7, 164, 2_399, 11, 46, 4_485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_232, 2_864, 15_785, 14_951, 105, 5, 8_581, 1_250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=snake_case__							,				model_name="microsoft/deberta-v2-xlarge"							,				revision="ad6e42c1532ddf3a15c39246b63f5559d558b670"							,				)






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


__lowerCamelCase				=			{
    """configuration_chinese_clip""": [
        """CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """ChineseCLIPConfig""",
        """ChineseCLIPOnnxConfig""",
        """ChineseCLIPTextConfig""",
        """ChineseCLIPVisionConfig""",
    ],
    """processing_chinese_clip""": ["""ChineseCLIPProcessor"""],
}

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __lowerCamelCase				=			["""ChineseCLIPFeatureExtractor"""]
   __lowerCamelCase				=			["""ChineseCLIPImageProcessor"""]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __lowerCamelCase				=			[
       """CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """ChineseCLIPModel""",
       """ChineseCLIPPreTrainedModel""",
       """ChineseCLIPTextModel""",
       """ChineseCLIPVisionModel""",
   ]

if TYPE_CHECKING:
   from .configuration_chinese_clip import (
       CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
       ChineseCLIPConfig,
       ChineseCLIPOnnxConfig,
       ChineseCLIPTextConfig,
       ChineseCLIPVisionConfig,
   )
   from .processing_chinese_clip import ChineseCLIPProcessor

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .feature_extraction_chinese_clip import ChineseCLIPFeatureExtractor, ChineseCLIPImageProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_chinese_clip import (
          CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
          ChineseCLIPModel,
          ChineseCLIPPreTrainedModel,
          ChineseCLIPTextModel,
          ChineseCLIPVisionModel,
      )

else:
   import sys

   __lowerCamelCase				=			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





