



import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE__      					=     get_tests_dir("fixtures/spiece.model")


@require_sentencepiece
@require_tokenizers
class 				snake_case (UpperCamelCase		,      unittest.TestCase   ):
       lowerCAmelCase__  :List[Any]						   =					DebertaVaTokenizer
       lowerCAmelCase__  :Tuple						   =					DebertaVaTokenizerFast
       lowerCAmelCase__  :Tuple						   =					True
       lowerCAmelCase__  :str						   =					True



       def 							_a      (				self      )  ->							Union[str, Any]:
           super().setUp()

           # We have a SentencePiece fixture for testing
           lowercase__		       =						DebertaVaTokenizer(UpperCAmelCase_     ,unk_token="<unk>"      )
           tokenizer.save_pretrained(self.tmpdirname      )



       def 							_a      (				self     ,UpperCAmelCase_      )  ->							List[str]:
           lowercase__		       =						"this is a test"
           lowercase__		       =						"this is a test"
           return input_text, output_text



       def 							_a      (				self      )  ->							Tuple:
           lowercase__		       =						"<pad>"
           lowercase__		       =						0

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase_      )     ,UpperCAmelCase_      )
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase_      )     ,UpperCAmelCase_      )



       def 							_a      (				self      )  ->							Optional[Any]:
           lowercase__		       =						list(self.get_tokenizer().get_vocab().keys()      )
           self.assertEqual(vocab_keys[0]     ,"<pad>"      )
           self.assertEqual(vocab_keys[1]     ,"<unk>"      )
           self.assertEqual(vocab_keys[-1]     ,"[PAD]"      )
           self.assertEqual(len(UpperCAmelCase_      )     ,30_001      )



       def 							_a      (				self      )  ->							List[str]:
           self.assertEqual(self.get_tokenizer().vocab_size     ,30_000      )



       def 							_a      (				self      )  ->							Union[str, Any]:
           # fmt: off
           lowercase__		       =						" \tHeLLo!how  \n Are yoU?  "
           lowercase__		       =						["▁hello", "!", "how", "▁are", "▁you", "?"]
           # fmt: on

           lowercase__		       =						DebertaVaTokenizer(UpperCAmelCase_     ,do_lower_case=UpperCAmelCase_      )
           lowercase__		       =						tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )      )

           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           lowercase__		       =						DebertaVaTokenizerFast(UpperCAmelCase_     ,do_lower_case=UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )      )

           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )



       @unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."      )
       def 							_a      (				self      )  ->							Optional[int]:
           pass



       @unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."      )
       def 							_a      (				self      )  ->							Any:
           pass



       def 							_a      (				self      )  ->							Tuple:
           # fmt: off
           lowercase__		       =						"I was born in 92000, and this is falsé."
           lowercase__		       =						["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
           # fmt: on

           lowercase__		       =						DebertaVaTokenizer(UpperCAmelCase_     ,split_by_punct=UpperCAmelCase_      )
           lowercase__		       =						tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )      )

           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           lowercase__		       =						DebertaVaTokenizerFast(UpperCAmelCase_     ,split_by_punct=UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )      )

           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )



       def 							_a      (				self      )  ->							Optional[Any]:
           # fmt: off
           lowercase__		       =						"I was born in 92000, and this is falsé."
           lowercase__		       =						["▁i", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
           # fmt: on

           lowercase__		       =						DebertaVaTokenizer(UpperCAmelCase_     ,do_lower_case=UpperCAmelCase_     ,split_by_punct=UpperCAmelCase_      )
           lowercase__		       =						tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           lowercase__		       =						DebertaVaTokenizerFast(UpperCAmelCase_     ,do_lower_case=UpperCAmelCase_     ,split_by_punct=UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )



       def 							_a      (				self      )  ->							int:
           # fmt: off
           lowercase__		       =						"I was born in 92000, and this is falsé."
           lowercase__		       =						["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
           # fmt: on

           lowercase__		       =						DebertaVaTokenizer(UpperCAmelCase_     ,do_lower_case=UpperCAmelCase_     ,split_by_punct=UpperCAmelCase_      )
           lowercase__		       =						tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )      )

           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           lowercase__		       =						DebertaVaTokenizerFast(UpperCAmelCase_     ,do_lower_case=UpperCAmelCase_     ,split_by_punct=UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )      )

           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )



       def 							_a      (				self      )  ->							Optional[int]:
           # fmt: off
           lowercase__		       =						"I was born in 92000, and this is falsé."
           lowercase__		       =						["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
           # fmt: on

           lowercase__		       =						DebertaVaTokenizer(UpperCAmelCase_     ,do_lower_case=UpperCAmelCase_     ,split_by_punct=UpperCAmelCase_      )
           lowercase__		       =						tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )      )

           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           lowercase__		       =						DebertaVaTokenizerFast(UpperCAmelCase_     ,do_lower_case=UpperCAmelCase_     ,split_by_punct=UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )      )

           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )



       def 							_a      (				self      )  ->							int:
           # fmt: off
           lowercase__		       =						" \tHeLLo!how  \n Are yoU?  "
           lowercase__		       =						["▁", "<unk>", "e", "<unk>", "o", "!", "how", "▁", "<unk>", "re", "▁yo", "<unk>", "?"]
           # fmt: on

           lowercase__		       =						DebertaVaTokenizer(UpperCAmelCase_     ,do_lower_case=UpperCAmelCase_     ,split_by_punct=UpperCAmelCase_      )
           lowercase__		       =						tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )      )

           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           lowercase__		       =						DebertaVaTokenizerFast(UpperCAmelCase_     ,do_lower_case=UpperCAmelCase_     ,split_by_punct=UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )      )

           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )



       def 							_a      (				self      )  ->							List[str]:
           lowercase__		       =						self.get_tokenizer()
           lowercase__		       =						self.get_rust_tokenizer()

           lowercase__		       =						"I was born in 92000, and this is falsé."

           lowercase__		       =						tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )      )
           lowercase__		       =						rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           lowercase__		       =						tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           lowercase__		       =						self.get_rust_tokenizer()
           lowercase__		       =						tokenizer.encode(UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.encode(UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )



       def 							_a      (				self      )  ->							List[str]:
           lowercase__		       =						"This is a test"
           lowercase__		       =						[13, 1, 4_398, 25, 21, 1_289]
           lowercase__		       =						["▁", "T", "his", "▁is", "▁a", "▁test"]
           lowercase__		       =						["▁", "<unk>", "his", "▁is", "▁a", "▁test"]

           lowercase__		       =						DebertaVaTokenizer(UpperCAmelCase_     ,keep_accents=UpperCAmelCase_      )
           lowercase__		       =						DebertaVaTokenizerFast(UpperCAmelCase_     ,keep_accents=UpperCAmelCase_      )

           lowercase__		       =						tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )
           lowercase__		       =						tokenizer.tokenize(UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )
           lowercase__		       =						tokenizer.convert_ids_to_tokens(UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           lowercase__		       =						rust_tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.tokenize(UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.convert_ids_to_tokens(UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           # fmt: off
           lowercase__		       =						"I was born in 92000, and this is falsé."
           lowercase__		       =						[13, 1, 23, 386, 19, 561, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]
           lowercase__		       =						["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", ".", ]
           lowercase__		       =						["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
           # fmt: on

           lowercase__		       =						tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )
           lowercase__		       =						tokenizer.tokenize(UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )
           lowercase__		       =						tokenizer.convert_ids_to_tokens(UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )

           lowercase__		       =						rust_tokenizer.encode(UpperCAmelCase_     ,add_special_tokens=UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.tokenize(UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )
           lowercase__		       =						rust_tokenizer.convert_ids_to_tokens(UpperCAmelCase_      )
           self.assertListEqual(UpperCAmelCase_     ,UpperCAmelCase_      )



       def 							_a      (				self      )  ->							Any:
           lowercase__		       =						DebertaVaTokenizer(UpperCAmelCase_      )

           lowercase__		       =						tokenizer.encode("sequence builders"      )
           lowercase__		       =						tokenizer.encode("multi-sequence build"      )

           lowercase__		       =						tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_      )
           lowercase__		       =						tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_     ,UpperCAmelCase_      )

           self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]     ,UpperCAmelCase_      )
           self.assertEqual(
               [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]     ,UpperCAmelCase_     ,)







       @slow
       def 							_a      (				self      )  ->							str:
           # fmt: off
           lowercase__		       =						{"input_ids": [[1, 39_867, 36, 19_390, 486, 27, 35_052, 81_436, 18, 60_685, 1_225, 7, 35_052, 81_436, 18, 9_367, 16_899, 18, 15_937, 53, 594, 773, 18, 16_287, 30_465, 36, 15_937, 6, 41_139, 38, 36_979, 60_763, 191, 6, 34_132, 99, 6, 50_538, 390, 43_230, 6, 34_132, 2_779, 20_850, 14, 699, 1_072, 1_194, 36, 382, 10_901, 53, 7, 699, 1_072, 2_084, 36, 20_422, 630, 53, 19, 105, 3_049, 1_896, 1_053, 16_899, 1_506, 11, 37_978, 4_243, 7, 1_237, 31_869, 200, 16_566, 654, 6, 35_052, 81_436, 7, 55_630, 13_593, 4, 2], [1, 26, 15_011, 13, 667, 8, 1_053, 18, 23_611, 1_237, 72_356, 12_820, 34, 104_134, 1_209, 35, 13_313, 6_627, 21, 202, 347, 7, 164, 2_399, 11, 46, 4_485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_232, 2_864, 15_785, 14_951, 105, 5, 8_581, 1_250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=UpperCAmelCase_     ,model_name="microsoft/deberta-v2-xlarge"     ,revision="ad6e42c1532ddf3a15c39246b63f5559d558b670"     ,)









'''simple docstring'''





import copy
import random

from transformers import CLIPTokenizer


class 				snake_case (UpperCamelCase   ):



       def __init__(				self     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )  ->							List[str]:
           super().__init__(*UpperCAmelCase_     ,**UpperCAmelCase_      )
           lowercase__		       =						{}



       def 							_a      (				self     ,UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )  ->							str:
           lowercase__		       =						super().add_tokens(UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )
           if num_added_tokens == 0:
               raise ValueError(
                   F'''The tokenizer already contains the token {placeholder_token}. Please pass a different'''
                   " `placeholder_token` that is not already in the tokenizer."      )



       def 							_a      (				self     ,UpperCAmelCase_     ,*UpperCAmelCase_     ,UpperCAmelCase_=1     ,**UpperCAmelCase_      )  ->							List[Any]:
           lowercase__		       =						[]
           if num_vec_per_token == 1:
               self.try_adding_tokens(UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )
               output.append(UpperCAmelCase_      )
           else:
               lowercase__		       =						[]
               for i in range(UpperCAmelCase_      ):
                   lowercase__		       =						placeholder_token + F'''_{i}'''
                   self.try_adding_tokens(UpperCAmelCase_     ,*UpperCAmelCase_     ,**UpperCAmelCase_      )
                   output.append(UpperCAmelCase_      )
        # handle cases where there is a new placeholder token that contains the current placeholder token but is larger
           for token in self.token_map:
               if token in placeholder_token:
                   raise ValueError(
                       F'''The tokenizer already has placeholder token {token} that can get confused with'''
                       F''' {placeholder_token}keep placeholder tokens independent'''      )
           lowercase__		       =						output



       def 							_a      (				self     ,UpperCAmelCase_     ,UpperCAmelCase_=False     ,UpperCAmelCase_=1.0      )  ->							Optional[int]:
           if isinstance(UpperCAmelCase_     ,UpperCAmelCase_      ):
               lowercase__		       =						[]
               for i in range(len(UpperCAmelCase_      )      ):
                   output.append(self.replace_placeholder_tokens_in_text(text[i]     ,vector_shuffle=UpperCAmelCase_      )      )
               return output
           for placeholder_token in self.token_map:
               if placeholder_token in text:
                   lowercase__		       =						self.token_map[placeholder_token]
                   lowercase__		       =						tokens[: 1 + int(len(UpperCAmelCase_      ) * prop_tokens_to_load      )]
                   if vector_shuffle:
                       lowercase__		       =						copy.copy(UpperCAmelCase_      )
                       random.shuffle(UpperCAmelCase_      )
                   lowercase__		       =						text.replace(UpperCAmelCase_     ," ".join(UpperCAmelCase_      )      )
           return text



       def __call__(				self     ,UpperCAmelCase_     ,*UpperCAmelCase_     ,UpperCAmelCase_=False     ,UpperCAmelCase_=1.0     ,**UpperCAmelCase_      )  ->							int:
           return super().__call__(
               self.replace_placeholder_tokens_in_text(
                   UpperCAmelCase_     ,vector_shuffle=UpperCAmelCase_     ,prop_tokens_to_load=UpperCAmelCase_      )     ,*UpperCAmelCase_     ,**UpperCAmelCase_     ,)







       def 							_a      (				self     ,UpperCAmelCase_     ,*UpperCAmelCase_     ,UpperCAmelCase_=False     ,UpperCAmelCase_=1.0     ,**UpperCAmelCase_      )  ->							Optional[int]:
           return super().encode(
               self.replace_placeholder_tokens_in_text(
                   UpperCAmelCase_     ,vector_shuffle=UpperCAmelCase_     ,prop_tokens_to_load=UpperCAmelCase_      )     ,*UpperCAmelCase_     ,**UpperCAmelCase_     ,)




