
"""simple docstring"""
import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE_					     =	get_tests_dir('''fixtures/spiece.model''')
@require_sentencepiece
@require_tokenizers
class       lowerCAmelCase_		(				A__				,				unittest.TestCase ):



							'''simple docstring'''





							_snake_case						      =		DebertaVaTokenizer
							_snake_case						      =		DebertaVaTokenizerFast
							_snake_case						      =		True
							_snake_case						      =		True




							def    A__						( self				)       ->  Any:
													super().setUp()

													# We have a SentencePiece fixture for testing
													__lowerCAmelCase     =  DebertaVaTokenizer(snake_case_    ,				unk_token="""<unk>"""				)
													tokenizer.save_pretrained(self.tmpdirname				)




							def    A__						( self    ,				snake_case_				)       ->  str:
													__lowerCAmelCase     =  """this is a test"""
													__lowerCAmelCase     =  """this is a test"""
													return input_text, output_text




							def    A__						( self				)       ->  Dict:
													__lowerCAmelCase     =  """<pad>"""
													__lowerCAmelCase     =  0

													self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case_				)    ,				snake_case_				)
													self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case_				)    ,				snake_case_				)




							def    A__						( self				)       ->  str:
													__lowerCAmelCase     =  list(self.get_tokenizer().get_vocab().keys()				)
													self.assertEqual(vocab_keys[0]    ,				"""<pad>"""				)
													self.assertEqual(vocab_keys[1]    ,				"""<unk>"""				)
													self.assertEqual(vocab_keys[-1]    ,				"""[PAD]"""				)
													self.assertEqual(len(snake_case_				)    ,				30_001				)




							def    A__						( self				)       ->  List[str]:
													self.assertEqual(self.get_tokenizer().vocab_size    ,				30_000				)




							def    A__						( self				)       ->  Any:
													# fmt: off
													__lowerCAmelCase     =  """ \tHeLLo!how  \n Are yoU?  """
													__lowerCAmelCase     =  ["""▁hello""", """!""", """how""", """▁are""", """▁you""", """?"""]
													# fmt: on

													__lowerCAmelCase     =  DebertaVaTokenizer(snake_case_    ,				do_lower_case=snake_case_				)
													__lowerCAmelCase     =  tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)				)

													self.assertListEqual(snake_case_    ,				snake_case_				)

													__lowerCAmelCase     =  DebertaVaTokenizerFast(snake_case_    ,				do_lower_case=snake_case_				)
													__lowerCAmelCase     =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)				)

													self.assertListEqual(snake_case_    ,				snake_case_				)




							@unittest.skip("""There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."""				)
							def    A__						( self				)       ->  List[str]:
													pass




							@unittest.skip("""There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."""				)
							def    A__						( self				)       ->  List[Any]:
													pass




							def    A__						( self				)       ->  List[str]:
													# fmt: off
													__lowerCAmelCase     =  """I was born in 92000, and this is falsé."""
													__lowerCAmelCase     =  ["""▁""", """<unk>""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """▁""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """▁""", """.""", ]
													# fmt: on

													__lowerCAmelCase     =  DebertaVaTokenizer(snake_case_    ,				split_by_punct=snake_case_				)
													__lowerCAmelCase     =  tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)				)

													self.assertListEqual(snake_case_    ,				snake_case_				)

													__lowerCAmelCase     =  DebertaVaTokenizerFast(snake_case_    ,				split_by_punct=snake_case_				)
													__lowerCAmelCase     =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)				)

													self.assertListEqual(snake_case_    ,				snake_case_				)




							def    A__						( self				)       ->  Union[str, Any]:
													# fmt: off
													__lowerCAmelCase     =  """I was born in 92000, and this is falsé."""
													__lowerCAmelCase     =  ["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """▁""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """▁""", """.""", ]
													# fmt: on

													__lowerCAmelCase     =  DebertaVaTokenizer(snake_case_    ,				do_lower_case=snake_case_    ,				split_by_punct=snake_case_				)
													__lowerCAmelCase     =  tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)				)
													self.assertListEqual(snake_case_    ,				snake_case_				)

													__lowerCAmelCase     =  DebertaVaTokenizerFast(snake_case_    ,				do_lower_case=snake_case_    ,				split_by_punct=snake_case_				)
													__lowerCAmelCase     =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)				)
													self.assertListEqual(snake_case_    ,				snake_case_				)




							def    A__						( self				)       ->  Optional[int]:
													# fmt: off
													__lowerCAmelCase     =  """I was born in 92000, and this is falsé."""
													__lowerCAmelCase     =  ["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """.""", ]
													# fmt: on

													__lowerCAmelCase     =  DebertaVaTokenizer(snake_case_    ,				do_lower_case=snake_case_    ,				split_by_punct=snake_case_				)
													__lowerCAmelCase     =  tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)				)

													self.assertListEqual(snake_case_    ,				snake_case_				)

													__lowerCAmelCase     =  DebertaVaTokenizerFast(snake_case_    ,				do_lower_case=snake_case_    ,				split_by_punct=snake_case_				)
													__lowerCAmelCase     =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)				)

													self.assertListEqual(snake_case_    ,				snake_case_				)




							def    A__						( self				)       ->  Union[str, Any]:
													# fmt: off
													__lowerCAmelCase     =  """I was born in 92000, and this is falsé."""
													__lowerCAmelCase     =  ["""▁""", """<unk>""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """▁""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """▁""", """.""", ]
													# fmt: on

													__lowerCAmelCase     =  DebertaVaTokenizer(snake_case_    ,				do_lower_case=snake_case_    ,				split_by_punct=snake_case_				)
													__lowerCAmelCase     =  tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)				)

													self.assertListEqual(snake_case_    ,				snake_case_				)

													__lowerCAmelCase     =  DebertaVaTokenizerFast(snake_case_    ,				do_lower_case=snake_case_    ,				split_by_punct=snake_case_				)
													__lowerCAmelCase     =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)				)

													self.assertListEqual(snake_case_    ,				snake_case_				)




							def    A__						( self				)       ->  List[str]:
													# fmt: off
													__lowerCAmelCase     =  """ \tHeLLo!how  \n Are yoU?  """
													__lowerCAmelCase     =  ["""▁""", """<unk>""", """e""", """<unk>""", """o""", """!""", """how""", """▁""", """<unk>""", """re""", """▁yo""", """<unk>""", """?"""]
													# fmt: on

													__lowerCAmelCase     =  DebertaVaTokenizer(snake_case_    ,				do_lower_case=snake_case_    ,				split_by_punct=snake_case_				)
													__lowerCAmelCase     =  tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)				)

													self.assertListEqual(snake_case_    ,				snake_case_				)

													__lowerCAmelCase     =  DebertaVaTokenizerFast(snake_case_    ,				do_lower_case=snake_case_    ,				split_by_punct=snake_case_				)
													__lowerCAmelCase     =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)				)

													self.assertListEqual(snake_case_    ,				snake_case_				)




							def    A__						( self				)       ->  Optional[Any]:
													__lowerCAmelCase     =  self.get_tokenizer()
													__lowerCAmelCase     =  self.get_rust_tokenizer()

													__lowerCAmelCase     =  """I was born in 92000, and this is falsé."""

													__lowerCAmelCase     =  tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)				)
													__lowerCAmelCase     =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)				)
													self.assertListEqual(snake_case_    ,				snake_case_				)

													__lowerCAmelCase     =  tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)
													__lowerCAmelCase     =  rust_tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)
													self.assertListEqual(snake_case_    ,				snake_case_				)

													__lowerCAmelCase     =  self.get_rust_tokenizer()
													__lowerCAmelCase     =  tokenizer.encode(snake_case_				)
													__lowerCAmelCase     =  rust_tokenizer.encode(snake_case_				)
													self.assertListEqual(snake_case_    ,				snake_case_				)




							def    A__						( self				)       ->  Union[str, Any]:
													__lowerCAmelCase     =  """This is a test"""
													__lowerCAmelCase     =  [13, 1, 4_398, 25, 21, 1_289]
													__lowerCAmelCase     =  ["""▁""", """T""", """his""", """▁is""", """▁a""", """▁test"""]
													__lowerCAmelCase     =  ["""▁""", """<unk>""", """his""", """▁is""", """▁a""", """▁test"""]

													__lowerCAmelCase     =  DebertaVaTokenizer(snake_case_    ,				keep_accents=snake_case_				)
													__lowerCAmelCase     =  DebertaVaTokenizerFast(snake_case_    ,				keep_accents=snake_case_				)

													__lowerCAmelCase     =  tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)
													self.assertListEqual(snake_case_    ,				snake_case_				)
													__lowerCAmelCase     =  tokenizer.tokenize(snake_case_				)
													self.assertListEqual(snake_case_    ,				snake_case_				)
													__lowerCAmelCase     =  tokenizer.convert_ids_to_tokens(snake_case_				)
													self.assertListEqual(snake_case_    ,				snake_case_				)

													__lowerCAmelCase     =  rust_tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)
													self.assertListEqual(snake_case_    ,				snake_case_				)
													__lowerCAmelCase     =  rust_tokenizer.tokenize(snake_case_				)
													self.assertListEqual(snake_case_    ,				snake_case_				)
													__lowerCAmelCase     =  rust_tokenizer.convert_ids_to_tokens(snake_case_				)
													self.assertListEqual(snake_case_    ,				snake_case_				)

													# fmt: off
													__lowerCAmelCase     =  """I was born in 92000, and this is falsé."""
													__lowerCAmelCase     =  [13, 1, 23, 386, 19, 561, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]
													__lowerCAmelCase     =  ["""▁""", """I""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """é""", """.""", ]
													__lowerCAmelCase     =  ["""▁""", """<unk>""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """.""", ]
													# fmt: on

													__lowerCAmelCase     =  tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)
													self.assertListEqual(snake_case_    ,				snake_case_				)
													__lowerCAmelCase     =  tokenizer.tokenize(snake_case_				)
													self.assertListEqual(snake_case_    ,				snake_case_				)
													__lowerCAmelCase     =  tokenizer.convert_ids_to_tokens(snake_case_				)
													self.assertListEqual(snake_case_    ,				snake_case_				)

													__lowerCAmelCase     =  rust_tokenizer.encode(snake_case_    ,				add_special_tokens=snake_case_				)
													self.assertListEqual(snake_case_    ,				snake_case_				)
													__lowerCAmelCase     =  rust_tokenizer.tokenize(snake_case_				)
													self.assertListEqual(snake_case_    ,				snake_case_				)
													__lowerCAmelCase     =  rust_tokenizer.convert_ids_to_tokens(snake_case_				)
													self.assertListEqual(snake_case_    ,				snake_case_				)




							def    A__						( self				)       ->  int:
													__lowerCAmelCase     =  DebertaVaTokenizer(snake_case_				)

													__lowerCAmelCase     =  tokenizer.encode("""sequence builders"""				)
													__lowerCAmelCase     =  tokenizer.encode("""multi-sequence build"""				)

													__lowerCAmelCase     =  tokenizer.build_inputs_with_special_tokens(snake_case_				)
													__lowerCAmelCase     =  tokenizer.build_inputs_with_special_tokens(snake_case_    ,				snake_case_				)

													self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]    ,				snake_case_				)
													self.assertEqual(
													    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]    ,				snake_case_    ,				)




							@slow
							def    A__						( self				)       ->  str:
													# fmt: off
													__lowerCAmelCase     =  {"""input_ids""": [[1, 39_867, 36, 19_390, 486, 27, 35_052, 81_436, 18, 60_685, 1_225, 7, 35_052, 81_436, 18, 9_367, 16_899, 18, 15_937, 53, 594, 773, 18, 16_287, 30_465, 36, 15_937, 6, 41_139, 38, 36_979, 60_763, 191, 6, 34_132, 99, 6, 50_538, 390, 43_230, 6, 34_132, 2_779, 20_850, 14, 699, 1_072, 1_194, 36, 382, 10_901, 53, 7, 699, 1_072, 2_084, 36, 20_422, 630, 53, 19, 105, 3_049, 1_896, 1_053, 16_899, 1_506, 11, 37_978, 4_243, 7, 1_237, 31_869, 200, 16_566, 654, 6, 35_052, 81_436, 7, 55_630, 13_593, 4, 2], [1, 26, 15_011, 13, 667, 8, 1_053, 18, 23_611, 1_237, 72_356, 12_820, 34, 104_134, 1_209, 35, 13_313, 6_627, 21, 202, 347, 7, 164, 2_399, 11, 46, 4_485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_232, 2_864, 15_785, 14_951, 105, 5, 8_581, 1_250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
													# fmt: on

													self.tokenizer_integration_test_util(
													    expected_encoding=snake_case_    ,				model_name="""microsoft/deberta-v2-xlarge"""    ,				revision="""ad6e42c1532ddf3a15c39246b63f5559d558b670"""    ,				)


"""simple docstring"""
from typing import List

from .keymap import KEYMAP, get_character

def  lowercase     (_lowerCAmelCase						):

						def decorator(_lowerCAmelCase						):
												__lowerCAmelCase     =  getattr(_lowerCAmelCase ,    """handle_key""" ,    []						)
												handle += [key]
												setattr(_lowerCAmelCase ,    """handle_key""" ,    _lowerCAmelCase						)
												return func

						return decorator

def  lowercase     (*_lowerCAmelCase						):

						def decorator(_lowerCAmelCase						):
												__lowerCAmelCase     =  getattr(_lowerCAmelCase ,    """handle_key""" ,    []						)
												handle += keys
												setattr(_lowerCAmelCase ,    """handle_key""" ,    _lowerCAmelCase						)
												return func

						return decorator
class       lowerCAmelCase_		(				A__ ):



							'''simple docstring'''





							def __new__( cls    ,				snake_case_    ,				snake_case_    ,				snake_case_				)       ->  Any:
													__lowerCAmelCase     =  super().__new__(cls    ,				snake_case_    ,				snake_case_    ,				snake_case_				)
													if not hasattr(snake_case_    ,				"""key_handler"""				):
																			setattr(snake_case_    ,				"""key_handler"""    ,				{}				)
													setattr(snake_case_    ,				"""handle_input"""    ,				KeyHandler.handle_input				)

													for value in attrs.values():
																			__lowerCAmelCase     =  getattr(snake_case_    ,				"""handle_key"""    ,				[]				)
																			for key in handled_keys:
																									__lowerCAmelCase     =  value
													return new_cls




							@staticmethod
							def    A__						( cls				)       ->  Tuple:
													__lowerCAmelCase     =  get_character()
													if char != KEYMAP["undefined"]:
																			__lowerCAmelCase     =  ord(snake_case_				)
													__lowerCAmelCase     =  cls.key_handler.get(snake_case_				)
													if handler:
																			__lowerCAmelCase     =  char
																			return handler(cls				)
													else:
																			return None

def  lowercase     (cls						):
						return KeyHandler(cls.__name__ ,    cls.__bases__ ,    cls.__dict__.copy()						)
