





"""simple docstring"""






import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


A			      =	get_tests_dir('''fixtures/spiece.model''')




@require_sentencepiece
@require_tokenizers
class 					__lowercase      (      a_ ,   unittest.TestCase				):







					'''simple docstring'''




					__lowerCAmelCase	=	DebertaVaTokenizer
					__lowerCAmelCase	=	DebertaVaTokenizerFast
					__lowerCAmelCase	=	True
					__lowerCAmelCase	=	True







					def 			_lowerCamelCase		(    self  ):
									super().setUp()

									# We have a SentencePiece fixture for testing
									__a			:    str      							=	DebertaVaTokenizer(_lowercase						,      unk_token='''<unk>'''  )
									tokenizer.save_pretrained(self.tmpdirname  )







					def 			_lowerCamelCase		(    self						,      _UpperCAmelCase  ):
									__a			:    Optional[Any]      							=	"""this is a test"""
									__a			:    str      							=	"""this is a test"""
									return input_text, output_text







					def 			_lowerCamelCase		(    self  ):
									__a			:    Dict      							=	"""<pad>"""
									__a			:    Union[str, Any]      							=	0

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowercase  )						,      _lowercase  )
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowercase  )						,      _lowercase  )







					def 			_lowerCamelCase		(    self  ):
									__a			:    List[Any]      							=	list(self.get_tokenizer().get_vocab().keys()  )
									self.assertEqual(vocab_keys[0]						,      '''<pad>'''  )
									self.assertEqual(vocab_keys[1]						,      '''<unk>'''  )
									self.assertEqual(vocab_keys[-1]						,      '''[PAD]'''  )
									self.assertEqual(len(_lowercase  )						,      30001  )







					def 			_lowerCamelCase		(    self  ):
									self.assertEqual(self.get_tokenizer().vocab_size						,      30000  )







					def 			_lowerCamelCase		(    self  ):
									# fmt: off
									__a			:    Tuple      							=	""" \tHeLLo!how  \n Are yoU?  """
									__a			:    Tuple      							=	["""▁hello""", """!""", """how""", """▁are""", """▁you""", """?"""]
									# fmt: on

									__a			:    Optional[Any]      							=	DebertaVaTokenizer(_lowercase						,      do_lower_case=_lowercase  )
									__a			:    Tuple      							=	tokenizer.convert_ids_to_tokens(tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )  )

									self.assertListEqual(_lowercase						,      _lowercase  )

									__a			:    Any      							=	DebertaVaTokenizerFast(_lowercase						,      do_lower_case=_lowercase  )
									__a			:    str      							=	rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )  )

									self.assertListEqual(_lowercase						,      _lowercase  )







					@unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''  )
					def 			_lowerCamelCase		(    self  ):
									pass







					@unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''  )
					def 			_lowerCamelCase		(    self  ):
									pass







					def 			_lowerCamelCase		(    self  ):
									# fmt: off
									__a			:    Dict      							=	"""I was born in 92000, and this is falsé."""
									__a			:    Union[str, Any]      							=	["""▁""", """<unk>""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """▁""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """▁""", """.""", ]
									# fmt: on

									__a			:    Dict      							=	DebertaVaTokenizer(_lowercase						,      split_by_punct=_lowercase  )
									__a			:    Optional[Any]      							=	tokenizer.convert_ids_to_tokens(tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )  )

									self.assertListEqual(_lowercase						,      _lowercase  )

									__a			:    str      							=	DebertaVaTokenizerFast(_lowercase						,      split_by_punct=_lowercase  )
									__a			:    List[str]      							=	rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )  )

									self.assertListEqual(_lowercase						,      _lowercase  )







					def 			_lowerCamelCase		(    self  ):
									# fmt: off
									__a			:    Union[str, Any]      							=	"""I was born in 92000, and this is falsé."""
									__a			:    Optional[int]      							=	["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """▁""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """▁""", """.""", ]
									# fmt: on

									__a			:    Any      							=	DebertaVaTokenizer(_lowercase						,      do_lower_case=_lowercase						,      split_by_punct=_lowercase  )
									__a			:    Union[str, Any]      							=	tokenizer.convert_ids_to_tokens(tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )  )
									self.assertListEqual(_lowercase						,      _lowercase  )

									__a			:    str      							=	DebertaVaTokenizerFast(_lowercase						,      do_lower_case=_lowercase						,      split_by_punct=_lowercase  )
									__a			:    Optional[Any]      							=	rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )  )
									self.assertListEqual(_lowercase						,      _lowercase  )







					def 			_lowerCamelCase		(    self  ):
									# fmt: off
									__a			:    List[str]      							=	"""I was born in 92000, and this is falsé."""
									__a			:    Any      							=	["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """.""", ]
									# fmt: on

									__a			:    int      							=	DebertaVaTokenizer(_lowercase						,      do_lower_case=_lowercase						,      split_by_punct=_lowercase  )
									__a			:    str      							=	tokenizer.convert_ids_to_tokens(tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )  )

									self.assertListEqual(_lowercase						,      _lowercase  )

									__a			:    Tuple      							=	DebertaVaTokenizerFast(_lowercase						,      do_lower_case=_lowercase						,      split_by_punct=_lowercase  )
									__a			:    int      							=	rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )  )

									self.assertListEqual(_lowercase						,      _lowercase  )







					def 			_lowerCamelCase		(    self  ):
									# fmt: off
									__a			:    Tuple      							=	"""I was born in 92000, and this is falsé."""
									__a			:    Any      							=	["""▁""", """<unk>""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """▁""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """▁""", """.""", ]
									# fmt: on

									__a			:    List[Any]      							=	DebertaVaTokenizer(_lowercase						,      do_lower_case=_lowercase						,      split_by_punct=_lowercase  )
									__a			:    str      							=	tokenizer.convert_ids_to_tokens(tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )  )

									self.assertListEqual(_lowercase						,      _lowercase  )

									__a			:    Union[str, Any]      							=	DebertaVaTokenizerFast(_lowercase						,      do_lower_case=_lowercase						,      split_by_punct=_lowercase  )
									__a			:    Optional[int]      							=	rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )  )

									self.assertListEqual(_lowercase						,      _lowercase  )







					def 			_lowerCamelCase		(    self  ):
									# fmt: off
									__a			:    List[Any]      							=	""" \tHeLLo!how  \n Are yoU?  """
									__a			:    Tuple      							=	["""▁""", """<unk>""", """e""", """<unk>""", """o""", """!""", """how""", """▁""", """<unk>""", """re""", """▁yo""", """<unk>""", """?"""]
									# fmt: on

									__a			:    Any      							=	DebertaVaTokenizer(_lowercase						,      do_lower_case=_lowercase						,      split_by_punct=_lowercase  )
									__a			:    Optional[int]      							=	tokenizer.convert_ids_to_tokens(tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )  )

									self.assertListEqual(_lowercase						,      _lowercase  )

									__a			:    Dict      							=	DebertaVaTokenizerFast(_lowercase						,      do_lower_case=_lowercase						,      split_by_punct=_lowercase  )
									__a			:    Optional[Any]      							=	rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )  )

									self.assertListEqual(_lowercase						,      _lowercase  )







					def 			_lowerCamelCase		(    self  ):
									__a			:    Tuple      							=	self.get_tokenizer()
									__a			:    Optional[int]      							=	self.get_rust_tokenizer()

									__a			:    str      							=	"""I was born in 92000, and this is falsé."""

									__a			:    Any      							=	tokenizer.convert_ids_to_tokens(tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )  )
									__a			:    List[str]      							=	rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )  )
									self.assertListEqual(_lowercase						,      _lowercase  )

									__a			:    Tuple      							=	tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )
									__a			:    str      							=	rust_tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )
									self.assertListEqual(_lowercase						,      _lowercase  )

									__a			:    Optional[Any]      							=	self.get_rust_tokenizer()
									__a			:    Optional[Any]      							=	tokenizer.encode(_lowercase  )
									__a			:    Optional[int]      							=	rust_tokenizer.encode(_lowercase  )
									self.assertListEqual(_lowercase						,      _lowercase  )







					def 			_lowerCamelCase		(    self  ):
									__a			:    Union[str, Any]      							=	"""This is a test"""
									__a			:    List[Any]      							=	[13, 1, 4398, 25, 21, 1289]
									__a			:    str      							=	["""▁""", """T""", """his""", """▁is""", """▁a""", """▁test"""]
									__a			:    Union[str, Any]      							=	["""▁""", """<unk>""", """his""", """▁is""", """▁a""", """▁test"""]

									__a			:    int      							=	DebertaVaTokenizer(_lowercase						,      keep_accents=_lowercase  )
									__a			:    Union[str, Any]      							=	DebertaVaTokenizerFast(_lowercase						,      keep_accents=_lowercase  )

									__a			:    int      							=	tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )
									self.assertListEqual(_lowercase						,      _lowercase  )
									__a			:    Tuple      							=	tokenizer.tokenize(_lowercase  )
									self.assertListEqual(_lowercase						,      _lowercase  )
									__a			:    Optional[int]      							=	tokenizer.convert_ids_to_tokens(_lowercase  )
									self.assertListEqual(_lowercase						,      _lowercase  )

									__a			:    Optional[Any]      							=	rust_tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )
									self.assertListEqual(_lowercase						,      _lowercase  )
									__a			:    Optional[Any]      							=	rust_tokenizer.tokenize(_lowercase  )
									self.assertListEqual(_lowercase						,      _lowercase  )
									__a			:    Any      							=	rust_tokenizer.convert_ids_to_tokens(_lowercase  )
									self.assertListEqual(_lowercase						,      _lowercase  )

									# fmt: off
									__a			:    List[Any]      							=	"""I was born in 92000, and this is falsé."""
									__a			:    int      							=	[13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
									__a			:    Tuple      							=	["""▁""", """I""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """é""", """.""", ]
									__a			:    str      							=	["""▁""", """<unk>""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """.""", ]
									# fmt: on

									__a			:    Any      							=	tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )
									self.assertListEqual(_lowercase						,      _lowercase  )
									__a			:    Union[str, Any]      							=	tokenizer.tokenize(_lowercase  )
									self.assertListEqual(_lowercase						,      _lowercase  )
									__a			:    Tuple      							=	tokenizer.convert_ids_to_tokens(_lowercase  )
									self.assertListEqual(_lowercase						,      _lowercase  )

									__a			:    Tuple      							=	rust_tokenizer.encode(_lowercase						,      add_special_tokens=_lowercase  )
									self.assertListEqual(_lowercase						,      _lowercase  )
									__a			:    int      							=	rust_tokenizer.tokenize(_lowercase  )
									self.assertListEqual(_lowercase						,      _lowercase  )
									__a			:    Any      							=	rust_tokenizer.convert_ids_to_tokens(_lowercase  )
									self.assertListEqual(_lowercase						,      _lowercase  )







					def 			_lowerCamelCase		(    self  ):
									__a			:    Any      							=	DebertaVaTokenizer(_lowercase  )

									__a			:    Optional[Any]      							=	tokenizer.encode('''sequence builders'''  )
									__a			:    int      							=	tokenizer.encode('''multi-sequence build'''  )

									__a			:    Dict      							=	tokenizer.build_inputs_with_special_tokens(_lowercase  )
									__a			:    Optional[Any]      							=	tokenizer.build_inputs_with_special_tokens(_lowercase						,      _lowercase  )

									self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]						,      _lowercase  )
									self.assertEqual(
									    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]						,      _lowercase						,      )







					@slow
					def 			_lowerCamelCase		(    self  ):
									# fmt: off
									__a			:    Union[str, Any]      							=	{"""input_ids""": [[1, 39867, 36, 19390, 486, 27, 35052, 81436, 18, 60685, 1225, 7, 35052, 81436, 18, 9367, 16899, 18, 15937, 53, 594, 773, 18, 16287, 30465, 36, 15937, 6, 41139, 38, 36979, 60763, 191, 6, 34132, 99, 6, 50538, 390, 43230, 6, 34132, 2779, 20850, 14, 699, 1072, 1194, 36, 382, 10901, 53, 7, 699, 1072, 2084, 36, 20422, 630, 53, 19, 105, 3049, 1896, 1053, 16899, 1506, 11, 37978, 4243, 7, 1237, 31869, 200, 16566, 654, 6, 35052, 81436, 7, 55630, 13593, 4, 2], [1, 26, 15011, 13, 667, 8, 1053, 18, 23611, 1237, 72356, 12820, 34, 104134, 1209, 35, 13313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 15785, 14951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
									# fmt: on

									self.tokenizer_integration_test_util(
									    expected_encoding=_lowercase						,      model_name='''microsoft/deberta-v2-xlarge'''						,      revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670'''						,      )


__snake_case				:      int   		=			[
    """Audio""",
    """Array2D""",
    """Array3D""",
    """Array4D""",
    """Array5D""",
    """ClassLabel""",
    """Features""",
    """Sequence""",
    """Value""",
    """Image""",
    """Translation""",
    """TranslationVariableLanguages""",
]
from .audio import Audio
from .features import ArrayaD, ArrayaD, ArrayaD, ArrayaD, ClassLabel, Features, Sequence, Value
from .image import Image
from .translation import Translation, TranslationVariableLanguages


