





import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_A					:				str				=			get_tests_dir("""fixtures/spiece.model""")



@require_sentencepiece
@require_tokenizers
class 					__snake_case	(  __SCREAMING_SNAKE_CASE			,     unittest.TestCase    ):


    '''simple docstring'''







    lowerCamelCase__    :		List[str]				=	DebertaVaTokenizer
    lowerCamelCase__    :		Union[str, Any]				=	DebertaVaTokenizerFast
    lowerCamelCase__    :		List[Any]				=	True
    lowerCamelCase__    :		List[Any]				=	True



    def  lowercase_					(       self     ):





        '''simple docstring'''

        super().setUp()

        # We have a SentencePiece fixture for testing
        SCREAMING_SNAKE_CASE__											=			DebertaVaTokenizer(A_   ,				unk_token='''<unk>'''     )
        tokenizer.save_pretrained(self.tmpdirname     )



    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			'''this is a test'''
        SCREAMING_SNAKE_CASE__											=			'''this is a test'''
        return input_text, output_text



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			'''<pad>'''
        SCREAMING_SNAKE_CASE__											=			0

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(A_     )   ,				A_     )
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(A_     )   ,				A_     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			list(self.get_tokenizer().get_vocab().keys()     )
        self.assertEqual(vocab_keys[0]   ,				'''<pad>'''     )
        self.assertEqual(vocab_keys[1]   ,				'''<unk>'''     )
        self.assertEqual(vocab_keys[-1]   ,				'''[PAD]'''     )
        self.assertEqual(len(A_     )   ,				3_00_01     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        self.assertEqual(self.get_tokenizer().vocab_size   ,				3_00_00     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			''' \tHeLLo!how  \n Are yoU?  '''
        SCREAMING_SNAKE_CASE__											=			['''▁hello''', '''!''', '''how''', '''▁are''', '''▁you''', '''?''']
        # fmt: on

        SCREAMING_SNAKE_CASE__											=			DebertaVaTokenizer(A_   ,				do_lower_case=A_     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.convert_ids_to_tokens(tokenizer.encode(A_   ,				add_special_tokens=A_     )     )

        self.assertListEqual(A_   ,				A_     )

        SCREAMING_SNAKE_CASE__											=			DebertaVaTokenizerFast(A_   ,				do_lower_case=A_     )
        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_   ,				add_special_tokens=A_     )     )

        self.assertListEqual(A_   ,				A_     )



    @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''     )
    def  lowercase_					(       self     ):





        '''simple docstring'''

        pass



    @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''     )
    def  lowercase_					(       self     ):





        '''simple docstring'''

        pass



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			'''I was born in 92000, and this is falsé.'''
        SCREAMING_SNAKE_CASE__											=			['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
        # fmt: on

        SCREAMING_SNAKE_CASE__											=			DebertaVaTokenizer(A_   ,				split_by_punct=A_     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.convert_ids_to_tokens(tokenizer.encode(A_   ,				add_special_tokens=A_     )     )

        self.assertListEqual(A_   ,				A_     )

        SCREAMING_SNAKE_CASE__											=			DebertaVaTokenizerFast(A_   ,				split_by_punct=A_     )
        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_   ,				add_special_tokens=A_     )     )

        self.assertListEqual(A_   ,				A_     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			'''I was born in 92000, and this is falsé.'''
        SCREAMING_SNAKE_CASE__											=			['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
        # fmt: on

        SCREAMING_SNAKE_CASE__											=			DebertaVaTokenizer(A_   ,				do_lower_case=A_   ,				split_by_punct=A_     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.convert_ids_to_tokens(tokenizer.encode(A_   ,				add_special_tokens=A_     )     )
        self.assertListEqual(A_   ,				A_     )

        SCREAMING_SNAKE_CASE__											=			DebertaVaTokenizerFast(A_   ,				do_lower_case=A_   ,				split_by_punct=A_     )
        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_   ,				add_special_tokens=A_     )     )
        self.assertListEqual(A_   ,				A_     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			'''I was born in 92000, and this is falsé.'''
        SCREAMING_SNAKE_CASE__											=			['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
        # fmt: on

        SCREAMING_SNAKE_CASE__											=			DebertaVaTokenizer(A_   ,				do_lower_case=A_   ,				split_by_punct=A_     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.convert_ids_to_tokens(tokenizer.encode(A_   ,				add_special_tokens=A_     )     )

        self.assertListEqual(A_   ,				A_     )

        SCREAMING_SNAKE_CASE__											=			DebertaVaTokenizerFast(A_   ,				do_lower_case=A_   ,				split_by_punct=A_     )
        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_   ,				add_special_tokens=A_     )     )

        self.assertListEqual(A_   ,				A_     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			'''I was born in 92000, and this is falsé.'''
        SCREAMING_SNAKE_CASE__											=			['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
        # fmt: on

        SCREAMING_SNAKE_CASE__											=			DebertaVaTokenizer(A_   ,				do_lower_case=A_   ,				split_by_punct=A_     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.convert_ids_to_tokens(tokenizer.encode(A_   ,				add_special_tokens=A_     )     )

        self.assertListEqual(A_   ,				A_     )

        SCREAMING_SNAKE_CASE__											=			DebertaVaTokenizerFast(A_   ,				do_lower_case=A_   ,				split_by_punct=A_     )
        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_   ,				add_special_tokens=A_     )     )

        self.assertListEqual(A_   ,				A_     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			''' \tHeLLo!how  \n Are yoU?  '''
        SCREAMING_SNAKE_CASE__											=			['''▁''', '''<unk>''', '''e''', '''<unk>''', '''o''', '''!''', '''how''', '''▁''', '''<unk>''', '''re''', '''▁yo''', '''<unk>''', '''?''']
        # fmt: on

        SCREAMING_SNAKE_CASE__											=			DebertaVaTokenizer(A_   ,				do_lower_case=A_   ,				split_by_punct=A_     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.convert_ids_to_tokens(tokenizer.encode(A_   ,				add_special_tokens=A_     )     )

        self.assertListEqual(A_   ,				A_     )

        SCREAMING_SNAKE_CASE__											=			DebertaVaTokenizerFast(A_   ,				do_lower_case=A_   ,				split_by_punct=A_     )
        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_   ,				add_special_tokens=A_     )     )

        self.assertListEqual(A_   ,				A_     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			self.get_tokenizer()
        SCREAMING_SNAKE_CASE__											=			self.get_rust_tokenizer()

        SCREAMING_SNAKE_CASE__											=			'''I was born in 92000, and this is falsé.'''

        SCREAMING_SNAKE_CASE__											=			tokenizer.convert_ids_to_tokens(tokenizer.encode(A_   ,				add_special_tokens=A_     )     )
        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_   ,				add_special_tokens=A_     )     )
        self.assertListEqual(A_   ,				A_     )

        SCREAMING_SNAKE_CASE__											=			tokenizer.encode(A_   ,				add_special_tokens=A_     )
        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.encode(A_   ,				add_special_tokens=A_     )
        self.assertListEqual(A_   ,				A_     )

        SCREAMING_SNAKE_CASE__											=			self.get_rust_tokenizer()
        SCREAMING_SNAKE_CASE__											=			tokenizer.encode(A_     )
        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.encode(A_     )
        self.assertListEqual(A_   ,				A_     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			'''This is a test'''
        SCREAMING_SNAKE_CASE__											=			[13, 1, 43_98, 25, 21, 12_89]
        SCREAMING_SNAKE_CASE__											=			['''▁''', '''T''', '''his''', '''▁is''', '''▁a''', '''▁test''']
        SCREAMING_SNAKE_CASE__											=			['''▁''', '''<unk>''', '''his''', '''▁is''', '''▁a''', '''▁test''']

        SCREAMING_SNAKE_CASE__											=			DebertaVaTokenizer(A_   ,				keep_accents=A_     )
        SCREAMING_SNAKE_CASE__											=			DebertaVaTokenizerFast(A_   ,				keep_accents=A_     )

        SCREAMING_SNAKE_CASE__											=			tokenizer.encode(A_   ,				add_special_tokens=A_     )
        self.assertListEqual(A_   ,				A_     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.tokenize(A_     )
        self.assertListEqual(A_   ,				A_     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.convert_ids_to_tokens(A_     )
        self.assertListEqual(A_   ,				A_     )

        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.encode(A_   ,				add_special_tokens=A_     )
        self.assertListEqual(A_   ,				A_     )
        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.tokenize(A_     )
        self.assertListEqual(A_   ,				A_     )
        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.convert_ids_to_tokens(A_     )
        self.assertListEqual(A_   ,				A_     )

        # fmt: off
        SCREAMING_SNAKE_CASE__											=			'''I was born in 92000, and this is falsé.'''
        SCREAMING_SNAKE_CASE__											=			[13, 1, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]
        SCREAMING_SNAKE_CASE__											=			['''▁''', '''I''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''', ]
        SCREAMING_SNAKE_CASE__											=			['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
        # fmt: on

        SCREAMING_SNAKE_CASE__											=			tokenizer.encode(A_   ,				add_special_tokens=A_     )
        self.assertListEqual(A_   ,				A_     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.tokenize(A_     )
        self.assertListEqual(A_   ,				A_     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.convert_ids_to_tokens(A_     )
        self.assertListEqual(A_   ,				A_     )

        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.encode(A_   ,				add_special_tokens=A_     )
        self.assertListEqual(A_   ,				A_     )
        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.tokenize(A_     )
        self.assertListEqual(A_   ,				A_     )
        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.convert_ids_to_tokens(A_     )
        self.assertListEqual(A_   ,				A_     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			DebertaVaTokenizer(A_     )

        SCREAMING_SNAKE_CASE__											=			tokenizer.encode('''sequence builders'''     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.encode('''multi-sequence build'''     )

        SCREAMING_SNAKE_CASE__											=			tokenizer.build_inputs_with_special_tokens(A_     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.build_inputs_with_special_tokens(A_   ,				A_     )

        self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]   ,				A_     )
        self.assertEqual(
            [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]   ,				A_   ,				)




    @slow
    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			{'''input_ids''': [[1, 3_98_67, 36, 1_93_90, 4_86, 27, 3_50_52, 8_14_36, 18, 6_06_85, 12_25, 7, 3_50_52, 8_14_36, 18, 93_67, 1_68_99, 18, 1_59_37, 53, 5_94, 7_73, 18, 1_62_87, 3_04_65, 36, 1_59_37, 6, 4_11_39, 38, 3_69_79, 6_07_63, 1_91, 6, 3_41_32, 99, 6, 5_05_38, 3_90, 4_32_30, 6, 3_41_32, 27_79, 2_08_50, 14, 6_99, 10_72, 11_94, 36, 3_82, 1_09_01, 53, 7, 6_99, 10_72, 20_84, 36, 2_04_22, 6_30, 53, 19, 1_05, 30_49, 18_96, 10_53, 1_68_99, 15_06, 11, 3_79_78, 42_43, 7, 12_37, 3_18_69, 2_00, 1_65_66, 6_54, 6, 3_50_52, 8_14_36, 7, 5_56_30, 1_35_93, 4, 2], [1, 26, 1_50_11, 13, 6_67, 8, 10_53, 18, 2_36_11, 12_37, 7_23_56, 1_28_20, 34, 10_41_34, 12_09, 35, 1_33_13, 66_27, 21, 2_02, 3_47, 7, 1_64, 23_99, 11, 46, 44_85, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 12_32, 28_64, 1_57_85, 1_49_51, 1_05, 5, 85_81, 12_50, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=A_   ,				model_name='''microsoft/deberta-v2-xlarge'''   ,				revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670'''   ,				)







from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Features, Value
from .base import TaskTemplate



@dataclass(frozen=__SCREAMING_SNAKE_CASE    )
class 					__snake_case	(  __SCREAMING_SNAKE_CASE    ):


    '''simple docstring'''







    lowerCamelCase__    :		str				=	field(default="""summarization"""			,     metadata={"""include_in_asdict_even_if_is_default""": True}    )
    lowerCamelCase__    :		ClassVar[Features]				=	Features({"""text""": Value("""string"""    )}    )
    lowerCamelCase__    :		ClassVar[Features]				=	Features({"""summary""": Value("""string"""    )}    )
    lowerCamelCase__    :		str				=	"text"
    lowerCamelCase__    :		str				=	"summary"



    @property
    def  lowercase_					(       self     ):





        '''simple docstring'''

        return {self.text_column: "text", self.summary_column: "summary"}
