'''simple docstring'''




import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCAmelCase					=get_tests_dir("fixtures/spiece.model")




@require_sentencepiece
@require_tokenizers
class        a__ (  UpperCAmelCase__				,  unittest.TestCase					):
     lowerCamelCase    :	List[str]				=DebertaVaTokenizer
     lowerCamelCase    :	Dict				=DebertaVaTokenizerFast
     lowerCamelCase    :	Optional[Any]				=True
     lowerCamelCase    :	Optional[int]				=True






     def   SCREAMING_SNAKE_CASE__  (					self						:						Optional[Any]       ):





         """simple docstring"""



         super().setUp()

         # We have a SentencePiece fixture for testing
         __lowerCamelCase            =    DebertaVaTokenizer(a    ,							unk_token='''<unk>'''       )
         tokenizer.save_pretrained(self.tmpdirname       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						List[str]    ,							a						:						List[str]       ):





         """simple docstring"""



         __lowerCamelCase            =    '''this is a test'''
         __lowerCamelCase            =    '''this is a test'''
         return input_text, output_text






     def   SCREAMING_SNAKE_CASE__  (					self						:						Optional[Any]       ):





         """simple docstring"""



         __lowerCamelCase            =    '''<pad>'''
         __lowerCamelCase            =    0

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(a       )    ,							a       )
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(a       )    ,							a       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						Dict       ):





         """simple docstring"""



         __lowerCamelCase            =    list(self.get_tokenizer().get_vocab().keys()       )
         self.assertEqual(vocab_keys[0]    ,							'''<pad>'''       )
         self.assertEqual(vocab_keys[1]    ,							'''<unk>'''       )
         self.assertEqual(vocab_keys[-1]    ,							'''[PAD]'''       )
         self.assertEqual(len(a       )    ,							3_00_01       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						Optional[int]       ):





         """simple docstring"""



         self.assertEqual(self.get_tokenizer().vocab_size    ,							3_00_00       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						List[str]       ):





         """simple docstring"""



         __lowerCamelCase            =    ''' \tHeLLo!how  \n Are yoU?  '''
         __lowerCamelCase            =    ['''▁hello''', '''!''', '''how''', '''▁are''', '''▁you''', '''?''']
         # fmt: on

         __lowerCamelCase            =    DebertaVaTokenizer(a    ,							do_lower_case=a       )
         __lowerCamelCase            =    tokenizer.convert_ids_to_tokens(tokenizer.encode(a    ,							add_special_tokens=a       )       )

         self.assertListEqual(a    ,							a       )

         __lowerCamelCase            =    DebertaVaTokenizerFast(a    ,							do_lower_case=a       )
         __lowerCamelCase            =    rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a    ,							add_special_tokens=a       )       )

         self.assertListEqual(a    ,							a       )






     @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''       )
     def   SCREAMING_SNAKE_CASE__  (					self						:						str       ):





         """simple docstring"""



         pass






     @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''       )
     def   SCREAMING_SNAKE_CASE__  (					self						:						Optional[Any]       ):





         """simple docstring"""



         pass






     def   SCREAMING_SNAKE_CASE__  (					self						:						int       ):





         """simple docstring"""



         __lowerCamelCase            =    '''I was born in 92000, and this is falsé.'''
         __lowerCamelCase            =    ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
         # fmt: on

         __lowerCamelCase            =    DebertaVaTokenizer(a    ,							split_by_punct=a       )
         __lowerCamelCase            =    tokenizer.convert_ids_to_tokens(tokenizer.encode(a    ,							add_special_tokens=a       )       )

         self.assertListEqual(a    ,							a       )

         __lowerCamelCase            =    DebertaVaTokenizerFast(a    ,							split_by_punct=a       )
         __lowerCamelCase            =    rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a    ,							add_special_tokens=a       )       )

         self.assertListEqual(a    ,							a       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						str       ):





         """simple docstring"""



         __lowerCamelCase            =    '''I was born in 92000, and this is falsé.'''
         __lowerCamelCase            =    ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
         # fmt: on

         __lowerCamelCase            =    DebertaVaTokenizer(a    ,							do_lower_case=a    ,							split_by_punct=a       )
         __lowerCamelCase            =    tokenizer.convert_ids_to_tokens(tokenizer.encode(a    ,							add_special_tokens=a       )       )
         self.assertListEqual(a    ,							a       )

         __lowerCamelCase            =    DebertaVaTokenizerFast(a    ,							do_lower_case=a    ,							split_by_punct=a       )
         __lowerCamelCase            =    rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a    ,							add_special_tokens=a       )       )
         self.assertListEqual(a    ,							a       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						Any       ):





         """simple docstring"""



         __lowerCamelCase            =    '''I was born in 92000, and this is falsé.'''
         __lowerCamelCase            =    ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
         # fmt: on

         __lowerCamelCase            =    DebertaVaTokenizer(a    ,							do_lower_case=a    ,							split_by_punct=a       )
         __lowerCamelCase            =    tokenizer.convert_ids_to_tokens(tokenizer.encode(a    ,							add_special_tokens=a       )       )

         self.assertListEqual(a    ,							a       )

         __lowerCamelCase            =    DebertaVaTokenizerFast(a    ,							do_lower_case=a    ,							split_by_punct=a       )
         __lowerCamelCase            =    rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a    ,							add_special_tokens=a       )       )

         self.assertListEqual(a    ,							a       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						Tuple       ):





         """simple docstring"""



         __lowerCamelCase            =    '''I was born in 92000, and this is falsé.'''
         __lowerCamelCase            =    ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
         # fmt: on

         __lowerCamelCase            =    DebertaVaTokenizer(a    ,							do_lower_case=a    ,							split_by_punct=a       )
         __lowerCamelCase            =    tokenizer.convert_ids_to_tokens(tokenizer.encode(a    ,							add_special_tokens=a       )       )

         self.assertListEqual(a    ,							a       )

         __lowerCamelCase            =    DebertaVaTokenizerFast(a    ,							do_lower_case=a    ,							split_by_punct=a       )
         __lowerCamelCase            =    rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a    ,							add_special_tokens=a       )       )

         self.assertListEqual(a    ,							a       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						List[Any]       ):





         """simple docstring"""



         __lowerCamelCase            =    ''' \tHeLLo!how  \n Are yoU?  '''
         __lowerCamelCase            =    ['''▁''', '''<unk>''', '''e''', '''<unk>''', '''o''', '''!''', '''how''', '''▁''', '''<unk>''', '''re''', '''▁yo''', '''<unk>''', '''?''']
         # fmt: on

         __lowerCamelCase            =    DebertaVaTokenizer(a    ,							do_lower_case=a    ,							split_by_punct=a       )
         __lowerCamelCase            =    tokenizer.convert_ids_to_tokens(tokenizer.encode(a    ,							add_special_tokens=a       )       )

         self.assertListEqual(a    ,							a       )

         __lowerCamelCase            =    DebertaVaTokenizerFast(a    ,							do_lower_case=a    ,							split_by_punct=a       )
         __lowerCamelCase            =    rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a    ,							add_special_tokens=a       )       )

         self.assertListEqual(a    ,							a       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						Union[str, Any]       ):





         """simple docstring"""



         __lowerCamelCase            =    self.get_tokenizer()
         __lowerCamelCase            =    self.get_rust_tokenizer()

         __lowerCamelCase            =    '''I was born in 92000, and this is falsé.'''

         __lowerCamelCase            =    tokenizer.convert_ids_to_tokens(tokenizer.encode(a    ,							add_special_tokens=a       )       )
         __lowerCamelCase            =    rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(a    ,							add_special_tokens=a       )       )
         self.assertListEqual(a    ,							a       )

         __lowerCamelCase            =    tokenizer.encode(a    ,							add_special_tokens=a       )
         __lowerCamelCase            =    rust_tokenizer.encode(a    ,							add_special_tokens=a       )
         self.assertListEqual(a    ,							a       )

         __lowerCamelCase            =    self.get_rust_tokenizer()
         __lowerCamelCase            =    tokenizer.encode(a       )
         __lowerCamelCase            =    rust_tokenizer.encode(a       )
         self.assertListEqual(a    ,							a       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						Dict       ):





         """simple docstring"""



         __lowerCamelCase            =    '''This is a test'''
         __lowerCamelCase            =    [13, 1, 43_98, 25, 21, 12_89]
         __lowerCamelCase            =    ['''▁''', '''T''', '''his''', '''▁is''', '''▁a''', '''▁test''']
         __lowerCamelCase            =    ['''▁''', '''<unk>''', '''his''', '''▁is''', '''▁a''', '''▁test''']

         __lowerCamelCase            =    DebertaVaTokenizer(a    ,							keep_accents=a       )
         __lowerCamelCase            =    DebertaVaTokenizerFast(a    ,							keep_accents=a       )

         __lowerCamelCase            =    tokenizer.encode(a    ,							add_special_tokens=a       )
         self.assertListEqual(a    ,							a       )
         __lowerCamelCase            =    tokenizer.tokenize(a       )
         self.assertListEqual(a    ,							a       )
         __lowerCamelCase            =    tokenizer.convert_ids_to_tokens(a       )
         self.assertListEqual(a    ,							a       )

         __lowerCamelCase            =    rust_tokenizer.encode(a    ,							add_special_tokens=a       )
         self.assertListEqual(a    ,							a       )
         __lowerCamelCase            =    rust_tokenizer.tokenize(a       )
         self.assertListEqual(a    ,							a       )
         __lowerCamelCase            =    rust_tokenizer.convert_ids_to_tokens(a       )
         self.assertListEqual(a    ,							a       )

         # fmt: off
         __lowerCamelCase            =    '''I was born in 92000, and this is falsé.'''
         __lowerCamelCase            =    [13, 1, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]
         __lowerCamelCase            =    ['''▁''', '''I''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''', ]
         __lowerCamelCase            =    ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
         # fmt: on

         __lowerCamelCase            =    tokenizer.encode(a    ,							add_special_tokens=a       )
         self.assertListEqual(a    ,							a       )
         __lowerCamelCase            =    tokenizer.tokenize(a       )
         self.assertListEqual(a    ,							a       )
         __lowerCamelCase            =    tokenizer.convert_ids_to_tokens(a       )
         self.assertListEqual(a    ,							a       )

         __lowerCamelCase            =    rust_tokenizer.encode(a    ,							add_special_tokens=a       )
         self.assertListEqual(a    ,							a       )
         __lowerCamelCase            =    rust_tokenizer.tokenize(a       )
         self.assertListEqual(a    ,							a       )
         __lowerCamelCase            =    rust_tokenizer.convert_ids_to_tokens(a       )
         self.assertListEqual(a    ,							a       )






     def   SCREAMING_SNAKE_CASE__  (					self						:						Optional[Any]       ):





         """simple docstring"""



         __lowerCamelCase            =    DebertaVaTokenizer(a       )

         __lowerCamelCase            =    tokenizer.encode('''sequence builders'''       )
         __lowerCamelCase            =    tokenizer.encode('''multi-sequence build'''       )

         __lowerCamelCase            =    tokenizer.build_inputs_with_special_tokens(a       )
         __lowerCamelCase            =    tokenizer.build_inputs_with_special_tokens(a    ,							a       )

         self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]    ,							a       )
         self.assertEqual(
             [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]    ,							a    ,							)






     @slow
     def   SCREAMING_SNAKE_CASE__  (					self						:						List[Any]       ):





         """simple docstring"""



         __lowerCamelCase            =    {'''input_ids''': [[1, 3_98_67, 36, 1_93_90, 4_86, 27, 3_50_52, 8_14_36, 18, 6_06_85, 12_25, 7, 3_50_52, 8_14_36, 18, 93_67, 1_68_99, 18, 1_59_37, 53, 5_94, 7_73, 18, 1_62_87, 3_04_65, 36, 1_59_37, 6, 4_11_39, 38, 3_69_79, 6_07_63, 1_91, 6, 3_41_32, 99, 6, 5_05_38, 3_90, 4_32_30, 6, 3_41_32, 27_79, 2_08_50, 14, 6_99, 10_72, 11_94, 36, 3_82, 1_09_01, 53, 7, 6_99, 10_72, 20_84, 36, 2_04_22, 6_30, 53, 19, 1_05, 30_49, 18_96, 10_53, 1_68_99, 15_06, 11, 3_79_78, 42_43, 7, 12_37, 3_18_69, 2_00, 1_65_66, 6_54, 6, 3_50_52, 8_14_36, 7, 5_56_30, 1_35_93, 4, 2], [1, 26, 1_50_11, 13, 6_67, 8, 10_53, 18, 2_36_11, 12_37, 7_23_56, 1_28_20, 34, 10_41_34, 12_09, 35, 1_33_13, 66_27, 21, 2_02, 3_47, 7, 1_64, 23_99, 11, 46, 44_85, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 12_32, 28_64, 1_57_85, 1_49_51, 1_05, 5, 85_81, 12_50, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=a    ,							model_name='''microsoft/deberta-v2-xlarge'''    ,							revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670'''    ,							)

'''simple docstring'''




import re

from flax.core.frozen_dict import freeze
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.experimental import PartitionSpec as P


# Sentinels
__UpperCAmelCase					=object()

# For specifying empty leaf dict `{}`
__UpperCAmelCase					=object()






def __lowerCAmelCase					(				UpperCamelCase__							,    UpperCamelCase__							)      ->       List[Any]:
    __lowerCamelCase            =    tuple((re.compile(x + '''$'''							) for x in qs)							)
    for i in range(len(UpperCamelCase__							) - len(UpperCamelCase__							) + 1							):
        __lowerCamelCase            =    [x.match(UpperCamelCase__							) for x, y in zip(UpperCamelCase__							,    ks[i:]							)]
        if matches and all(UpperCamelCase__							):
            return True
    return False






def __lowerCAmelCase					(				UpperCamelCase__							)      ->       Optional[int]:
    def replace(UpperCamelCase__							,    UpperCamelCase__							):
        for rule, replacement in rules:
            if _match(UpperCamelCase__							,    UpperCamelCase__							):
                return replacement
        return val

    return replace






def __lowerCAmelCase					(				)      ->       Dict:
    return [
        # embeddings
        (("transformer", "wpe", "embedding"), P('''mp'''							,    UpperCamelCase__							)),
        (("transformer", "wte", "embedding"), P('''mp'''							,    UpperCamelCase__							)),
        # atention
        (("attention", "(q_proj|k_proj|v_proj)", "kernel"), P(UpperCamelCase__							,    '''mp'''							)),
        (("attention", "out_proj", "kernel"), P('''mp'''							,    UpperCamelCase__							)),
        (("attention", "out_proj", "bias"), None),
        # mlp
        (("mlp", "c_fc", "kernel"), P(UpperCamelCase__							,    '''mp'''							)),
        (("mlp", "c_fc", "bias"), P('''mp'''							)),
        (("mlp", "c_proj", "kernel"), P('''mp'''							,    UpperCamelCase__							)),
        (("mlp", "c_proj", "bias"), None),
        # layer norms
        ((r"ln_\d+", "bias"), None),
        ((r"\d+", r"ln_\d+", "scale"), None),
        (("ln_f", "bias"), None),
        (("ln_f", "scale"), None),
    ]






def __lowerCAmelCase					(				UpperCamelCase__							)      ->       int:
    __lowerCamelCase            =    _get_partition_rules()
    __lowerCamelCase            =    _replacement_rules(UpperCamelCase__							)
    __lowerCamelCase            =    {k: _unmatched for k in flatten_dict(UpperCamelCase__							)}
    __lowerCamelCase            =    {k: replace(UpperCamelCase__							,    UpperCamelCase__							) for k, v in initd.items()}
    assert _unmatched not in result.values(), "Incomplete partition spec."
    return freeze(unflatten_dict(UpperCamelCase__							)							)
