






from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
       import numpy
       import tensorflow as tf

       from transformers import (
           TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
           TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
           TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
           BertConfig,
           DPRConfig,
           TFDPRContextEncoder,
           TFDPRQuestionEncoder,
           TFDPRReader,
       )





class 					A__     :

     '''simple docstring'''






     def __init__( self      :    int   ,						_SCREAMING_SNAKE_CASE      :    Optional[int]   ,						_SCREAMING_SNAKE_CASE      :    Any=13   ,						_SCREAMING_SNAKE_CASE      :    Optional[int]=7   ,						_SCREAMING_SNAKE_CASE      :    Dict=True   ,						_SCREAMING_SNAKE_CASE      :    str=True   ,						_SCREAMING_SNAKE_CASE      :    Optional[Any]=True   ,						_SCREAMING_SNAKE_CASE      :    Any=True   ,						_SCREAMING_SNAKE_CASE      :    Dict=99   ,						_SCREAMING_SNAKE_CASE      :    str=32   ,						_SCREAMING_SNAKE_CASE      :    Optional[int]=2   ,						_SCREAMING_SNAKE_CASE      :    Optional[Any]=4   ,						_SCREAMING_SNAKE_CASE      :    List[Any]=37   ,						_SCREAMING_SNAKE_CASE      :    List[Any]="gelu"   ,						_SCREAMING_SNAKE_CASE      :    Optional[int]=0.1   ,						_SCREAMING_SNAKE_CASE      :    int=0.1   ,						_SCREAMING_SNAKE_CASE      :    Optional[int]=512   ,						_SCREAMING_SNAKE_CASE      :    Union[str, Any]=16   ,						_SCREAMING_SNAKE_CASE      :    Optional[int]=2   ,						_SCREAMING_SNAKE_CASE      :    Optional[int]=0.0_2   ,						_SCREAMING_SNAKE_CASE      :    str=3   ,						_SCREAMING_SNAKE_CASE      :    int=4   ,						_SCREAMING_SNAKE_CASE      :    Optional[int]=None   ,						_SCREAMING_SNAKE_CASE      :    Optional[Any]=0   ,						):






           """simple docstring"""




           UpperCamelCase					       =  parent
           UpperCamelCase					       =  batch_size
           UpperCamelCase					       =  seq_length
           UpperCamelCase					       =  is_training
           UpperCamelCase					       =  use_input_mask
           UpperCamelCase					       =  use_token_type_ids
           UpperCamelCase					       =  use_labels
           UpperCamelCase					       =  vocab_size
           UpperCamelCase					       =  hidden_size
           UpperCamelCase					       =  num_hidden_layers
           UpperCamelCase					       =  num_attention_heads
           UpperCamelCase					       =  intermediate_size
           UpperCamelCase					       =  hidden_act
           UpperCamelCase					       =  hidden_dropout_prob
           UpperCamelCase					       =  attention_probs_dropout_prob
           UpperCamelCase					       =  max_position_embeddings
           UpperCamelCase					       =  type_vocab_size
           UpperCamelCase					       =  type_sequence_label_size
           UpperCamelCase					       =  initializer_range
           UpperCamelCase					       =  num_labels
           UpperCamelCase					       =  num_choices
           UpperCamelCase					       =  scope
           UpperCamelCase					       =  projection_dim







     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[Any] ):






           """simple docstring"""




           UpperCamelCase					       =  ids_tensor([self.batch_size, self.seq_length]   ,						self.vocab_size )

           UpperCamelCase					       =  None
           if self.use_input_mask:
                 # follow test_modeling_tf_ctrl.py
                 UpperCamelCase					       =  random_attention_mask([self.batch_size, self.seq_length] )

           UpperCamelCase					       =  None
           if self.use_token_type_ids:
                 UpperCamelCase					       =  ids_tensor([self.batch_size, self.seq_length]   ,						self.type_vocab_size )

           UpperCamelCase					       =  None
           UpperCamelCase					       =  None
           UpperCamelCase					       =  None
           if self.use_labels:
                 UpperCamelCase					       =  ids_tensor([self.batch_size]   ,						self.type_sequence_label_size )
                 UpperCamelCase					       =  ids_tensor([self.batch_size, self.seq_length]   ,						self.num_labels )
                 UpperCamelCase					       =  ids_tensor([self.batch_size]   ,						self.num_choices )

           UpperCamelCase					       =  BertConfig(
               vocab_size=self.vocab_size   ,						hidden_size=self.hidden_size   ,						num_hidden_layers=self.num_hidden_layers   ,						num_attention_heads=self.num_attention_heads   ,						intermediate_size=self.intermediate_size   ,						hidden_act=self.hidden_act   ,						hidden_dropout_prob=self.hidden_dropout_prob   ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,						max_position_embeddings=self.max_position_embeddings   ,						type_vocab_size=self.type_vocab_size   ,						is_decoder=_SCREAMING_SNAKE_CASE   ,						initializer_range=self.initializer_range   ,						)
           UpperCamelCase					       =  DPRConfig(projection_dim=self.projection_dim   ,						**config.to_dict() )

           return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







     def   _SCREAMING_SNAKE_CASE     ( self      :    Dict   ,						_SCREAMING_SNAKE_CASE      :    str   ,						_SCREAMING_SNAKE_CASE      :    str   ,						_SCREAMING_SNAKE_CASE      :    str   ,						_SCREAMING_SNAKE_CASE      :    Tuple   ,						_SCREAMING_SNAKE_CASE      :    List[str]   ,						_SCREAMING_SNAKE_CASE      :    Any   ,						_SCREAMING_SNAKE_CASE      :    Optional[int] ):






           """simple docstring"""




           UpperCamelCase					       =  TFDPRContextEncoder(config=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  model(_SCREAMING_SNAKE_CASE   ,						attention_mask=_SCREAMING_SNAKE_CASE   ,						token_type_ids=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  model(_SCREAMING_SNAKE_CASE   ,						token_type_ids=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  model(_SCREAMING_SNAKE_CASE )
           self.parent.assertEqual(result.pooler_output.shape   ,						(self.batch_size, self.projection_dim or self.hidden_size) )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[Any]   ,						_SCREAMING_SNAKE_CASE      :    str   ,						_SCREAMING_SNAKE_CASE      :    int   ,						_SCREAMING_SNAKE_CASE      :    List[str]   ,						_SCREAMING_SNAKE_CASE      :    Any   ,						_SCREAMING_SNAKE_CASE      :    Tuple   ,						_SCREAMING_SNAKE_CASE      :    Optional[int]   ,						_SCREAMING_SNAKE_CASE      :    List[str] ):






           """simple docstring"""




           UpperCamelCase					       =  TFDPRQuestionEncoder(config=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  model(_SCREAMING_SNAKE_CASE   ,						attention_mask=_SCREAMING_SNAKE_CASE   ,						token_type_ids=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  model(_SCREAMING_SNAKE_CASE   ,						token_type_ids=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  model(_SCREAMING_SNAKE_CASE )
           self.parent.assertEqual(result.pooler_output.shape   ,						(self.batch_size, self.projection_dim or self.hidden_size) )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Tuple   ,						_SCREAMING_SNAKE_CASE      :    Optional[Any]   ,						_SCREAMING_SNAKE_CASE      :    Optional[Any]   ,						_SCREAMING_SNAKE_CASE      :    List[str]   ,						_SCREAMING_SNAKE_CASE      :    int   ,						_SCREAMING_SNAKE_CASE      :    Tuple   ,						_SCREAMING_SNAKE_CASE      :    Any   ,						_SCREAMING_SNAKE_CASE      :    Optional[Any] ):






           """simple docstring"""




           UpperCamelCase					       =  TFDPRReader(config=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  model(_SCREAMING_SNAKE_CASE   ,						attention_mask=_SCREAMING_SNAKE_CASE )

           self.parent.assertEqual(result.start_logits.shape   ,						(self.batch_size, self.seq_length) )
           self.parent.assertEqual(result.end_logits.shape   ,						(self.batch_size, self.seq_length) )
           self.parent.assertEqual(result.relevance_logits.shape   ,						(self.batch_size,) )







     def   _SCREAMING_SNAKE_CASE     ( self      :    List[Any] ):






           """simple docstring"""




           UpperCamelCase					       =  self.prepare_config_and_inputs()
           (
               (
               UpperCamelCase
           )       ,  (
               UpperCamelCase
           )       ,  (
               UpperCamelCase
           )       ,  (
               UpperCamelCase
           )       ,  (
               UpperCamelCase
           )       ,  (
               UpperCamelCase
           )       ,  (
               UpperCamelCase
           )       ,  
           )					       =  config_and_inputs
           UpperCamelCase					       =  {'input_ids': input_ids}
           return config, inputs_dict





@require_tf
class 					A__     (     __snake_case   ,    __snake_case   ,    unittest.TestCase ):

     '''simple docstring'''






     snake_case__					=					(
         (
             TFDPRContextEncoder,
             TFDPRQuestionEncoder,
             TFDPRReader,
         )
         if is_tf_available()
         else ()
     )
     snake_case__					=					{"""feature-extraction""": TFDPRQuestionEncoder} if is_tf_available() else {}

     snake_case__					=					False
     snake_case__					=					False
     snake_case__					=					False
     snake_case__					=					False
     snake_case__					=					False







     def   _SCREAMING_SNAKE_CASE     ( self      :    List[Any] ):






           """simple docstring"""




           UpperCamelCase					       =  TFDPRModelTester(self )
           UpperCamelCase					       =  ConfigTester(self   ,						config_class=_SCREAMING_SNAKE_CASE   ,						hidden_size=37 )







     def   _SCREAMING_SNAKE_CASE     ( self      :    List[Any] ):






           """simple docstring"""




           self.config_tester.run_common_tests()







     def   _SCREAMING_SNAKE_CASE     ( self      :    Tuple ):






           """simple docstring"""




           UpperCamelCase					       =  self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_dpr_context_encoder(*_SCREAMING_SNAKE_CASE )







     def   _SCREAMING_SNAKE_CASE     ( self      :    List[Any] ):






           """simple docstring"""




           UpperCamelCase					       =  self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_dpr_question_encoder(*_SCREAMING_SNAKE_CASE )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Tuple ):






           """simple docstring"""




           UpperCamelCase					       =  self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_dpr_reader(*_SCREAMING_SNAKE_CASE )







     @slow
     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[int] ):






           """simple docstring"""




           for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 UpperCamelCase					       =  TFDPRContextEncoder.from_pretrained(_SCREAMING_SNAKE_CASE )
                 self.assertIsNotNone(_SCREAMING_SNAKE_CASE )

           for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 UpperCamelCase					       =  TFDPRContextEncoder.from_pretrained(_SCREAMING_SNAKE_CASE )
                 self.assertIsNotNone(_SCREAMING_SNAKE_CASE )

           for model_name in TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 UpperCamelCase					       =  TFDPRQuestionEncoder.from_pretrained(_SCREAMING_SNAKE_CASE )
                 self.assertIsNotNone(_SCREAMING_SNAKE_CASE )

           for model_name in TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                 UpperCamelCase					       =  TFDPRReader.from_pretrained(_SCREAMING_SNAKE_CASE )
                 self.assertIsNotNone(_SCREAMING_SNAKE_CASE )





@require_tf
class 					A__     (     unittest.TestCase ):

     '''simple docstring'''






     @slow
     def   _SCREAMING_SNAKE_CASE     ( self      :    List[str] ):






           """simple docstring"""




           UpperCamelCase					       =  TFDPRQuestionEncoder.from_pretrained('facebook/dpr-question_encoder-single-nq-base' )

           UpperCamelCase					       =  tf.constant(
               [[101, 7592, 1010, 2003, 2026, 3899, 1_0140, 1029, 102]] )  # [CLS] hello, is my dog cute? [SEP]
           UpperCamelCase					       =  model(_SCREAMING_SNAKE_CASE )[0]  # embedding shape = (1, 768)
           # compare the actual values for a slice.
           UpperCamelCase					       =  tf.constant(
               [
                   [
                       0.0_3_2_3_6_2_5_3,
                       0.1_2_7_5_3_3_3_5,
                       0.1_6_8_1_8_5_0_9,
                       0.0_0_2_7_9_7_8_6,
                       0.3_8_9_6_9_3_3,
                       0.2_4_2_6_4_9_4_5,
                       0.2_1_7_8_9_7_1,
                       -0.0_2_3_3_5_2_2_7,
                       -0.0_8_4_8_1_9_5_9,
                       -0.1_4_3_2_4_1_1_7,
                   ]
               ] )
           self.assertTrue(numpy.allclose(output[:, :10].numpy()   ,						expected_slice.numpy()   ,						atol=1E-4 ) )











import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__   :  Optional[int]        =							get_tests_dir('''fixtures/spiece.model''')





@require_sentencepiece
@require_tokenizers
class 					A__     (     __snake_case   ,    unittest.TestCase ):

     '''simple docstring'''






     snake_case__					=					DebertaVaTokenizer
     snake_case__					=					DebertaVaTokenizerFast
     snake_case__					=					True
     snake_case__					=					True







     def   _SCREAMING_SNAKE_CASE     ( self      :    Tuple ):






           """simple docstring"""




           super().setUp()

           # We have a SentencePiece fixture for testing
           UpperCamelCase					       =  DebertaVaTokenizer(_SCREAMING_SNAKE_CASE   ,						unk_token='<unk>' )
           tokenizer.save_pretrained(self.tmpdirname )







     def   _SCREAMING_SNAKE_CASE     ( self      :    List[Any]   ,						_SCREAMING_SNAKE_CASE      :    Optional[Any] ):






           """simple docstring"""




           UpperCamelCase					       =  'this is a test'
           UpperCamelCase					       =  'this is a test'
           return input_text, output_text







     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[Any] ):






           """simple docstring"""




           UpperCamelCase					       =  '<pad>'
           UpperCamelCase					       =  0

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE )   ,						_SCREAMING_SNAKE_CASE )
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE )   ,						_SCREAMING_SNAKE_CASE )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Dict ):






           """simple docstring"""




           UpperCamelCase					       =  list(self.get_tokenizer().get_vocab().keys() )
           self.assertEqual(vocab_keys[0]   ,						'<pad>' )
           self.assertEqual(vocab_keys[1]   ,						'<unk>' )
           self.assertEqual(vocab_keys[-1]   ,						'[PAD]' )
           self.assertEqual(len(_SCREAMING_SNAKE_CASE )   ,						3_0001 )







     def   _SCREAMING_SNAKE_CASE     ( self      :    str ):






           """simple docstring"""




           self.assertEqual(self.get_tokenizer().vocab_size   ,						3_0000 )







     def   _SCREAMING_SNAKE_CASE     ( self      :    int ):






           """simple docstring"""




           UpperCamelCase					       =  ' \tHeLLo!how  \n Are yoU?  '
           UpperCamelCase					       =  ['▁hello', '!', 'how', '▁are', '▁you', '?']
           # fmt: on

           UpperCamelCase					       =  DebertaVaTokenizer(_SCREAMING_SNAKE_CASE   ,						do_lower_case=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  tokenizer.convert_ids_to_tokens(tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE ) )

           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           UpperCamelCase					       =  DebertaVaTokenizerFast(_SCREAMING_SNAKE_CASE   ,						do_lower_case=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE ) )

           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )







     @unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.' )
     def   _SCREAMING_SNAKE_CASE     ( self      :    int ):






           """simple docstring"""




           pass







     @unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.' )
     def   _SCREAMING_SNAKE_CASE     ( self      :    Any ):






           """simple docstring"""




           pass







     def   _SCREAMING_SNAKE_CASE     ( self      :    int ):






           """simple docstring"""




           UpperCamelCase					       =  'I was born in 92000, and this is falsé.'
           UpperCamelCase					       =  ['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
           # fmt: on

           UpperCamelCase					       =  DebertaVaTokenizer(_SCREAMING_SNAKE_CASE   ,						split_by_punct=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  tokenizer.convert_ids_to_tokens(tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE ) )

           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           UpperCamelCase					       =  DebertaVaTokenizerFast(_SCREAMING_SNAKE_CASE   ,						split_by_punct=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE ) )

           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )







     def   _SCREAMING_SNAKE_CASE     ( self      :    str ):






           """simple docstring"""




           UpperCamelCase					       =  'I was born in 92000, and this is falsé.'
           UpperCamelCase					       =  ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
           # fmt: on

           UpperCamelCase					       =  DebertaVaTokenizer(_SCREAMING_SNAKE_CASE   ,						do_lower_case=_SCREAMING_SNAKE_CASE   ,						split_by_punct=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  tokenizer.convert_ids_to_tokens(tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE ) )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           UpperCamelCase					       =  DebertaVaTokenizerFast(_SCREAMING_SNAKE_CASE   ,						do_lower_case=_SCREAMING_SNAKE_CASE   ,						split_by_punct=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE ) )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Tuple ):






           """simple docstring"""




           UpperCamelCase					       =  'I was born in 92000, and this is falsé.'
           UpperCamelCase					       =  ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
           # fmt: on

           UpperCamelCase					       =  DebertaVaTokenizer(_SCREAMING_SNAKE_CASE   ,						do_lower_case=_SCREAMING_SNAKE_CASE   ,						split_by_punct=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  tokenizer.convert_ids_to_tokens(tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE ) )

           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           UpperCamelCase					       =  DebertaVaTokenizerFast(_SCREAMING_SNAKE_CASE   ,						do_lower_case=_SCREAMING_SNAKE_CASE   ,						split_by_punct=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE ) )

           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )







     def   _SCREAMING_SNAKE_CASE     ( self      :    List[Any] ):






           """simple docstring"""




           UpperCamelCase					       =  'I was born in 92000, and this is falsé.'
           UpperCamelCase					       =  ['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
           # fmt: on

           UpperCamelCase					       =  DebertaVaTokenizer(_SCREAMING_SNAKE_CASE   ,						do_lower_case=_SCREAMING_SNAKE_CASE   ,						split_by_punct=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  tokenizer.convert_ids_to_tokens(tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE ) )

           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           UpperCamelCase					       =  DebertaVaTokenizerFast(_SCREAMING_SNAKE_CASE   ,						do_lower_case=_SCREAMING_SNAKE_CASE   ,						split_by_punct=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE ) )

           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Tuple ):






           """simple docstring"""




           UpperCamelCase					       =  ' \tHeLLo!how  \n Are yoU?  '
           UpperCamelCase					       =  ['▁', '<unk>', 'e', '<unk>', 'o', '!', 'how', '▁', '<unk>', 're', '▁yo', '<unk>', '?']
           # fmt: on

           UpperCamelCase					       =  DebertaVaTokenizer(_SCREAMING_SNAKE_CASE   ,						do_lower_case=_SCREAMING_SNAKE_CASE   ,						split_by_punct=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  tokenizer.convert_ids_to_tokens(tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE ) )

           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           UpperCamelCase					       =  DebertaVaTokenizerFast(_SCREAMING_SNAKE_CASE   ,						do_lower_case=_SCREAMING_SNAKE_CASE   ,						split_by_punct=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE ) )

           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[int] ):






           """simple docstring"""




           UpperCamelCase					       =  self.get_tokenizer()
           UpperCamelCase					       =  self.get_rust_tokenizer()

           UpperCamelCase					       =  'I was born in 92000, and this is falsé.'

           UpperCamelCase					       =  tokenizer.convert_ids_to_tokens(tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE ) )
           UpperCamelCase					       =  rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE ) )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           UpperCamelCase					       =  tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  rust_tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           UpperCamelCase					       =  self.get_rust_tokenizer()
           UpperCamelCase					       =  tokenizer.encode(_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  rust_tokenizer.encode(_SCREAMING_SNAKE_CASE )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[int] ):






           """simple docstring"""




           UpperCamelCase					       =  'This is a test'
           UpperCamelCase					       =  [13, 1, 4398, 25, 21, 1289]
           UpperCamelCase					       =  ['▁', 'T', 'his', '▁is', '▁a', '▁test']
           UpperCamelCase					       =  ['▁', '<unk>', 'his', '▁is', '▁a', '▁test']

           UpperCamelCase					       =  DebertaVaTokenizer(_SCREAMING_SNAKE_CASE   ,						keep_accents=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  DebertaVaTokenizerFast(_SCREAMING_SNAKE_CASE   ,						keep_accents=_SCREAMING_SNAKE_CASE )

           UpperCamelCase					       =  tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  tokenizer.tokenize(_SCREAMING_SNAKE_CASE )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           UpperCamelCase					       =  rust_tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  rust_tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           # fmt: off
           UpperCamelCase					       =  'I was born in 92000, and this is falsé.'
           UpperCamelCase					       =  [13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
           UpperCamelCase					       =  ['▁', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.', ]
           UpperCamelCase					       =  ['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
           # fmt: on

           UpperCamelCase					       =  tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  tokenizer.tokenize(_SCREAMING_SNAKE_CASE )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           UpperCamelCase					       =  rust_tokenizer.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  rust_tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Any ):






           """simple docstring"""




           UpperCamelCase					       =  DebertaVaTokenizer(_SCREAMING_SNAKE_CASE )

           UpperCamelCase					       =  tokenizer.encode('sequence builders' )
           UpperCamelCase					       =  tokenizer.encode('multi-sequence build' )

           UpperCamelCase					       =  tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]   ,						_SCREAMING_SNAKE_CASE )
           self.assertEqual(
               [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]   ,						_SCREAMING_SNAKE_CASE   ,						)







     @slow
     def   _SCREAMING_SNAKE_CASE     ( self      :    List[Any] ):






           """simple docstring"""




           UpperCamelCase					       =  {'input_ids': [[1, 3_9867, 36, 1_9390, 486, 27, 3_5052, 8_1436, 18, 6_0685, 1225, 7, 3_5052, 8_1436, 18, 9367, 1_6899, 18, 1_5937, 53, 594, 773, 18, 1_6287, 3_0465, 36, 1_5937, 6, 4_1139, 38, 3_6979, 6_0763, 191, 6, 3_4132, 99, 6, 5_0538, 390, 4_3230, 6, 3_4132, 2779, 2_0850, 14, 699, 1072, 1194, 36, 382, 1_0901, 53, 7, 699, 1072, 2084, 36, 2_0422, 630, 53, 19, 105, 3049, 1896, 1053, 1_6899, 1506, 11, 3_7978, 4243, 7, 1237, 3_1869, 200, 1_6566, 654, 6, 3_5052, 8_1436, 7, 5_5630, 1_3593, 4, 2], [1, 26, 1_5011, 13, 667, 8, 1053, 18, 2_3611, 1237, 7_2356, 1_2820, 34, 10_4134, 1209, 35, 1_3313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 1_5785, 1_4951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=_SCREAMING_SNAKE_CASE   ,						model_name='microsoft/deberta-v2-xlarge'   ,						revision='ad6e42c1532ddf3a15c39246b63f5559d558b670'   ,						)



