


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase		   =			{'configuration_xlnet': ['XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP', 'XLNetConfig']}

try:
 if not is_sentencepiece_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase		   =			['XLNetTokenizer']

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase		   =			['XLNetTokenizerFast']

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase		   =			[
     'XLNET_PRETRAINED_MODEL_ARCHIVE_LIST',
     'XLNetForMultipleChoice',
     'XLNetForQuestionAnswering',
     'XLNetForQuestionAnsweringSimple',
     'XLNetForSequenceClassification',
     'XLNetForTokenClassification',
     'XLNetLMHeadModel',
     'XLNetModel',
     'XLNetPreTrainedModel',
     'load_tf_weights_in_xlnet',
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase		   =			[
     'TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST',
     'TFXLNetForMultipleChoice',
     'TFXLNetForQuestionAnsweringSimple',
     'TFXLNetForSequenceClassification',
     'TFXLNetForTokenClassification',
     'TFXLNetLMHeadModel',
     'TFXLNetMainLayer',
     'TFXLNetModel',
     'TFXLNetPreTrainedModel',
 ]


if TYPE_CHECKING:
 from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig

 try:
  if not is_sentencepiece_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_xlnet import XLNetTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_xlnet_fast import XLNetTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_xlnet import (
      XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
      XLNetForMultipleChoice,
      XLNetForQuestionAnswering,
      XLNetForQuestionAnsweringSimple,
      XLNetForSequenceClassification,
      XLNetForTokenClassification,
      XLNetLMHeadModel,
      XLNetModel,
      XLNetPreTrainedModel,
      load_tf_weights_in_xlnet,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_xlnet import (
      TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFXLNetForMultipleChoice,
      TFXLNetForQuestionAnsweringSimple,
      TFXLNetForSequenceClassification,
      TFXLNetForTokenClassification,
      TFXLNetLMHeadModel,
      TFXLNetMainLayer,
      TFXLNetModel,
      TFXLNetPreTrainedModel,
  )

else:
 import sys

 lowerCamelCase		   =			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)






import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase		   =			get_tests_dir('fixtures/spiece.model')




@require_sentencepiece
@require_tokenizers
class A    (  UpperCamelCase_       ,				unittest.TestCase	):
      UpperCamelCase__				:    Dict =DebertaVaTokenizer
      UpperCamelCase__				:    Union[str, Any] =DebertaVaTokenizerFast
      UpperCamelCase__				:    Optional[int] =True
      UpperCamelCase__				:    Tuple =True






      def 			lowerCamelCase			(   self		:	Tuple					)   ->		Optional[Any]:
             """simple docstring"""




             super().setUp()

             # We have a SentencePiece fixture for testing
             _lowerCamelCase			:						Optional[Any]            =DebertaVaTokenizer(lowercase_     ,  unk_token='<unk>'					)
             tokenizer.save_pretrained(self.tmpdirname					)






      def 			lowerCamelCase			(   self		:	Optional[Any]     ,  lowercase_		:	List[Any]					)   ->		Optional[int]:
             """simple docstring"""




             _lowerCamelCase			:						Dict            ='this is a test'
             _lowerCamelCase			:						List[Any]            ='this is a test'
             return input_text, output_text






      def 			lowerCamelCase			(   self		:	int					)   ->		str:
             """simple docstring"""




             _lowerCamelCase			:						Union[str, Any]            ='<pad>'
             _lowerCamelCase			:						Union[str, Any]            =0

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase_					)     ,  lowercase_					)
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase_					)     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	Optional[Any]					)   ->		Any:
             """simple docstring"""




             _lowerCamelCase			:						Optional[Any]            =list(self.get_tokenizer().get_vocab().keys()					)
             self.assertEqual(vocab_keys[0]     ,  '<pad>'					)
             self.assertEqual(vocab_keys[1]     ,  '<unk>'					)
             self.assertEqual(vocab_keys[-1]     ,  '[PAD]'					)
             self.assertEqual(len(lowercase_					)     ,  3_0001					)






      def 			lowerCamelCase			(   self		:	List[str]					)   ->		Union[str, Any]:
             """simple docstring"""




             self.assertEqual(self.get_tokenizer().vocab_size     ,  3_0000					)






      def 			lowerCamelCase			(   self		:	Union[str, Any]					)   ->		Optional[Any]:
             """simple docstring"""




             _lowerCamelCase			:						List[str]            =' \tHeLLo!how  \n Are yoU?  '
             _lowerCamelCase			:						List[Any]            =['▁hello', '!', 'how', '▁are', '▁you', '?']
             # fmt: on

             _lowerCamelCase			:						Any            =DebertaVaTokenizer(lowercase_     ,  do_lower_case=lowercase_					)
             _lowerCamelCase			:						Dict            =tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						Optional[Any]            =DebertaVaTokenizerFast(lowercase_     ,  do_lower_case=lowercase_					)
             _lowerCamelCase			:						Optional[Any]            =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)






      @unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'					)
      def 			lowerCamelCase			(   self		:	Tuple					)   ->		int:
             """simple docstring"""




             pass






      @unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'					)
      def 			lowerCamelCase			(   self		:	Any					)   ->		Optional[Any]:
             """simple docstring"""




             pass






      def 			lowerCamelCase			(   self		:	List[Any]					)   ->		str:
             """simple docstring"""




             _lowerCamelCase			:						Optional[Any]            ='I was born in 92000, and this is falsé.'
             _lowerCamelCase			:						int            =['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
             # fmt: on

             _lowerCamelCase			:						int            =DebertaVaTokenizer(lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						List[Any]            =tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						Union[str, Any]            =DebertaVaTokenizerFast(lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						Tuple            =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	Optional[Any]					)   ->		Dict:
             """simple docstring"""




             _lowerCamelCase			:						Union[str, Any]            ='I was born in 92000, and this is falsé.'
             _lowerCamelCase			:						Union[str, Any]            =['▁i', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
             # fmt: on

             _lowerCamelCase			:						Optional[int]            =DebertaVaTokenizer(lowercase_     ,  do_lower_case=lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						Any            =tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)
             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						List[Any]            =DebertaVaTokenizerFast(lowercase_     ,  do_lower_case=lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						Optional[int]            =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)
             self.assertListEqual(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	Any					)   ->		Any:
             """simple docstring"""




             _lowerCamelCase			:						int            ='I was born in 92000, and this is falsé.'
             _lowerCamelCase			:						Union[str, Any]            =['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
             # fmt: on

             _lowerCamelCase			:						Any            =DebertaVaTokenizer(lowercase_     ,  do_lower_case=lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						Tuple            =tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						Tuple            =DebertaVaTokenizerFast(lowercase_     ,  do_lower_case=lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						str            =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	Optional[Any]					)   ->		Any:
             """simple docstring"""




             _lowerCamelCase			:						Any            ='I was born in 92000, and this is falsé.'
             _lowerCamelCase			:						Optional[int]            =['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
             # fmt: on

             _lowerCamelCase			:						Optional[Any]            =DebertaVaTokenizer(lowercase_     ,  do_lower_case=lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						Dict            =tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						List[str]            =DebertaVaTokenizerFast(lowercase_     ,  do_lower_case=lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						Optional[Any]            =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	int					)   ->		Optional[int]:
             """simple docstring"""




             _lowerCamelCase			:						int            =' \tHeLLo!how  \n Are yoU?  '
             _lowerCamelCase			:						Optional[Any]            =['▁', '<unk>', 'e', '<unk>', 'o', '!', 'how', '▁', '<unk>', 're', '▁yo', '<unk>', '?']
             # fmt: on

             _lowerCamelCase			:						Optional[int]            =DebertaVaTokenizer(lowercase_     ,  do_lower_case=lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						Any            =tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						Optional[Any]            =DebertaVaTokenizerFast(lowercase_     ,  do_lower_case=lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						Any            =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	Union[str, Any]					)   ->		int:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            =self.get_tokenizer()
             _lowerCamelCase			:						Dict            =self.get_rust_tokenizer()

             _lowerCamelCase			:						str            ='I was born in 92000, and this is falsé.'

             _lowerCamelCase			:						Optional[Any]            =tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)
             _lowerCamelCase			:						Union[str, Any]            =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)
             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						List[str]            =tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
             _lowerCamelCase			:						Optional[int]            =rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						Any            =self.get_rust_tokenizer()
             _lowerCamelCase			:						List[Any]            =tokenizer.encode(lowercase_					)
             _lowerCamelCase			:						str            =rust_tokenizer.encode(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	List[Any]					)   ->		Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            ='This is a test'
             _lowerCamelCase			:						Optional[int]            =[13, 1, 4398, 25, 21, 1289]
             _lowerCamelCase			:						List[Any]            =['▁', 'T', 'his', '▁is', '▁a', '▁test']
             _lowerCamelCase			:						Optional[int]            =['▁', '<unk>', 'his', '▁is', '▁a', '▁test']

             _lowerCamelCase			:						Optional[Any]            =DebertaVaTokenizer(lowercase_     ,  keep_accents=lowercase_					)
             _lowerCamelCase			:						Any            =DebertaVaTokenizerFast(lowercase_     ,  keep_accents=lowercase_					)

             _lowerCamelCase			:						Any            =tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)
             _lowerCamelCase			:						Tuple            =tokenizer.tokenize(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)
             _lowerCamelCase			:						Union[str, Any]            =tokenizer.convert_ids_to_tokens(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						Dict            =rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)
             _lowerCamelCase			:						Optional[int]            =rust_tokenizer.tokenize(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)
             _lowerCamelCase			:						Any            =rust_tokenizer.convert_ids_to_tokens(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)

             # fmt: off
             _lowerCamelCase			:						Optional[Any]            ='I was born in 92000, and this is falsé.'
             _lowerCamelCase			:						str            =[13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
             _lowerCamelCase			:						Optional[int]            =['▁', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.', ]
             _lowerCamelCase			:						Dict            =['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
             # fmt: on

             _lowerCamelCase			:						Any            =tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)
             _lowerCamelCase			:						List[Any]            =tokenizer.tokenize(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)
             _lowerCamelCase			:						Optional[Any]            =tokenizer.convert_ids_to_tokens(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						Union[str, Any]            =rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)
             _lowerCamelCase			:						List[str]            =rust_tokenizer.tokenize(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)
             _lowerCamelCase			:						Union[str, Any]            =rust_tokenizer.convert_ids_to_tokens(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	List[Any]					)   ->		int:
             """simple docstring"""




             _lowerCamelCase			:						int            =DebertaVaTokenizer(lowercase_					)

             _lowerCamelCase			:						str            =tokenizer.encode('sequence builders'					)
             _lowerCamelCase			:						List[Any]            =tokenizer.encode('multi-sequence build'					)

             _lowerCamelCase			:						List[Any]            =tokenizer.build_inputs_with_special_tokens(lowercase_					)
             _lowerCamelCase			:						Dict            =tokenizer.build_inputs_with_special_tokens(lowercase_     ,  lowercase_					)

             self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]     ,  lowercase_					)
             self.assertEqual(
                 [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]     ,  lowercase_     ,  )






      @slow
      def 			lowerCamelCase			(   self		:	Dict					)   ->		Optional[Any]:
             """simple docstring"""




             _lowerCamelCase			:						Dict            ={'input_ids': [[1, 3_9867, 36, 1_9390, 486, 27, 3_5052, 8_1436, 18, 6_0685, 1225, 7, 3_5052, 8_1436, 18, 9367, 1_6899, 18, 1_5937, 53, 594, 773, 18, 1_6287, 3_0465, 36, 1_5937, 6, 4_1139, 38, 3_6979, 6_0763, 191, 6, 3_4132, 99, 6, 5_0538, 390, 4_3230, 6, 3_4132, 2779, 2_0850, 14, 699, 1072, 1194, 36, 382, 1_0901, 53, 7, 699, 1072, 2084, 36, 2_0422, 630, 53, 19, 105, 3049, 1896, 1053, 1_6899, 1506, 11, 3_7978, 4243, 7, 1237, 3_1869, 200, 1_6566, 654, 6, 3_5052, 8_1436, 7, 5_5630, 1_3593, 4, 2], [1, 26, 1_5011, 13, 667, 8, 1053, 18, 2_3611, 1237, 7_2356, 1_2820, 34, 10_4134, 1209, 35, 1_3313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 1_5785, 1_4951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=lowercase_     ,  model_name='microsoft/deberta-v2-xlarge'     ,  revision='ad6e42c1532ddf3a15c39246b63f5559d558b670'     ,  )


