



'''simple docstring'''




from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    convert_to_rgb,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


lowerCamelCase				:Any							       =   logging.get_logger(__name__)


if is_vision_available():
						import PIL







class       _lowerCAmelCase				(     UpperCamelCase_					):
	__SCREAMING_SNAKE_CASE  :						List[str]						     =   ['pixel_values']


	def __init__(self			, lowercase = True			, lowercase = None			, lowercase = PILImageResampling.BICUBIC			, lowercase = True			, lowercase = None			, lowercase = True			, lowercase = 1 / 255			, lowercase = True			, lowercase = None			, lowercase = None			, lowercase = True			, **lowercase			, ):
				super().__init__(**lowercase_				)
				A_      :		Optional[int]      =	size if size is not None else {'shortest_edge': 224}
				A_      :		List[Any]      =	get_size_dict(lowercase_			, default_to_square=lowercase_				)
				A_      :		str      =	crop_size if crop_size is not None else {'height': 224, 'width': 224}
				A_      :		str      =	get_size_dict(lowercase_			, default_to_square=lowercase_			, param_name="""crop_size"""				)

				A_      :		Dict      =	do_resize
				A_      :		int      =	size
				A_      :		Optional[Any]      =	resample
				A_      :		Optional[int]      =	do_center_crop
				A_      :		Tuple      =	crop_size
				A_      :		Optional[int]      =	do_rescale
				A_      :		Optional[Any]      =	rescale_factor
				A_      :		Union[str, Any]      =	do_normalize
				A_      :		Optional[int]      =	image_mean if image_mean is not None else OPENAI_CLIP_MEAN
				A_      :		Any      =	image_std if image_std is not None else OPENAI_CLIP_STD
				A_      :		Any      =	do_convert_rgb


	def        _a      (self			, lowercase			, lowercase			, lowercase = PILImageResampling.BICUBIC			, lowercase = None			, **lowercase			, ):
				A_      :		int      =	get_size_dict(lowercase_			, default_to_square=lowercase_				)
				if "shortest_edge" not in size:
							raise ValueError(F'The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}'				)
				A_      :		Union[str, Any]      =	get_resize_output_image_size(lowercase_			, size=size["""shortest_edge"""]			, default_to_square=lowercase_				)
				return resize(lowercase_			, size=lowercase_			, resample=lowercase_			, data_format=lowercase_			, **lowercase_				)


	def        _a      (self			, lowercase			, lowercase			, lowercase = None			, **lowercase			, ):
				A_      :		Union[str, Any]      =	get_size_dict(lowercase_				)
				if "height" not in size or "width" not in size:
							raise ValueError(F'The `size` parameter must contain the keys (height, width). Got {size.keys()}'				)
				return center_crop(lowercase_			, size=(size["""height"""], size["""width"""])			, data_format=lowercase_			, **lowercase_				)


	def        _a      (self			, lowercase			, lowercase			, lowercase = None			, **lowercase			, ):
				return rescale(lowercase_			, scale=lowercase_			, data_format=lowercase_			, **lowercase_				)


	def        _a      (self			, lowercase			, lowercase			, lowercase			, lowercase = None			, **lowercase			, ):
				return normalize(lowercase_			, mean=lowercase_			, std=lowercase_			, data_format=lowercase_			, **lowercase_				)


	def        _a      (self			, lowercase			, lowercase = None			, lowercase = None			, lowercase = None			, lowercase = None			, lowercase = None			, lowercase = None			, lowercase = None			, lowercase = None			, lowercase = None			, lowercase = None			, lowercase = None			, lowercase = None			, lowercase = ChannelDimension.FIRST			, **lowercase			, ):
				A_      :		Union[str, Any]      =	do_resize if do_resize is not None else self.do_resize
				A_      :		List[str]      =	size if size is not None else self.size
				A_      :		Any      =	get_size_dict(lowercase_			, param_name="""size"""			, default_to_square=lowercase_				)
				A_      :		str      =	resample if resample is not None else self.resample
				A_      :		List[Any]      =	do_center_crop if do_center_crop is not None else self.do_center_crop
				A_      :		Optional[Any]      =	crop_size if crop_size is not None else self.crop_size
				A_      :		Dict      =	get_size_dict(lowercase_			, param_name="""crop_size"""			, default_to_square=lowercase_				)
				A_      :		str      =	do_rescale if do_rescale is not None else self.do_rescale
				A_      :		Tuple      =	rescale_factor if rescale_factor is not None else self.rescale_factor
				A_      :		List[str]      =	do_normalize if do_normalize is not None else self.do_normalize
				A_      :		Tuple      =	image_mean if image_mean is not None else self.image_mean
				A_      :		int      =	image_std if image_std is not None else self.image_std
				A_      :		Tuple      =	do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

				A_      :		Any      =	make_list_of_images(lowercase_				)

				if not valid_images(lowercase_				):
							raise ValueError(
							    """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
							    """torch.Tensor, tf.Tensor or jax.ndarray."""				)

				if do_resize and size is None:
							raise ValueError("""Size must be specified if do_resize is True."""				)

				if do_center_crop and crop_size is None:
							raise ValueError("""Crop size must be specified if do_center_crop is True."""				)

				if do_rescale and rescale_factor is None:
							raise ValueError("""Rescale factor must be specified if do_rescale is True."""				)

				if do_normalize and (image_mean is None or image_std is None):
							raise ValueError("""Image mean and std must be specified if do_normalize is True."""				)

				# PIL RGBA images are converted to RGB
				if do_convert_rgb:
							A_      :		Tuple      =	[convert_to_rgb(lowercase_				) for image in images]

				# All transformations expect numpy arrays.
				A_      :		Optional[Any]      =	[to_numpy_array(lowercase_				) for image in images]

				if do_resize:
							A_      :		Optional[Any]      =	[self.resize(image=lowercase_			, size=lowercase_			, resample=lowercase_				) for image in images]

				if do_center_crop:
							A_      :		Optional[int]      =	[self.center_crop(image=lowercase_			, size=lowercase_				) for image in images]

				if do_rescale:
							A_      :		Optional[Any]      =	[self.rescale(image=lowercase_			, scale=lowercase_				) for image in images]

				if do_normalize:
							A_      :		List[Any]      =	[self.normalize(image=lowercase_			, mean=lowercase_			, std=lowercase_				) for image in images]

				A_      :		List[str]      =	[to_channel_dimension_format(lowercase_			, lowercase_				) for image in images]

				A_      :		Tuple      =	{'pixel_values': images}
				return BatchFeature(data=lowercase_			, tensor_type=lowercase_				)



import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase		   =			get_tests_dir('fixtures/spiece.model')




@require_sentencepiece
@require_tokenizers
class A    (  UpperCamelCase_       ,				unittest.TestCase	):
      UpperCamelCase__				:    Dict =DebertaVaTokenizer
      UpperCamelCase__				:    Union[str, Any] =DebertaVaTokenizerFast
      UpperCamelCase__				:    Optional[int] =True
      UpperCamelCase__				:    Tuple =True






      def 			lowerCamelCase			(   self		:	Tuple					)   ->		Optional[Any]:
             """simple docstring"""




             super().setUp()

             # We have a SentencePiece fixture for testing
             _lowerCamelCase			:						Optional[Any]            =DebertaVaTokenizer(lowercase_     ,  unk_token='<unk>'					)
             tokenizer.save_pretrained(self.tmpdirname					)






      def 			lowerCamelCase			(   self		:	Optional[Any]     ,  lowercase_		:	List[Any]					)   ->		Optional[int]:
             """simple docstring"""




             _lowerCamelCase			:						Dict            ='this is a test'
             _lowerCamelCase			:						List[Any]            ='this is a test'
             return input_text, output_text






      def 			lowerCamelCase			(   self		:	int					)   ->		str:
             """simple docstring"""




             _lowerCamelCase			:						Union[str, Any]            ='<pad>'
             _lowerCamelCase			:						Union[str, Any]            =0

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase_					)     ,  lowercase_					)
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase_					)     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	Optional[Any]					)   ->		Any:
             """simple docstring"""




             _lowerCamelCase			:						Optional[Any]            =list(self.get_tokenizer().get_vocab().keys()					)
             self.assertEqual(vocab_keys[0]     ,  '<pad>'					)
             self.assertEqual(vocab_keys[1]     ,  '<unk>'					)
             self.assertEqual(vocab_keys[-1]     ,  '[PAD]'					)
             self.assertEqual(len(lowercase_					)     ,  3_0001					)






      def 			lowerCamelCase			(   self		:	List[str]					)   ->		Union[str, Any]:
             """simple docstring"""




             self.assertEqual(self.get_tokenizer().vocab_size     ,  3_0000					)






      def 			lowerCamelCase			(   self		:	Union[str, Any]					)   ->		Optional[Any]:
             """simple docstring"""




             _lowerCamelCase			:						List[str]            =' \tHeLLo!how  \n Are yoU?  '
             _lowerCamelCase			:						List[Any]            =['▁hello', '!', 'how', '▁are', '▁you', '?']
             # fmt: on

             _lowerCamelCase			:						Any            =DebertaVaTokenizer(lowercase_     ,  do_lower_case=lowercase_					)
             _lowerCamelCase			:						Dict            =tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						Optional[Any]            =DebertaVaTokenizerFast(lowercase_     ,  do_lower_case=lowercase_					)
             _lowerCamelCase			:						Optional[Any]            =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)






      @unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'					)
      def 			lowerCamelCase			(   self		:	Tuple					)   ->		int:
             """simple docstring"""




             pass






      @unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'					)
      def 			lowerCamelCase			(   self		:	Any					)   ->		Optional[Any]:
             """simple docstring"""




             pass






      def 			lowerCamelCase			(   self		:	List[Any]					)   ->		str:
             """simple docstring"""




             _lowerCamelCase			:						Optional[Any]            ='I was born in 92000, and this is falsé.'
             _lowerCamelCase			:						int            =['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
             # fmt: on

             _lowerCamelCase			:						int            =DebertaVaTokenizer(lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						List[Any]            =tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						Union[str, Any]            =DebertaVaTokenizerFast(lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						Tuple            =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	Optional[Any]					)   ->		Dict:
             """simple docstring"""




             _lowerCamelCase			:						Union[str, Any]            ='I was born in 92000, and this is falsé.'
             _lowerCamelCase			:						Union[str, Any]            =['▁i', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
             # fmt: on

             _lowerCamelCase			:						Optional[int]            =DebertaVaTokenizer(lowercase_     ,  do_lower_case=lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						Any            =tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)
             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						List[Any]            =DebertaVaTokenizerFast(lowercase_     ,  do_lower_case=lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						Optional[int]            =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)
             self.assertListEqual(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	Any					)   ->		Any:
             """simple docstring"""




             _lowerCamelCase			:						int            ='I was born in 92000, and this is falsé.'
             _lowerCamelCase			:						Union[str, Any]            =['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
             # fmt: on

             _lowerCamelCase			:						Any            =DebertaVaTokenizer(lowercase_     ,  do_lower_case=lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						Tuple            =tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						Tuple            =DebertaVaTokenizerFast(lowercase_     ,  do_lower_case=lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						str            =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	Optional[Any]					)   ->		Any:
             """simple docstring"""




             _lowerCamelCase			:						Any            ='I was born in 92000, and this is falsé.'
             _lowerCamelCase			:						Optional[int]            =['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
             # fmt: on

             _lowerCamelCase			:						Optional[Any]            =DebertaVaTokenizer(lowercase_     ,  do_lower_case=lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						Dict            =tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						List[str]            =DebertaVaTokenizerFast(lowercase_     ,  do_lower_case=lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						Optional[Any]            =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	int					)   ->		Optional[int]:
             """simple docstring"""




             _lowerCamelCase			:						int            =' \tHeLLo!how  \n Are yoU?  '
             _lowerCamelCase			:						Optional[Any]            =['▁', '<unk>', 'e', '<unk>', 'o', '!', 'how', '▁', '<unk>', 're', '▁yo', '<unk>', '?']
             # fmt: on

             _lowerCamelCase			:						Optional[int]            =DebertaVaTokenizer(lowercase_     ,  do_lower_case=lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						Any            =tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						Optional[Any]            =DebertaVaTokenizerFast(lowercase_     ,  do_lower_case=lowercase_     ,  split_by_punct=lowercase_					)
             _lowerCamelCase			:						Any            =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)

             self.assertListEqual(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	Union[str, Any]					)   ->		int:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            =self.get_tokenizer()
             _lowerCamelCase			:						Dict            =self.get_rust_tokenizer()

             _lowerCamelCase			:						str            ='I was born in 92000, and this is falsé.'

             _lowerCamelCase			:						Optional[Any]            =tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)
             _lowerCamelCase			:						Union[str, Any]            =rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)					)
             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						List[str]            =tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
             _lowerCamelCase			:						Optional[int]            =rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						Any            =self.get_rust_tokenizer()
             _lowerCamelCase			:						List[Any]            =tokenizer.encode(lowercase_					)
             _lowerCamelCase			:						str            =rust_tokenizer.encode(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	List[Any]					)   ->		Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase			:						List[Any]            ='This is a test'
             _lowerCamelCase			:						Optional[int]            =[13, 1, 4398, 25, 21, 1289]
             _lowerCamelCase			:						List[Any]            =['▁', 'T', 'his', '▁is', '▁a', '▁test']
             _lowerCamelCase			:						Optional[int]            =['▁', '<unk>', 'his', '▁is', '▁a', '▁test']

             _lowerCamelCase			:						Optional[Any]            =DebertaVaTokenizer(lowercase_     ,  keep_accents=lowercase_					)
             _lowerCamelCase			:						Any            =DebertaVaTokenizerFast(lowercase_     ,  keep_accents=lowercase_					)

             _lowerCamelCase			:						Any            =tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)
             _lowerCamelCase			:						Tuple            =tokenizer.tokenize(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)
             _lowerCamelCase			:						Union[str, Any]            =tokenizer.convert_ids_to_tokens(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						Dict            =rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)
             _lowerCamelCase			:						Optional[int]            =rust_tokenizer.tokenize(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)
             _lowerCamelCase			:						Any            =rust_tokenizer.convert_ids_to_tokens(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)

             # fmt: off
             _lowerCamelCase			:						Optional[Any]            ='I was born in 92000, and this is falsé.'
             _lowerCamelCase			:						str            =[13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
             _lowerCamelCase			:						Optional[int]            =['▁', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.', ]
             _lowerCamelCase			:						Dict            =['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
             # fmt: on

             _lowerCamelCase			:						Any            =tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)
             _lowerCamelCase			:						List[Any]            =tokenizer.tokenize(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)
             _lowerCamelCase			:						Optional[Any]            =tokenizer.convert_ids_to_tokens(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)

             _lowerCamelCase			:						Union[str, Any]            =rust_tokenizer.encode(lowercase_     ,  add_special_tokens=lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)
             _lowerCamelCase			:						List[str]            =rust_tokenizer.tokenize(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)
             _lowerCamelCase			:						Union[str, Any]            =rust_tokenizer.convert_ids_to_tokens(lowercase_					)
             self.assertListEqual(lowercase_     ,  lowercase_					)






      def 			lowerCamelCase			(   self		:	List[Any]					)   ->		int:
             """simple docstring"""




             _lowerCamelCase			:						int            =DebertaVaTokenizer(lowercase_					)

             _lowerCamelCase			:						str            =tokenizer.encode('sequence builders'					)
             _lowerCamelCase			:						List[Any]            =tokenizer.encode('multi-sequence build'					)

             _lowerCamelCase			:						List[Any]            =tokenizer.build_inputs_with_special_tokens(lowercase_					)
             _lowerCamelCase			:						Dict            =tokenizer.build_inputs_with_special_tokens(lowercase_     ,  lowercase_					)

             self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]     ,  lowercase_					)
             self.assertEqual(
                 [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]     ,  lowercase_     ,  )






      @slow
      def 			lowerCamelCase			(   self		:	Dict					)   ->		Optional[Any]:
             """simple docstring"""




             _lowerCamelCase			:						Dict            ={'input_ids': [[1, 3_9867, 36, 1_9390, 486, 27, 3_5052, 8_1436, 18, 6_0685, 1225, 7, 3_5052, 8_1436, 18, 9367, 1_6899, 18, 1_5937, 53, 594, 773, 18, 1_6287, 3_0465, 36, 1_5937, 6, 4_1139, 38, 3_6979, 6_0763, 191, 6, 3_4132, 99, 6, 5_0538, 390, 4_3230, 6, 3_4132, 2779, 2_0850, 14, 699, 1072, 1194, 36, 382, 1_0901, 53, 7, 699, 1072, 2084, 36, 2_0422, 630, 53, 19, 105, 3049, 1896, 1053, 1_6899, 1506, 11, 3_7978, 4243, 7, 1237, 3_1869, 200, 1_6566, 654, 6, 3_5052, 8_1436, 7, 5_5630, 1_3593, 4, 2], [1, 26, 1_5011, 13, 667, 8, 1053, 18, 2_3611, 1237, 7_2356, 1_2820, 34, 10_4134, 1209, 35, 1_3313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 1_5785, 1_4951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=lowercase_     ,  model_name='microsoft/deberta-v2-xlarge'     ,  revision='ad6e42c1532ddf3a15c39246b63f5559d558b670'     ,  )


