
from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import center_crop, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
					import PIL


UpperCamelCase_   	=	logging.get_logger(__name__)

class       a		(   __UpperCAmelCase					):
							lowercase_		:						Dict   							=  ['pixel_values']







							def __init__(						self  :  Optional[Any]      ,				snake_case__  :  bool = True      ,				snake_case__  :  Dict[str, int] = None      ,				snake_case__  :  PILImageResampling = PIL.Image.BICUBIC      ,				snake_case__  :  bool = True      ,				snake_case__  :  Dict[str, int] = None      ,				snake_case__  :  Union[int, float] = 1 / 255      ,				snake_case__  :  bool = True      ,				snake_case__  :  bool = True      ,				snake_case__  :  Optional[Union[float, List[float]]] = None      ,				snake_case__  :  Optional[Union[float, List[float]]] = None      ,				**snake_case__  :  Tuple      ,				):
								"""simple docstring"""







								super().__init__(**snake_case__ )
								__lowerCAmelCase         =							size if size is not None else {"height": 256, "width": 256}
								__lowerCAmelCase         =							get_size_dict(snake_case__ )
								__lowerCAmelCase         =							crop_size if crop_size is not None else {"height": 224, "width": 224}
								__lowerCAmelCase         =							get_size_dict(snake_case__      ,				param_name="crop_size" )

								__lowerCAmelCase         =							do_resize
								__lowerCAmelCase         =							size
								__lowerCAmelCase         =							resample
								__lowerCAmelCase         =							do_center_crop
								__lowerCAmelCase         =							crop_size
								__lowerCAmelCase         =							do_rescale
								__lowerCAmelCase         =							rescale_factor
								__lowerCAmelCase         =							do_normalize
								__lowerCAmelCase         =							image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
								__lowerCAmelCase         =							image_std if image_std is not None else IMAGENET_STANDARD_STD







							def       UpperCAmelCase__					(						self  :  int      ,				snake_case__  :  np.ndarray      ,				snake_case__  :  Dict[str, int]      ,				snake_case__  :  PILImageResampling = PIL.Image.BICUBIC      ,				snake_case__  :  Optional[Union[str, ChannelDimension]] = None      ,				**snake_case__  :  Optional[Any]      ,				):
								"""simple docstring"""







								__lowerCAmelCase         =							get_size_dict(snake_case__ )
								if "height" not in size or "width" not in size:
									raise ValueError(F"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}" )
								return resize(
								    snake_case__      ,				size=(size["height"], size["width"])      ,				resample=snake_case__      ,				data_format=snake_case__      ,				**snake_case__ )







							def       UpperCAmelCase__					(						self  :  List[str]      ,				snake_case__  :  np.ndarray      ,				snake_case__  :  Dict[str, int]      ,				snake_case__  :  Optional[Union[str, ChannelDimension]] = None      ,				**snake_case__  :  Union[str, Any]      ,				):
								"""simple docstring"""







								__lowerCAmelCase         =							get_size_dict(snake_case__ )
								if "height" not in size or "width" not in size:
									raise ValueError(F"The size dictionary must have keys 'height' and 'width'. Got {size.keys()}" )
								return center_crop(snake_case__      ,				size=(size["height"], size["width"])      ,				data_format=snake_case__      ,				**snake_case__ )







							def       UpperCAmelCase__					(						self  :  Union[str, Any]      ,				snake_case__  :  np.ndarray      ,				snake_case__  :  Union[int, float]      ,				snake_case__  :  Optional[Union[str, ChannelDimension]] = None      ,				**snake_case__  :  Union[str, Any]      ,				):
								"""simple docstring"""







								return rescale(snake_case__      ,				scale=snake_case__      ,				data_format=snake_case__      ,				**snake_case__ )







							def       UpperCAmelCase__					(						self  :  Tuple      ,				snake_case__  :  np.ndarray      ,				snake_case__  :  Union[float, List[float]]      ,				snake_case__  :  Union[float, List[float]]      ,				snake_case__  :  Optional[Union[str, ChannelDimension]] = None      ,				**snake_case__  :  Tuple      ,				):
								"""simple docstring"""







								return normalize(snake_case__      ,				mean=snake_case__      ,				std=snake_case__      ,				data_format=snake_case__      ,				**snake_case__ )







							def       UpperCAmelCase__					(						self  :  Union[str, Any]      ,				snake_case__  :  ImageInput      ,				snake_case__  :  bool = None      ,				snake_case__  :  Dict[str, int] = None      ,				snake_case__  :  str=None      ,				snake_case__  :  bool = None      ,				snake_case__  :  Dict[str, int] = None      ,				snake_case__  :  bool = None      ,				snake_case__  :  float = None      ,				snake_case__  :  bool = None      ,				snake_case__  :  Optional[Union[float, List[float]]] = None      ,				snake_case__  :  Optional[Union[float, List[float]]] = None      ,				snake_case__  :  Optional[Union[str, TensorType]] = None      ,				snake_case__  :  ChannelDimension = ChannelDimension.FIRST      ,				**snake_case__  :  int      ,				):
								"""simple docstring"""







								__lowerCAmelCase         =							do_resize if do_resize is not None else self.do_resize
								__lowerCAmelCase         =							resample if resample is not None else self.resample
								__lowerCAmelCase         =							do_center_crop if do_center_crop is not None else self.do_center_crop
								__lowerCAmelCase         =							do_rescale if do_rescale is not None else self.do_rescale
								__lowerCAmelCase         =							rescale_factor if rescale_factor is not None else self.rescale_factor
								__lowerCAmelCase         =							do_normalize if do_normalize is not None else self.do_normalize
								__lowerCAmelCase         =							image_mean if image_mean is not None else self.image_mean
								__lowerCAmelCase         =							image_std if image_std is not None else self.image_std

								__lowerCAmelCase         =							size if size is not None else self.size
								__lowerCAmelCase         =							get_size_dict(snake_case__ )
								__lowerCAmelCase         =							crop_size if crop_size is not None else self.crop_size
								__lowerCAmelCase         =							get_size_dict(snake_case__      ,				param_name="crop_size" )

								__lowerCAmelCase         =							make_list_of_images(snake_case__ )

								if not valid_images(snake_case__ ):
									raise ValueError(
									    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
									    "torch.Tensor, tf.Tensor or jax.ndarray." )

								if do_resize and size is None or resample is None:
									raise ValueError("Size and resample must be specified if do_resize is True." )

								if do_center_crop and crop_size is None:
									raise ValueError("Crop size must be specified if do_center_crop is True." )

								if do_rescale and rescale_factor is None:
									raise ValueError("Rescale factor must be specified if do_rescale is True." )

								if do_normalize and (image_mean is None or image_std is None):
									raise ValueError("Image mean and std must be specified if do_normalize is True." )

								# All transformations expect numpy arrays.
								__lowerCAmelCase         =							[to_numpy_array(snake_case__ ) for image in images]

								if do_resize:
									__lowerCAmelCase         =							[self.resize(image=snake_case__      ,				size=snake_case__      ,				resample=snake_case__ ) for image in images]

								if do_center_crop:
									__lowerCAmelCase         =							[self.center_crop(image=snake_case__      ,				size=snake_case__ ) for image in images]

								if do_rescale:
									__lowerCAmelCase         =							[self.rescale(image=snake_case__      ,				scale=snake_case__ ) for image in images]

								if do_normalize:
									__lowerCAmelCase         =							[self.normalize(image=snake_case__      ,				mean=snake_case__      ,				std=snake_case__ ) for image in images]

								__lowerCAmelCase         =							[to_channel_dimension_format(snake_case__      ,				snake_case__ ) for image in images]

								__lowerCAmelCase         =							{"pixel_values": images}
								return BatchFeature(data=snake_case__      ,				tensor_type=snake_case__ )








import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase_   	=	get_tests_dir("fixtures/spiece.model")

@require_sentencepiece
@require_tokenizers
class       a		(   __UpperCAmelCase      ,	unittest.TestCase					):
							lowercase_		:						Union[str, Any]   							=  DebertaVaTokenizer
							lowercase_		:						List[str]   							=  DebertaVaTokenizerFast
							lowercase_		:						str   							=  True
							lowercase_		:						List[Any]   							=  True







							def       UpperCAmelCase__					(						self  :  Optional[Any] ):
								"""simple docstring"""







								super().setUp()

								# We have a SentencePiece fixture for testing
								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__      ,				unk_token="<unk>" )
								tokenizer.save_pretrained(self.tmpdirname )







							def       UpperCAmelCase__					(						self  :  str      ,				snake_case__  :  Dict ):
								"""simple docstring"""







								__lowerCAmelCase         =							"this is a test"
								__lowerCAmelCase         =							"this is a test"
								return input_text, output_text







							def       UpperCAmelCase__					(						self  :  int ):
								"""simple docstring"""







								__lowerCAmelCase         =							"<pad>"
								__lowerCAmelCase         =							0

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__ )      ,				snake_case__ )
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__ )      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  Any ):
								"""simple docstring"""







								__lowerCAmelCase         =							list(self.get_tokenizer().get_vocab().keys() )
								self.assertEqual(vocab_keys[0]      ,				"<pad>" )
								self.assertEqual(vocab_keys[1]      ,				"<unk>" )
								self.assertEqual(vocab_keys[-1]      ,				"[PAD]" )
								self.assertEqual(len(snake_case__ )      ,				30_001 )







							def       UpperCAmelCase__					(						self  :  Tuple ):
								"""simple docstring"""







								self.assertEqual(self.get_tokenizer().vocab_size      ,				30_000 )







							def       UpperCAmelCase__					(						self  :  List[str] ):
								"""simple docstring"""







								__lowerCAmelCase         =							" \tHeLLo!how  \n Are yoU?  "
								__lowerCAmelCase         =							["▁hello", "!", "how", "▁are", "▁you", "?"]
								# fmt: on

								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__      ,				do_lower_case=snake_case__ )
								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							DebertaVaTokenizerFast(snake_case__      ,				do_lower_case=snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )







							@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one." )
							def       UpperCAmelCase__					(						self  :  Union[str, Any] ):
								"""simple docstring"""







								pass







							@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one." )
							def       UpperCAmelCase__					(						self  :  List[str] ):
								"""simple docstring"""







								pass







							def       UpperCAmelCase__					(						self  :  Any ):
								"""simple docstring"""







								__lowerCAmelCase         =							"I was born in 92000, and this is falsé."
								__lowerCAmelCase         =							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
								# fmt: on

								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							DebertaVaTokenizerFast(snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  Tuple ):
								"""simple docstring"""







								__lowerCAmelCase         =							"I was born in 92000, and this is falsé."
								__lowerCAmelCase         =							["▁i", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
								# fmt: on

								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__      ,				do_lower_case=snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )
								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							DebertaVaTokenizerFast(snake_case__      ,				do_lower_case=snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )
								self.assertListEqual(snake_case__      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  Optional[int] ):
								"""simple docstring"""







								__lowerCAmelCase         =							"I was born in 92000, and this is falsé."
								__lowerCAmelCase         =							["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
								# fmt: on

								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__      ,				do_lower_case=snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							DebertaVaTokenizerFast(snake_case__      ,				do_lower_case=snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  int ):
								"""simple docstring"""







								__lowerCAmelCase         =							"I was born in 92000, and this is falsé."
								__lowerCAmelCase         =							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
								# fmt: on

								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__      ,				do_lower_case=snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							DebertaVaTokenizerFast(snake_case__      ,				do_lower_case=snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  Tuple ):
								"""simple docstring"""







								__lowerCAmelCase         =							" \tHeLLo!how  \n Are yoU?  "
								__lowerCAmelCase         =							["▁", "<unk>", "e", "<unk>", "o", "!", "how", "▁", "<unk>", "re", "▁yo", "<unk>", "?"]
								# fmt: on

								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__      ,				do_lower_case=snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							DebertaVaTokenizerFast(snake_case__      ,				do_lower_case=snake_case__      ,				split_by_punct=snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )

								self.assertListEqual(snake_case__      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  Optional[int] ):
								"""simple docstring"""







								__lowerCAmelCase         =							self.get_tokenizer()
								__lowerCAmelCase         =							self.get_rust_tokenizer()

								__lowerCAmelCase         =							"I was born in 92000, and this is falsé."

								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ ) )
								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							self.get_rust_tokenizer()
								__lowerCAmelCase         =							tokenizer.encode(snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.encode(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  Optional[int] ):
								"""simple docstring"""







								__lowerCAmelCase         =							"This is a test"
								__lowerCAmelCase         =							[13, 1, 4_398, 25, 21, 1_289]
								__lowerCAmelCase         =							["▁", "T", "his", "▁is", "▁a", "▁test"]
								__lowerCAmelCase         =							["▁", "<unk>", "his", "▁is", "▁a", "▁test"]

								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__      ,				keep_accents=snake_case__ )
								__lowerCAmelCase         =							DebertaVaTokenizerFast(snake_case__      ,				keep_accents=snake_case__ )

								__lowerCAmelCase         =							tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							tokenizer.tokenize(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.tokenize(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )

								# fmt: off
								__lowerCAmelCase         =							"I was born in 92000, and this is falsé."
								__lowerCAmelCase         =							[13, 1, 23, 386, 19, 561, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]
								__lowerCAmelCase         =							["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", ".", ]
								__lowerCAmelCase         =							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
								# fmt: on

								__lowerCAmelCase         =							tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							tokenizer.tokenize(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							tokenizer.convert_ids_to_tokens(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )

								__lowerCAmelCase         =							rust_tokenizer.encode(snake_case__      ,				add_special_tokens=snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.tokenize(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )
								__lowerCAmelCase         =							rust_tokenizer.convert_ids_to_tokens(snake_case__ )
								self.assertListEqual(snake_case__      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  Any ):
								"""simple docstring"""







								__lowerCAmelCase         =							DebertaVaTokenizer(snake_case__ )

								__lowerCAmelCase         =							tokenizer.encode("sequence builders" )
								__lowerCAmelCase         =							tokenizer.encode("multi-sequence build" )

								__lowerCAmelCase         =							tokenizer.build_inputs_with_special_tokens(snake_case__ )
								__lowerCAmelCase         =							tokenizer.build_inputs_with_special_tokens(snake_case__      ,				snake_case__ )

								self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]      ,				snake_case__ )
								self.assertEqual(
								    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]      ,				snake_case__      ,				)







							@slow
							def       UpperCAmelCase__					(						self  :  Optional[int] ):
								"""simple docstring"""







								__lowerCAmelCase         =							{"input_ids": [[1, 39_867, 36, 19_390, 486, 27, 35_052, 81_436, 18, 60_685, 1_225, 7, 35_052, 81_436, 18, 9_367, 16_899, 18, 15_937, 53, 594, 773, 18, 16_287, 30_465, 36, 15_937, 6, 41_139, 38, 36_979, 60_763, 191, 6, 34_132, 99, 6, 50_538, 390, 43_230, 6, 34_132, 2_779, 20_850, 14, 699, 1_072, 1_194, 36, 382, 10_901, 53, 7, 699, 1_072, 2_084, 36, 20_422, 630, 53, 19, 105, 3_049, 1_896, 1_053, 16_899, 1_506, 11, 37_978, 4_243, 7, 1_237, 31_869, 200, 16_566, 654, 6, 35_052, 81_436, 7, 55_630, 13_593, 4, 2], [1, 26, 15_011, 13, 667, 8, 1_053, 18, 23_611, 1_237, 72_356, 12_820, 34, 104_134, 1_209, 35, 13_313, 6_627, 21, 202, 347, 7, 164, 2_399, 11, 46, 4_485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_232, 2_864, 15_785, 14_951, 105, 5, 8_581, 1_250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=snake_case__      ,				model_name="microsoft/deberta-v2-xlarge"      ,				revision="ad6e42c1532ddf3a15c39246b63f5559d558b670"      ,				)






