import inspect
import tempfile
from collections import OrderedDict, UserDict
from collections.abc import MutableMapping
from contextlib import ExitStack, contextmanager
from dataclasses import fields
from enum import Enum
from typing import Any, ContextManager, List, Tuple

import numpy as np

from .import_utils import is_flax_available, is_tf_available, is_torch_available, is_torch_fx_proxy


if is_flax_available():
					import jax.numpy as jnp
class     lowerCAmelCase_     (       __lowercase ):







							def __get__(   self      :     Union[str, Any] ,			_A      :     Any ,			_A      :     Union[str, Any]=None		):
												# See docs.python.org/3/howto/descriptor.html#properties
												if obj is None:
																	return self
												if self.fget is None:
																	raise AttributeError('''unreadable attribute'''		)
												_UpperCamelCase			     =     '''__cached_''' + self.fget.__name__
												_UpperCamelCase			     =     getattr(_A ,			_A ,			_A		)
												if cached is None:
																	_UpperCamelCase			     =     self.fget(_A		)
																	setattr(_A ,			_A ,			_A		)
												return cached




def 					_snake_case	(  __snake_case						):
					_UpperCamelCase			     =     val.lower()
					if val in {"y", "yes", "t", "true", "on", "1"}:
										return 1
					if val in {"n", "no", "f", "false", "off", "0"}:
										return 0
					raise ValueError(f"""invalid truth value {val!r}"""						)
def 					_snake_case	(  __snake_case						):
					if is_torch_fx_proxy(__snake_case						):
										return True
					if is_torch_available():
										import torch

										if isinstance(__snake_case  ,     torch.Tensor						):
															return True
					if is_tf_available():
										import tensorflow as tf

										if isinstance(__snake_case  ,     tf.Tensor						):
															return True

					if is_flax_available():
										import jax.numpy as jnp
										from jax.core import Tracer

										if isinstance(__snake_case  ,     (jnp.ndarray, Tracer)						):
															return True

					return isinstance(__snake_case  ,     np.ndarray						)
def 					_snake_case	(  __snake_case						):
					return isinstance(__snake_case  ,     np.ndarray						)
def 					_snake_case	(  __snake_case						):
					return _is_numpy(__snake_case						)
def 					_snake_case	(  __snake_case						):
					import torch

					return isinstance(__snake_case  ,     torch.Tensor						)
def 					_snake_case	(  __snake_case						):
					return False if not is_torch_available() else _is_torch(__snake_case						)
def 					_snake_case	(  __snake_case						):
					import torch

					return isinstance(__snake_case  ,     torch.device						)
def 					_snake_case	(  __snake_case						):
					return False if not is_torch_available() else _is_torch_device(__snake_case						)
def 					_snake_case	(  __snake_case						):
					import torch

					if isinstance(__snake_case  ,     __snake_case						):
										if hasattr(__snake_case  ,     __snake_case						):
															_UpperCamelCase			     =     getattr(__snake_case  ,     __snake_case						)
										else:
															return False
					return isinstance(__snake_case  ,     torch.dtype						)
def 					_snake_case	(  __snake_case						):
					return False if not is_torch_available() else _is_torch_dtype(__snake_case						)
def 					_snake_case	(  __snake_case						):
					import tensorflow as tf

					return isinstance(__snake_case  ,     tf.Tensor						)
def 					_snake_case	(  __snake_case						):
					return False if not is_tf_available() else _is_tensorflow(__snake_case						)
def 					_snake_case	(  __snake_case						):
					import tensorflow as tf

					# the `is_symbolic_tensor` predicate is only available starting with TF 2.14
					if hasattr(__snake_case  ,     '''is_symbolic_tensor'''						):
										return tf.is_symbolic_tensor(__snake_case						)
					return type(__snake_case						) == tf.Tensor
def 					_snake_case	(  __snake_case						):
					return False if not is_tf_available() else _is_tf_symbolic_tensor(__snake_case						)
def 					_snake_case	(  __snake_case						):
					import jax.numpy as jnp  # noqa: F811

					return isinstance(__snake_case  ,     jnp.ndarray						)
def 					_snake_case	(  __snake_case						):
					return False if not is_flax_available() else _is_jax(__snake_case						)
def 					_snake_case	(  __snake_case						):
					if isinstance(__snake_case  ,     (dict, UserDict)						):
										return {k: to_py_obj(__snake_case						) for k, v in obj.items()}
					elif isinstance(__snake_case  ,     (list, tuple)						):
										return [to_py_obj(__snake_case						) for o in obj]
					elif is_tf_tensor(__snake_case						):
										return obj.numpy().tolist()
					elif is_torch_tensor(__snake_case						):
										return obj.detach().cpu().tolist()
					elif is_jax_tensor(__snake_case						):
										return np.asarray(__snake_case						).tolist()
					elif isinstance(__snake_case  ,     (np.ndarray, np.number)						):  # tolist also works on 0d np arrays
										return obj.tolist()
					else:
										return obj
def 					_snake_case	(  __snake_case						):
					if isinstance(__snake_case  ,     (dict, UserDict)						):
										return {k: to_numpy(__snake_case						) for k, v in obj.items()}
					elif isinstance(__snake_case  ,     (list, tuple)						):
										return np.array(__snake_case						)
					elif is_tf_tensor(__snake_case						):
										return obj.numpy()
					elif is_torch_tensor(__snake_case						):
										return obj.detach().cpu().numpy()
					elif is_jax_tensor(__snake_case						):
										return np.asarray(__snake_case						)
					else:
										return obj
class     lowerCAmelCase_     (       __lowercase ):







							def  UpperCamelCase_						(   self      :     Tuple		):
												_UpperCamelCase			     =     fields(self		)

												# Safety and consistency checks
												if not len(_A		):
																	raise ValueError(F"""{self.__class__.__name__} has no fields."""		)
												if not all(field.default is None for field in class_fields[1:]		):
																	raise ValueError(F"""{self.__class__.__name__} should not have more than one required field."""		)

												_UpperCamelCase			     =     getattr(self ,			class_fields[0].name		)
												_UpperCamelCase			     =     all(getattr(self ,			field.name		) is None for field in class_fields[1:]		)

												if other_fields_are_none and not is_tensor(_A		):
																	if isinstance(_A ,			_A		):
																						_UpperCamelCase			     =     first_field.items()
																						_UpperCamelCase			     =     True
																	else:
																						try:
																											_UpperCamelCase			     =     iter(_A		)
																											_UpperCamelCase			     =     True
																						except TypeError:
																											_UpperCamelCase			     =     False

            # if we provided an iterator as first field and the iterator is a (key, value) iterator
            # set the associated fields
																	if first_field_iterator:
																						for idx, element in enumerate(_A		):
																											if (
																											    not isinstance(_A ,			(list, tuple)		)
																											    or not len(_A		) == 2
																											    or not isinstance(element[0] ,			_A		)
																											):
																																if idx == 0:
																																					# If we do not have an iterator of key/values, set it as attribute
																																					_UpperCamelCase			     =     first_field
																																else:
																																					# If we have a mixed iterator, raise an error
																																					raise ValueError(
																																					    F"""Cannot set key/value for {element}. It needs to be a tuple (key, value)."""		)
																																break
																											setattr(self ,			element[0] ,			element[1]		)
																											if element[1] is not None:
																																_UpperCamelCase			     =     element[1]
																	elif first_field is not None:
																						_UpperCamelCase			     =     first_field
												else:
																	for field in class_fields:
																						_UpperCamelCase			     =     getattr(self ,			field.name		)
																						if v is not None:
																											_UpperCamelCase			     =     v







							def __delitem__(   self      :     Any ,			*_A      :     Union[str, Any] ,			**_A      :     Tuple		):
												raise Exception(F"""You cannot use ``__delitem__`` on a {self.__class__.__name__} instance."""		)







							def  UpperCamelCase_						(   self      :     Optional[Any] ,			*_A      :     Dict ,			**_A      :     Any		):
												raise Exception(F"""You cannot use ``setdefault`` on a {self.__class__.__name__} instance."""		)







							def  UpperCamelCase_						(   self      :     Dict ,			*_A      :     int ,			**_A      :     Any		):
												raise Exception(F"""You cannot use ``pop`` on a {self.__class__.__name__} instance."""		)







							def  UpperCamelCase_						(   self      :     Union[str, Any] ,			*_A      :     List[str] ,			**_A      :     int		):
												raise Exception(F"""You cannot use ``update`` on a {self.__class__.__name__} instance."""		)







							def __getitem__(   self      :     Dict ,			_A      :     str		):
												if isinstance(_A ,			_A		):
																	_UpperCamelCase			     =     dict(self.items()		)
																	return inner_dict[k]
												else:
																	return self.to_tuple()[k]







							def __setattr__(   self      :     int ,			_A      :     Dict ,			_A      :     Optional[Any]		):
												if name in self.keys() and value is not None:
																	# Don't call self.__setitem__ to avoid recursion errors
																	super().__setitem__(_A ,			_A		)
												super().__setattr__(_A ,			_A		)







							def __setitem__(   self      :     List[str] ,			_A      :     Optional[int] ,			_A      :     Optional[Any]		):
												# Will raise a KeyException if needed
												super().__setitem__(_A ,			_A		)
												# Don't call self.__setattr__ to avoid recursion errors
												super().__setattr__(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     Any		):
												return tuple(self[k] for k in self.keys()		)




class     lowerCAmelCase_     (       __lowercase,  __lowercase ):







							@classmethod
							def  UpperCamelCase_						(   cls      :     Optional[int] ,			_A      :     str		):
												raise ValueError(
												    F"""{value} is not a valid {cls.__name__}, please select one of {list(cls._valueamember_map_.keys()		)}"""		)




class     lowerCAmelCase_     (       __lowercase ):
							UpperCAmelCase				   =     "longest"
							UpperCAmelCase				   =     "max_length"
							UpperCAmelCase				   =     "do_not_pad"




class     lowerCAmelCase_     (       __lowercase ):
							UpperCAmelCase				   =     "pt"
							UpperCAmelCase				   =     "tf"
							UpperCAmelCase				   =     "np"
							UpperCAmelCase				   =     "jax"




class     lowerCAmelCase_     :







							def __init__(   self      :     Union[str, Any] ,			_A      :     List[ContextManager]		):
												_UpperCamelCase			     =     context_managers
												_UpperCamelCase			     =     ExitStack()







							def __enter__(   self      :     int		):
												for context_manager in self.context_managers:
																	self.stack.enter_context(_A		)







							def __exit__(   self      :     str ,			*_A      :     Dict ,			**_A      :     Tuple		):
												self.stack.__exit__(*_A ,			**_A		)




def 					_snake_case	(  __snake_case						):
					_UpperCamelCase			     =     infer_framework(__snake_case						)
					if framework == "tf":
										_UpperCamelCase			     =     inspect.signature(model_class.call						)  # TensorFlow models
					elif framework == "pt":
										_UpperCamelCase			     =     inspect.signature(model_class.forward						)  # PyTorch models
					else:
										_UpperCamelCase			     =     inspect.signature(model_class.__call__						)  # Flax models

					for p in signature.parameters:
										if p == "return_loss" and signature.parameters[p].default is True:
															return True

					return False
def 					_snake_case	(  __snake_case						):
					_UpperCamelCase			     =     model_class.__name__
					_UpperCamelCase			     =     infer_framework(__snake_case						)
					if framework == "tf":
										_UpperCamelCase			     =     inspect.signature(model_class.call						)  # TensorFlow models
					elif framework == "pt":
										_UpperCamelCase			     =     inspect.signature(model_class.forward						)  # PyTorch models
					else:
										_UpperCamelCase			     =     inspect.signature(model_class.__call__						)  # Flax models

					if "QuestionAnswering" in model_name:
										return [p for p in signature.parameters if "label" in p or p in ("start_positions", "end_positions")]
					else:
										return [p for p in signature.parameters if "label" in p]
def 					_snake_case	(  __snake_case  ,     __snake_case = ""  ,     __snake_case = "."						):

					def _flatten_dict(__snake_case  ,     __snake_case=""  ,     __snake_case="."						):
										for k, v in d.items():
															_UpperCamelCase			     =     str(__snake_case						) + delimiter + str(__snake_case						) if parent_key else k
															if v and isinstance(__snake_case  ,     __snake_case						):
																				yield from flatten_dict(__snake_case  ,     __snake_case  ,     delimiter=__snake_case						).items()
															else:
																				yield key, v

					return dict(_flatten_dict(__snake_case  ,     __snake_case  ,     __snake_case						)						)
@contextmanager
def 					_snake_case	(  __snake_case  ,     __snake_case = False						):
					if use_temp_dir:
										with tempfile.TemporaryDirectory() as tmp_dir:
															yield tmp_dir
					else:
										yield working_dir
def 					_snake_case	(  __snake_case  ,     __snake_case=None						):
					if is_numpy_array(__snake_case						):
										return np.transpose(__snake_case  ,     axes=__snake_case						)
					elif is_torch_tensor(__snake_case						):
										return array.T if axes is None else array.permute(*__snake_case						)
					elif is_tf_tensor(__snake_case						):
										import tensorflow as tf

										return tf.transpose(__snake_case  ,     perm=__snake_case						)
					elif is_jax_tensor(__snake_case						):
										return jnp.transpose(__snake_case  ,     axes=__snake_case						)
					else:
										raise ValueError(f"""Type not supported for transpose: {type(__snake_case						)}."""						)
def 					_snake_case	(  __snake_case  ,     __snake_case						):
					if is_numpy_array(__snake_case						):
										return np.reshape(__snake_case  ,     __snake_case						)
					elif is_torch_tensor(__snake_case						):
										return array.reshape(*__snake_case						)
					elif is_tf_tensor(__snake_case						):
										import tensorflow as tf

										return tf.reshape(__snake_case  ,     __snake_case						)
					elif is_jax_tensor(__snake_case						):
										return jnp.reshape(__snake_case  ,     __snake_case						)
					else:
										raise ValueError(f"""Type not supported for reshape: {type(__snake_case						)}."""						)
def 					_snake_case	(  __snake_case  ,     __snake_case=None						):
					if is_numpy_array(__snake_case						):
										return np.squeeze(__snake_case  ,     axis=__snake_case						)
					elif is_torch_tensor(__snake_case						):
										return array.squeeze() if axis is None else array.squeeze(dim=__snake_case						)
					elif is_tf_tensor(__snake_case						):
										import tensorflow as tf

										return tf.squeeze(__snake_case  ,     axis=__snake_case						)
					elif is_jax_tensor(__snake_case						):
										return jnp.squeeze(__snake_case  ,     axis=__snake_case						)
					else:
										raise ValueError(f"""Type not supported for squeeze: {type(__snake_case						)}."""						)
def 					_snake_case	(  __snake_case  ,     __snake_case						):
					if is_numpy_array(__snake_case						):
										return np.expand_dims(__snake_case  ,     __snake_case						)
					elif is_torch_tensor(__snake_case						):
										return array.unsqueeze(dim=__snake_case						)
					elif is_tf_tensor(__snake_case						):
										import tensorflow as tf

										return tf.expand_dims(__snake_case  ,     axis=__snake_case						)
					elif is_jax_tensor(__snake_case						):
										return jnp.expand_dims(__snake_case  ,     axis=__snake_case						)
					else:
										raise ValueError(f"""Type not supported for expand_dims: {type(__snake_case						)}."""						)
def 					_snake_case	(  __snake_case						):
					if is_numpy_array(__snake_case						):
										return np.size(__snake_case						)
					elif is_torch_tensor(__snake_case						):
										return array.numel()
					elif is_tf_tensor(__snake_case						):
										import tensorflow as tf

										return tf.size(__snake_case						)
					elif is_jax_tensor(__snake_case						):
										return array.size
					else:
										raise ValueError(f"""Type not supported for expand_dims: {type(__snake_case						)}."""						)
def 					_snake_case	(  __snake_case  ,     __snake_case						):
					for key, value in auto_map.items():
										if isinstance(__snake_case  ,     (tuple, list)						):
															_UpperCamelCase			     =     [f"""{repo_id}--{v}""" if (v is not None and '''--''' not in v) else v for v in value]
										elif value is not None and "--" not in value:
															_UpperCamelCase			     =     f"""{repo_id}--{value}"""

					return auto_map







def 					_snake_case	(  __snake_case						):
					for base_class in inspect.getmro(__snake_case						):
										_UpperCamelCase			     =     base_class.__module__
										_UpperCamelCase			     =     base_class.__name__
										if module.startswith('''tensorflow'''						) or module.startswith('''keras'''						) or name == "TFPreTrainedModel":
															return "tf"
										elif module.startswith('''torch'''						) or name == "PreTrainedModel":
															return "pt"
										elif module.startswith('''flax'''						) or module.startswith('''jax'''						) or name == "FlaxPreTrainedModel":
															return "flax"
					else:
										raise TypeError(f"""Could not infer framework from class {model_class}."""						)

import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCAmelCase       =		get_tests_dir("fixtures/spiece.model")
@require_sentencepiece
@require_tokenizers
class     lowerCAmelCase_     (       __lowercase,  unittest.TestCase ):
							UpperCAmelCase				   =     DebertaVaTokenizer
							UpperCAmelCase				   =     DebertaVaTokenizerFast
							UpperCAmelCase				   =     True
							UpperCAmelCase				   =     True







							def  UpperCamelCase_						(   self      :     List[Any]		):
												super().setUp()

												# We have a SentencePiece fixture for testing
												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			unk_token='''<unk>'''		)
												tokenizer.save_pretrained(self.tmpdirname		)







							def  UpperCamelCase_						(   self      :     Dict ,			_A      :     Union[str, Any]		):
												_UpperCamelCase			     =     '''this is a test'''
												_UpperCamelCase			     =     '''this is a test'''
												return input_text, output_text







							def  UpperCamelCase_						(   self      :     Optional[Any]		):
												_UpperCamelCase			     =     '''<pad>'''
												_UpperCamelCase			     =     0

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A		) ,			_A		)
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A		) ,			_A		)







							def  UpperCamelCase_						(   self      :     Any		):
												_UpperCamelCase			     =     list(self.get_tokenizer().get_vocab().keys()		)
												self.assertEqual(vocab_keys[0] ,			'''<pad>'''		)
												self.assertEqual(vocab_keys[1] ,			'''<unk>'''		)
												self.assertEqual(vocab_keys[-1] ,			'''[PAD]'''		)
												self.assertEqual(len(_A		) ,			3_0001		)







							def  UpperCamelCase_						(   self      :     List[Any]		):
												self.assertEqual(self.get_tokenizer().vocab_size ,			3_0000		)







							def  UpperCamelCase_						(   self      :     List[str]		):
												# fmt: off
												_UpperCamelCase			     =     ''' \tHeLLo!how  \n Are yoU?  '''
												_UpperCamelCase			     =     ['''▁hello''', '''!''', '''how''', '''▁are''', '''▁you''', '''?''']
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			do_lower_case=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			do_lower_case=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)







							@unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''		)
							def  UpperCamelCase_						(   self      :     Dict		):
												pass







							@unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''		)
							def  UpperCamelCase_						(   self      :     Optional[Any]		):
												pass







							def  UpperCamelCase_						(   self      :     Dict		):
												# fmt: off
												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''
												_UpperCamelCase			     =     ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     List[Any]		):
												# fmt: off
												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''
												_UpperCamelCase			     =     ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)
												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)
												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     Dict		):
												# fmt: off
												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''
												_UpperCamelCase			     =     ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     int		):
												# fmt: off
												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''
												_UpperCamelCase			     =     ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     Tuple		):
												# fmt: off
												_UpperCamelCase			     =     ''' \tHeLLo!how  \n Are yoU?  '''
												_UpperCamelCase			     =     ['''▁''', '''<unk>''', '''e''', '''<unk>''', '''o''', '''!''', '''how''', '''▁''', '''<unk>''', '''re''', '''▁yo''', '''<unk>''', '''?''']
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     List[str]		):
												_UpperCamelCase			     =     self.get_tokenizer()
												_UpperCamelCase			     =     self.get_rust_tokenizer()

												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''

												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)
												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     tokenizer.encode(_A ,			add_special_tokens=_A		)
												_UpperCamelCase			     =     rust_tokenizer.encode(_A ,			add_special_tokens=_A		)
												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     self.get_rust_tokenizer()
												_UpperCamelCase			     =     tokenizer.encode(_A		)
												_UpperCamelCase			     =     rust_tokenizer.encode(_A		)
												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     Dict		):
												_UpperCamelCase			     =     '''This is a test'''
												_UpperCamelCase			     =     [13, 1, 4398, 25, 21, 1289]
												_UpperCamelCase			     =     ['''▁''', '''T''', '''his''', '''▁is''', '''▁a''', '''▁test''']
												_UpperCamelCase			     =     ['''▁''', '''<unk>''', '''his''', '''▁is''', '''▁a''', '''▁test''']

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			keep_accents=_A		)
												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			keep_accents=_A		)

												_UpperCamelCase			     =     tokenizer.encode(_A ,			add_special_tokens=_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     tokenizer.tokenize(_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(_A		)
												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     rust_tokenizer.encode(_A ,			add_special_tokens=_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     rust_tokenizer.tokenize(_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(_A		)
												self.assertListEqual(_A ,			_A		)

												# fmt: off
												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''
												_UpperCamelCase			     =     [13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
												_UpperCamelCase			     =     ['''▁''', '''I''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''', ]
												_UpperCamelCase			     =     ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
												# fmt: on

												_UpperCamelCase			     =     tokenizer.encode(_A ,			add_special_tokens=_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     tokenizer.tokenize(_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(_A		)
												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     rust_tokenizer.encode(_A ,			add_special_tokens=_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     rust_tokenizer.tokenize(_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(_A		)
												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     Any		):
												_UpperCamelCase			     =     DebertaVaTokenizer(_A		)

												_UpperCamelCase			     =     tokenizer.encode('''sequence builders'''		)
												_UpperCamelCase			     =     tokenizer.encode('''multi-sequence build'''		)

												_UpperCamelCase			     =     tokenizer.build_inputs_with_special_tokens(_A		)
												_UpperCamelCase			     =     tokenizer.build_inputs_with_special_tokens(_A ,			_A		)

												self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] ,			_A		)
												self.assertEqual(
												    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id] ,			_A ,			)







							@slow
							def  UpperCamelCase_						(   self      :     Optional[Any]		):
												# fmt: off
												_UpperCamelCase			     =     {'''input_ids''': [[1, 3_9867, 36, 1_9390, 486, 27, 3_5052, 8_1436, 18, 6_0685, 1225, 7, 3_5052, 8_1436, 18, 9367, 1_6899, 18, 1_5937, 53, 594, 773, 18, 1_6287, 3_0465, 36, 1_5937, 6, 4_1139, 38, 3_6979, 6_0763, 191, 6, 3_4132, 99, 6, 5_0538, 390, 4_3230, 6, 3_4132, 2779, 2_0850, 14, 699, 1072, 1194, 36, 382, 1_0901, 53, 7, 699, 1072, 2084, 36, 2_0422, 630, 53, 19, 105, 3049, 1896, 1053, 1_6899, 1506, 11, 3_7978, 4243, 7, 1237, 3_1869, 200, 1_6566, 654, 6, 3_5052, 8_1436, 7, 5_5630, 1_3593, 4, 2], [1, 26, 1_5011, 13, 667, 8, 1053, 18, 2_3611, 1237, 7_2356, 1_2820, 34, 10_4134, 1209, 35, 1_3313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 1_5785, 1_4951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
												# fmt: on

												self.tokenizer_integration_test_util(
												    expected_encoding=_A ,			model_name='''microsoft/deberta-v2-xlarge''' ,			revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670''' ,			)
