"""simple docstring"""






import unittest

from transformers import (
    MODEL_FOR_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
    TextGenerationPipeline,
    logging,
    pipeline,
)
from transformers.testing_utils import (
    CaptureLogger,
    is_pipeline_test,
    require_accelerate,
    require_tf,
    require_torch,
    require_torch_gpu,
    require_torch_or_tf,
)

from .test_pipelines_common import ANY
@is_pipeline_test
@require_torch_or_tf
class 	UpperCAmelCase__					(     unittest.TestCase      ):


	"""simple docstring"""
	lowerCAmelCase__     :       Tuple       =				MODEL_FOR_CAUSAL_LM_MAPPING
	lowerCAmelCase__     :       Union[str, Any]       =				TF_MODEL_FOR_CAUSAL_LM_MAPPING







	@require_torch
	def    A     (      self			)     ->						Union[str, Any]:
							a_				:      Tuple  		=       pipeline(task="text-generation"							,  model="sshleifer/tiny-ctrl"							,  framework="pt"			)
							# Using `do_sample=False` to force deterministic output
							a_				:      Union[str, Any]  		=       text_generator("This is a test"							,  do_sample=_A			)
							self.assertEqual(
							    _A							,  [
							        {
							            "generated_text": (
							                "This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope."
							                " oscope. FiliFili@@"
							            )
							        }
							    ]							,  )

							a_				:      List[Any]  		=       text_generator(["This is a test", "This is a second test"]			)
							self.assertEqual(
							    _A							,  [
							        [
							            {
							                "generated_text": (
							                    "This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope."
							                    " oscope. FiliFili@@"
							                )
							            }
							        ],
							        [
							            {
							                "generated_text": (
							                    "This is a second test ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy"
							                    " oscope. oscope. FiliFili@@"
							                )
							            }
							        ],
							    ]							,  )

							a_				:      str  		=       text_generator("This is a test"							,  do_sample=_A							,  num_return_sequences=2							,  return_tensors=_A			)
							self.assertEqual(
							    _A							,  [
							        {"generated_token_ids": ANY(_A			)},
							        {"generated_token_ids": ANY(_A			)},
							    ]							,  )
							a_				:      Any  		=       text_generator.model.config.eos_token_id
							a_				:      str  		=       "<pad>"
							a_				:      str  		=       text_generator(
							    ["This is a test", "This is a second test"]							,  do_sample=_A							,  num_return_sequences=2							,  batch_size=2							,  return_tensors=_A							,  )
							self.assertEqual(
							    _A							,  [
							        [
							            {"generated_token_ids": ANY(_A			)},
							            {"generated_token_ids": ANY(_A			)},
							        ],
							        [
							            {"generated_token_ids": ANY(_A			)},
							            {"generated_token_ids": ANY(_A			)},
							        ],
							    ]							,  )







	@require_tf
	def    A     (      self			)     ->						Union[str, Any]:
							a_				:      str  		=       pipeline(task="text-generation"							,  model="sshleifer/tiny-ctrl"							,  framework="tf"			)

							# Using `do_sample=False` to force deterministic output
							a_				:      int  		=       text_generator("This is a test"							,  do_sample=_A			)
							self.assertEqual(
							    _A							,  [
							        {
							            "generated_text": (
							                "This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵"
							                " please,"
							            )
							        }
							    ]							,  )

							a_				:      str  		=       text_generator(["This is a test", "This is a second test"]							,  do_sample=_A			)
							self.assertEqual(
							    _A							,  [
							        [
							            {
							                "generated_text": (
							                    "This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵"
							                    " please,"
							                )
							            }
							        ],
							        [
							            {
							                "generated_text": (
							                    "This is a second test Chieftain Chieftain prefecture prefecture prefecture Cannes Cannes"
							                    " Cannes 閲閲Cannes Cannes Cannes 攵 please,"
							                )
							            }
							        ],
							    ]							,  )







	def    A     (      self							,  _SCREAMING_SNAKE_CASE							,  _SCREAMING_SNAKE_CASE							,  _SCREAMING_SNAKE_CASE			)     ->						Union[str, Any]:
							a_				:      Any  		=       TextGenerationPipeline(model=_A							,  tokenizer=_A			)
							return text_generator, ["This is a test", "Another test"]







	def    A     (      self			)     ->						List[str]:
							a_				:      str  		=       "Hello I believe in"
							a_				:      List[str]  		=       pipeline("text-generation"							,  model="hf-internal-testing/tiny-random-gpt2"			)
							a_				:      Optional[Any]  		=       text_generator(_A			)
							self.assertEqual(
							    _A							,  [{"generated_text": "Hello I believe in fe fe fe fe fe fe fe fe fe fe fe fe"}]							,  )

							a_				:      Optional[Any]  		=       text_generator(_A							,  stop_sequence=" fe"			)
							self.assertEqual(_A							,  [{"generated_text": "Hello I believe in fe"}]			)







	def    A     (      self							,  _SCREAMING_SNAKE_CASE							,  _SCREAMING_SNAKE_CASE			)     ->						Dict:
							a_				:      int  		=       text_generator.model
							a_				:      Optional[int]  		=       text_generator.tokenizer

							a_				:      str  		=       text_generator("This is a test"			)
							self.assertEqual(_A							,  [{"generated_text": ANY(_A			)}]			)
							self.assertTrue(outputs[0]["generated_text"].startswith("This is a test"			)			)

							a_				:      Tuple  		=       text_generator("This is a test"							,  return_full_text=_A			)
							self.assertEqual(_A							,  [{"generated_text": ANY(_A			)}]			)
							self.assertNotIn("This is a test"							,  outputs[0]["generated_text"]			)

							a_				:      int  		=       pipeline(task="text-generation"							,  model=_A							,  tokenizer=_A							,  return_full_text=_A			)
							a_				:      Union[str, Any]  		=       text_generator("This is a test"			)
							self.assertEqual(_A							,  [{"generated_text": ANY(_A			)}]			)
							self.assertNotIn("This is a test"							,  outputs[0]["generated_text"]			)

							a_				:      Any  		=       text_generator("This is a test"							,  return_full_text=_A			)
							self.assertEqual(_A							,  [{"generated_text": ANY(_A			)}]			)
							self.assertTrue(outputs[0]["generated_text"].startswith("This is a test"			)			)

							a_				:      int  		=       text_generator(["This is great !", "Something else"]							,  num_return_sequences=2							,  do_sample=_A			)
							self.assertEqual(
							    _A							,  [
							        [{"generated_text": ANY(_A			)}, {"generated_text": ANY(_A			)}],
							        [{"generated_text": ANY(_A			)}, {"generated_text": ANY(_A			)}],
							    ]							,  )

							if text_generator.tokenizer.pad_token is not None:
													a_				:      Optional[Any]  		=       text_generator(
													    ["This is great !", "Something else"]							,  num_return_sequences=2							,  batch_size=2							,  do_sample=_A			)
													self.assertEqual(
													    _A							,  [
													        [{"generated_text": ANY(_A			)}, {"generated_text": ANY(_A			)}],
													        [{"generated_text": ANY(_A			)}, {"generated_text": ANY(_A			)}],
													    ]							,  )

							with self.assertRaises(_A			):
													a_				:      List[str]  		=       text_generator("test"							,  return_full_text=_A							,  return_text=_A			)
							with self.assertRaises(_A			):
													a_				:      Optional[int]  		=       text_generator("test"							,  return_full_text=_A							,  return_tensors=_A			)
							with self.assertRaises(_A			):
													a_				:      str  		=       text_generator("test"							,  return_text=_A							,  return_tensors=_A			)

							# Empty prompt is slighly special
							# it requires BOS token to exist.
							# Special case for Pegasus which will always append EOS so will
							# work even without BOS.
							if (
							    text_generator.tokenizer.bos_token_id is not None
							    or "Pegasus" in tokenizer.__class__.__name__
							    or "Git" in model.__class__.__name__
							):
													a_				:      str  		=       text_generator(""			)
													self.assertEqual(_A							,  [{"generated_text": ANY(_A			)}]			)
							else:
													with self.assertRaises((ValueError, AssertionError)			):
																			a_				:      Any  		=       text_generator(""			)

							if text_generator.framework == "tf":
													# TF generation does not support max_new_tokens, and it's impossible
													# to control long generation with only max_length without
													# fancy calculation, dismissing tests for now.
													return
							# We don't care about infinite range models.
							# They already work.
							# Skip this test for XGLM, since it uses sinusoidal positional embeddings which are resized on-the-fly.
							a_				:      List[str]  		=       ["RwkvForCausalLM", "XGLMForCausalLM", "GPTNeoXForCausalLM"]
							if (
							    tokenizer.model_max_length < 1_0_0_0_0
							    and text_generator.model.__class__.__name__ not in EXTRA_MODELS_CAN_HANDLE_LONG_INPUTS
							):
													# Handling of large generations
													with self.assertRaises((RuntimeError, IndexError, ValueError, AssertionError)			):
																			text_generator("This is a test" * 5_0_0							,  max_new_tokens=2_0			)

													a_				:      Tuple  		=       text_generator("This is a test" * 5_0_0							,  handle_long_generation="hole"							,  max_new_tokens=2_0			)
													# Hole strategy cannot work
													with self.assertRaises(_A			):
																			text_generator(
																			    "This is a test" * 5_0_0							,  handle_long_generation="hole"							,  max_new_tokens=tokenizer.model_max_length + 1_0							,  )







	@require_torch
	@require_accelerate
	@require_torch_gpu
	def    A     (      self			)     ->						int:
							import torch

							# Classic `model_kwargs`
							a_				:      str  		=       pipeline(
							    model="hf-internal-testing/tiny-random-bloom"							,  model_kwargs={"device_map": "auto", "torch_dtype": torch.bfloataa}							,  )
							self.assertEqual(pipe.model.device							,  torch.device(0			)			)
							self.assertEqual(pipe.model.lm_head.weight.dtype							,  torch.bfloataa			)
							a_				:      List[Any]  		=       pipe("This is a test"			)
							self.assertEqual(
							    _A							,  [
							        {
							            "generated_text": (
							                "This is a test test test test test test test test test test test test test test test test"
							                " test"
							            )
							        }
							    ]							,  )

							# Upgraded those two to real pipeline arguments (they just get sent for the model as they're unlikely to mean anything else.)
							a_				:      List[Any]  		=       pipeline(model="hf-internal-testing/tiny-random-bloom"							,  device_map="auto"							,  torch_dtype=torch.bfloataa			)
							self.assertEqual(pipe.model.device							,  torch.device(0			)			)
							self.assertEqual(pipe.model.lm_head.weight.dtype							,  torch.bfloataa			)
							a_				:      Optional[int]  		=       pipe("This is a test"			)
							self.assertEqual(
							    _A							,  [
							        {
							            "generated_text": (
							                "This is a test test test test test test test test test test test test test test test test"
							                " test"
							            )
							        }
							    ]							,  )

							# torch_dtype will be automatically set to float32 if not provided - check: https://github.com/huggingface/transformers/pull/20602
							a_				:      int  		=       pipeline(model="hf-internal-testing/tiny-random-bloom"							,  device_map="auto"			)
							self.assertEqual(pipe.model.device							,  torch.device(0			)			)
							self.assertEqual(pipe.model.lm_head.weight.dtype							,  torch.floataa			)
							a_				:      Union[str, Any]  		=       pipe("This is a test"			)
							self.assertEqual(
							    _A							,  [
							        {
							            "generated_text": (
							                "This is a test test test test test test test test test test test test test test test test"
							                " test"
							            )
							        }
							    ]							,  )







	@require_torch
	@require_torch_gpu
	def    A     (      self			)     ->						Optional[int]:
							import torch

							a_				:      List[str]  		=       pipeline(model="hf-internal-testing/tiny-random-bloom"							,  device=0							,  torch_dtype=torch.floataa			)
							pipe("This is a test"			)







	@require_torch
	@require_accelerate
	@require_torch_gpu
	def    A     (      self			)     ->						str:
							import torch

							a_				:      Any  		=       pipeline(model="hf-internal-testing/tiny-random-bloom"							,  device_map="auto"							,  torch_dtype=torch.floataa			)
							pipe("This is a test"							,  do_sample=_A							,  top_p=0.5			)







	def    A     (      self			)     ->						Dict:
							a_				:      Optional[int]  		=       "Hello world"
							a_				:      Dict  		=       pipeline("text-generation"							,  model="hf-internal-testing/tiny-random-gpt2"			)
							if text_generator.model.framework == "tf":
													a_				:      Any  		=       logging.get_logger("transformers.generation.tf_utils"			)
							else:
													a_				:      Optional[int]  		=       logging.get_logger("transformers.generation.utils"			)
							a_				:      Tuple  		=       "Both `max_new_tokens`"  # The beggining of the message to be checked in this test

							# Both are set by the user -> log warning
							with CaptureLogger(_A			) as cl:
													a_				:      str  		=       text_generator(_A							,  max_length=1_0							,  max_new_tokens=1			)
							self.assertIn(_A							,  cl.out			)

							# The user only sets one -> no warning
							with CaptureLogger(_A			) as cl:
													a_				:      Optional[Any]  		=       text_generator(_A							,  max_new_tokens=1			)
							self.assertNotIn(_A							,  cl.out			)

							with CaptureLogger(_A			) as cl:
													a_				:      List[Any]  		=       text_generator(_A							,  max_length=1_0			)
							self.assertNotIn(_A							,  cl.out			)





import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCAmelCase       =		get_tests_dir("fixtures/spiece.model")
@require_sentencepiece
@require_tokenizers
class     lowerCAmelCase_     (       __lowercase,  unittest.TestCase ):
							UpperCAmelCase				   =     DebertaVaTokenizer
							UpperCAmelCase				   =     DebertaVaTokenizerFast
							UpperCAmelCase				   =     True
							UpperCAmelCase				   =     True







							def  UpperCamelCase_						(   self      :     List[Any]		):
												super().setUp()

												# We have a SentencePiece fixture for testing
												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			unk_token='''<unk>'''		)
												tokenizer.save_pretrained(self.tmpdirname		)







							def  UpperCamelCase_						(   self      :     Dict ,			_A      :     Union[str, Any]		):
												_UpperCamelCase			     =     '''this is a test'''
												_UpperCamelCase			     =     '''this is a test'''
												return input_text, output_text







							def  UpperCamelCase_						(   self      :     Optional[Any]		):
												_UpperCamelCase			     =     '''<pad>'''
												_UpperCamelCase			     =     0

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A		) ,			_A		)
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A		) ,			_A		)







							def  UpperCamelCase_						(   self      :     Any		):
												_UpperCamelCase			     =     list(self.get_tokenizer().get_vocab().keys()		)
												self.assertEqual(vocab_keys[0] ,			'''<pad>'''		)
												self.assertEqual(vocab_keys[1] ,			'''<unk>'''		)
												self.assertEqual(vocab_keys[-1] ,			'''[PAD]'''		)
												self.assertEqual(len(_A		) ,			3_0001		)







							def  UpperCamelCase_						(   self      :     List[Any]		):
												self.assertEqual(self.get_tokenizer().vocab_size ,			3_0000		)







							def  UpperCamelCase_						(   self      :     List[str]		):
												# fmt: off
												_UpperCamelCase			     =     ''' \tHeLLo!how  \n Are yoU?  '''
												_UpperCamelCase			     =     ['''▁hello''', '''!''', '''how''', '''▁are''', '''▁you''', '''?''']
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			do_lower_case=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			do_lower_case=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)







							@unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''		)
							def  UpperCamelCase_						(   self      :     Dict		):
												pass







							@unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''		)
							def  UpperCamelCase_						(   self      :     Optional[Any]		):
												pass







							def  UpperCamelCase_						(   self      :     Dict		):
												# fmt: off
												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''
												_UpperCamelCase			     =     ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     List[Any]		):
												# fmt: off
												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''
												_UpperCamelCase			     =     ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)
												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)
												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     Dict		):
												# fmt: off
												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''
												_UpperCamelCase			     =     ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     int		):
												# fmt: off
												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''
												_UpperCamelCase			     =     ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     Tuple		):
												# fmt: off
												_UpperCamelCase			     =     ''' \tHeLLo!how  \n Are yoU?  '''
												_UpperCamelCase			     =     ['''▁''', '''<unk>''', '''e''', '''<unk>''', '''o''', '''!''', '''how''', '''▁''', '''<unk>''', '''re''', '''▁yo''', '''<unk>''', '''?''']
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     List[str]		):
												_UpperCamelCase			     =     self.get_tokenizer()
												_UpperCamelCase			     =     self.get_rust_tokenizer()

												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''

												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)
												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     tokenizer.encode(_A ,			add_special_tokens=_A		)
												_UpperCamelCase			     =     rust_tokenizer.encode(_A ,			add_special_tokens=_A		)
												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     self.get_rust_tokenizer()
												_UpperCamelCase			     =     tokenizer.encode(_A		)
												_UpperCamelCase			     =     rust_tokenizer.encode(_A		)
												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     Dict		):
												_UpperCamelCase			     =     '''This is a test'''
												_UpperCamelCase			     =     [13, 1, 4398, 25, 21, 1289]
												_UpperCamelCase			     =     ['''▁''', '''T''', '''his''', '''▁is''', '''▁a''', '''▁test''']
												_UpperCamelCase			     =     ['''▁''', '''<unk>''', '''his''', '''▁is''', '''▁a''', '''▁test''']

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			keep_accents=_A		)
												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			keep_accents=_A		)

												_UpperCamelCase			     =     tokenizer.encode(_A ,			add_special_tokens=_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     tokenizer.tokenize(_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(_A		)
												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     rust_tokenizer.encode(_A ,			add_special_tokens=_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     rust_tokenizer.tokenize(_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(_A		)
												self.assertListEqual(_A ,			_A		)

												# fmt: off
												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''
												_UpperCamelCase			     =     [13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
												_UpperCamelCase			     =     ['''▁''', '''I''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''', ]
												_UpperCamelCase			     =     ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
												# fmt: on

												_UpperCamelCase			     =     tokenizer.encode(_A ,			add_special_tokens=_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     tokenizer.tokenize(_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(_A		)
												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     rust_tokenizer.encode(_A ,			add_special_tokens=_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     rust_tokenizer.tokenize(_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(_A		)
												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     Any		):
												_UpperCamelCase			     =     DebertaVaTokenizer(_A		)

												_UpperCamelCase			     =     tokenizer.encode('''sequence builders'''		)
												_UpperCamelCase			     =     tokenizer.encode('''multi-sequence build'''		)

												_UpperCamelCase			     =     tokenizer.build_inputs_with_special_tokens(_A		)
												_UpperCamelCase			     =     tokenizer.build_inputs_with_special_tokens(_A ,			_A		)

												self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] ,			_A		)
												self.assertEqual(
												    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id] ,			_A ,			)







							@slow
							def  UpperCamelCase_						(   self      :     Optional[Any]		):
												# fmt: off
												_UpperCamelCase			     =     {'''input_ids''': [[1, 3_9867, 36, 1_9390, 486, 27, 3_5052, 8_1436, 18, 6_0685, 1225, 7, 3_5052, 8_1436, 18, 9367, 1_6899, 18, 1_5937, 53, 594, 773, 18, 1_6287, 3_0465, 36, 1_5937, 6, 4_1139, 38, 3_6979, 6_0763, 191, 6, 3_4132, 99, 6, 5_0538, 390, 4_3230, 6, 3_4132, 2779, 2_0850, 14, 699, 1072, 1194, 36, 382, 1_0901, 53, 7, 699, 1072, 2084, 36, 2_0422, 630, 53, 19, 105, 3049, 1896, 1053, 1_6899, 1506, 11, 3_7978, 4243, 7, 1237, 3_1869, 200, 1_6566, 654, 6, 3_5052, 8_1436, 7, 5_5630, 1_3593, 4, 2], [1, 26, 1_5011, 13, 667, 8, 1053, 18, 2_3611, 1237, 7_2356, 1_2820, 34, 10_4134, 1209, 35, 1_3313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 1_5785, 1_4951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
												# fmt: on

												self.tokenizer_integration_test_util(
												    expected_encoding=_A ,			model_name='''microsoft/deberta-v2-xlarge''' ,			revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670''' ,			)
