




'''simple docstring'''


import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase_					=   get_tests_dir("fixtures/spiece.model")





@require_sentencepiece
@require_tokenizers
class  __lowercase    (	__magic_name__							,				unittest.TestCase			):
   _a    						=   DebertaVaTokenizer
   _a    						=   DebertaVaTokenizerFast
   _a    						=   True
   _a    						=   True







   def 		UpperCamelCase__      (						self   )					->     Optional[int]:
     super().setUp()

     # We have a SentencePiece fixture for testing
     __a								=       DebertaVaTokenizer(UpperCamelCase   ,				unk_token='<unk>'   )
     tokenizer.save_pretrained(self.tmpdirname   )







   def 		UpperCamelCase__      (						self   ,				UpperCamelCase   )					->     Union[str, Any]:
     __a								=       'this is a test'
     __a								=       'this is a test'
     return input_text, output_text







   def 		UpperCamelCase__      (						self   )					->     Any:
     __a								=       '<pad>'
     __a								=       0

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase   )   ,				UpperCamelCase   )
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase   )   ,				UpperCamelCase   )







   def 		UpperCamelCase__      (						self   )					->     Any:
     __a								=       list(self.get_tokenizer().get_vocab().keys()   )
     self.assertEqual(vocab_keys[0]   ,				'<pad>'   )
     self.assertEqual(vocab_keys[1]   ,				'<unk>'   )
     self.assertEqual(vocab_keys[-1]   ,				'[PAD]'   )
     self.assertEqual(len(UpperCamelCase   )   ,				3_0001   )







   def 		UpperCamelCase__      (						self   )					->     Union[str, Any]:
     self.assertEqual(self.get_tokenizer().vocab_size   ,				3_0000   )







   def 		UpperCamelCase__      (						self   )					->     Tuple:
     # fmt: off
     __a								=       ' \tHeLLo!how  \n Are yoU?  '
     __a								=       ['▁hello', '!', 'how', '▁are', '▁you', '?']
     # fmt: on

     __a								=       DebertaVaTokenizer(UpperCamelCase   ,				do_lower_case=UpperCamelCase   )
     __a								=       tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )   )

     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )

     __a								=       DebertaVaTokenizerFast(UpperCamelCase   ,				do_lower_case=UpperCamelCase   )
     __a								=       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )   )

     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )







   @unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'   )
   def 		UpperCamelCase__      (						self   )					->     List[Any]:
     pass







   @unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'   )
   def 		UpperCamelCase__      (						self   )					->     List[Any]:
     pass







   def 		UpperCamelCase__      (						self   )					->     List[Any]:
     # fmt: off
     __a								=       'I was born in 92000, and this is falsé.'
     __a								=       ['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
     # fmt: on

     __a								=       DebertaVaTokenizer(UpperCamelCase   ,				split_by_punct=UpperCamelCase   )
     __a								=       tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )   )

     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )

     __a								=       DebertaVaTokenizerFast(UpperCamelCase   ,				split_by_punct=UpperCamelCase   )
     __a								=       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )   )

     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )







   def 		UpperCamelCase__      (						self   )					->     Any:
     # fmt: off
     __a								=       'I was born in 92000, and this is falsé.'
     __a								=       ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
     # fmt: on

     __a								=       DebertaVaTokenizer(UpperCamelCase   ,				do_lower_case=UpperCamelCase   ,				split_by_punct=UpperCamelCase   )
     __a								=       tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )

     __a								=       DebertaVaTokenizerFast(UpperCamelCase   ,				do_lower_case=UpperCamelCase   ,				split_by_punct=UpperCamelCase   )
     __a								=       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )







   def 		UpperCamelCase__      (						self   )					->     str:
     # fmt: off
     __a								=       'I was born in 92000, and this is falsé.'
     __a								=       ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
     # fmt: on

     __a								=       DebertaVaTokenizer(UpperCamelCase   ,				do_lower_case=UpperCamelCase   ,				split_by_punct=UpperCamelCase   )
     __a								=       tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )   )

     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )

     __a								=       DebertaVaTokenizerFast(UpperCamelCase   ,				do_lower_case=UpperCamelCase   ,				split_by_punct=UpperCamelCase   )
     __a								=       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )   )

     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )







   def 		UpperCamelCase__      (						self   )					->     Optional[Any]:
     # fmt: off
     __a								=       'I was born in 92000, and this is falsé.'
     __a								=       ['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
     # fmt: on

     __a								=       DebertaVaTokenizer(UpperCamelCase   ,				do_lower_case=UpperCamelCase   ,				split_by_punct=UpperCamelCase   )
     __a								=       tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )   )

     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )

     __a								=       DebertaVaTokenizerFast(UpperCamelCase   ,				do_lower_case=UpperCamelCase   ,				split_by_punct=UpperCamelCase   )
     __a								=       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )   )

     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )







   def 		UpperCamelCase__      (						self   )					->     Optional[int]:
     # fmt: off
     __a								=       ' \tHeLLo!how  \n Are yoU?  '
     __a								=       ['▁', '<unk>', 'e', '<unk>', 'o', '!', 'how', '▁', '<unk>', 're', '▁yo', '<unk>', '?']
     # fmt: on

     __a								=       DebertaVaTokenizer(UpperCamelCase   ,				do_lower_case=UpperCamelCase   ,				split_by_punct=UpperCamelCase   )
     __a								=       tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )   )

     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )

     __a								=       DebertaVaTokenizerFast(UpperCamelCase   ,				do_lower_case=UpperCamelCase   ,				split_by_punct=UpperCamelCase   )
     __a								=       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )   )

     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )







   def 		UpperCamelCase__      (						self   )					->     Dict:
     __a								=       self.get_tokenizer()
     __a								=       self.get_rust_tokenizer()

     __a								=       'I was born in 92000, and this is falsé.'

     __a								=       tokenizer.convert_ids_to_tokens(tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )   )
     __a								=       rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )

     __a								=       tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )
     __a								=       rust_tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )

     __a								=       self.get_rust_tokenizer()
     __a								=       tokenizer.encode(UpperCamelCase   )
     __a								=       rust_tokenizer.encode(UpperCamelCase   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )







   def 		UpperCamelCase__      (						self   )					->     Union[str, Any]:
     __a								=       'This is a test'
     __a								=       [13, 1, 4398, 25, 21, 1289]
     __a								=       ['▁', 'T', 'his', '▁is', '▁a', '▁test']
     __a								=       ['▁', '<unk>', 'his', '▁is', '▁a', '▁test']

     __a								=       DebertaVaTokenizer(UpperCamelCase   ,				keep_accents=UpperCamelCase   )
     __a								=       DebertaVaTokenizerFast(UpperCamelCase   ,				keep_accents=UpperCamelCase   )

     __a								=       tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )
     __a								=       tokenizer.tokenize(UpperCamelCase   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )
     __a								=       tokenizer.convert_ids_to_tokens(UpperCamelCase   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )

     __a								=       rust_tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )
     __a								=       rust_tokenizer.tokenize(UpperCamelCase   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )
     __a								=       rust_tokenizer.convert_ids_to_tokens(UpperCamelCase   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )

     # fmt: off
     __a								=       'I was born in 92000, and this is falsé.'
     __a								=       [13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
     __a								=       ['▁', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.', ]
     __a								=       ['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
     # fmt: on

     __a								=       tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )
     __a								=       tokenizer.tokenize(UpperCamelCase   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )
     __a								=       tokenizer.convert_ids_to_tokens(UpperCamelCase   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )

     __a								=       rust_tokenizer.encode(UpperCamelCase   ,				add_special_tokens=UpperCamelCase   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )
     __a								=       rust_tokenizer.tokenize(UpperCamelCase   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )
     __a								=       rust_tokenizer.convert_ids_to_tokens(UpperCamelCase   )
     self.assertListEqual(UpperCamelCase   ,				UpperCamelCase   )







   def 		UpperCamelCase__      (						self   )					->     int:
     __a								=       DebertaVaTokenizer(UpperCamelCase   )

     __a								=       tokenizer.encode('sequence builders'   )
     __a								=       tokenizer.encode('multi-sequence build'   )

     __a								=       tokenizer.build_inputs_with_special_tokens(UpperCamelCase   )
     __a								=       tokenizer.build_inputs_with_special_tokens(UpperCamelCase   ,				UpperCamelCase   )

     self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]   ,				UpperCamelCase   )
     self.assertEqual(
         [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]   ,				UpperCamelCase   ,				)







   @slow
   def 		UpperCamelCase__      (						self   )					->     Optional[int]:
     # fmt: off
     __a								=       {'input_ids': [[1, 3_9867, 36, 1_9390, 486, 27, 3_5052, 8_1436, 18, 6_0685, 1225, 7, 3_5052, 8_1436, 18, 9367, 1_6899, 18, 1_5937, 53, 594, 773, 18, 1_6287, 3_0465, 36, 1_5937, 6, 4_1139, 38, 3_6979, 6_0763, 191, 6, 3_4132, 99, 6, 5_0538, 390, 4_3230, 6, 3_4132, 2779, 2_0850, 14, 699, 1072, 1194, 36, 382, 1_0901, 53, 7, 699, 1072, 2084, 36, 2_0422, 630, 53, 19, 105, 3049, 1896, 1053, 1_6899, 1506, 11, 3_7978, 4243, 7, 1237, 3_1869, 200, 1_6566, 654, 6, 3_5052, 8_1436, 7, 5_5630, 1_3593, 4, 2], [1, 26, 1_5011, 13, 667, 8, 1053, 18, 2_3611, 1237, 7_2356, 1_2820, 34, 10_4134, 1209, 35, 1_3313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 1_5785, 1_4951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=UpperCamelCase   ,				model_name='microsoft/deberta-v2-xlarge'   ,				revision='ad6e42c1532ddf3a15c39246b63f5559d558b670'   ,				)









'''simple docstring'''


from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


UpperCAmelCase_					=   {
    "configuration_trajectory_transformer": [
        "TRAJECTORY_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "TrajectoryTransformerConfig",
    ],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase_					=   [
      "TRAJECTORY_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
      "TrajectoryTransformerModel",
      "TrajectoryTransformerPreTrainedModel",
      "load_tf_weights_in_trajectory_transformer",
  ]


if TYPE_CHECKING:
  from .configuration_trajectory_transformer import (
      TRAJECTORY_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
      TrajectoryTransformerConfig,
  )

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_trajectory_transformer import (
        TRAJECTORY_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
        TrajectoryTransformerModel,
        TrajectoryTransformerPreTrainedModel,
        load_tf_weights_in_trajectory_transformer,
    )


else:
  import sys

  UpperCAmelCase_					=   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



