




"""simple docstring"""


import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase__  		=						get_tests_dir("""fixtures/spiece.model""")


@require_sentencepiece
@require_tokenizers
class        a				(		_UpperCAmelCase	,							unittest.TestCase   ):
    _snake_case     :  List[Any]     							=							DebertaVaTokenizer
    _snake_case     :  str     							=							DebertaVaTokenizerFast
    _snake_case     :  Dict     							=							True
    _snake_case     :  List[Any]     							=							True


    def      lowerCAmelCase_  (					self	:  Tuple					):
      super().setUp()

      # We have a SentencePiece fixture for testing
      _UpperCAmelCase						 =      DebertaVaTokenizer(A_   ,				unk_token="""<unk>"""					)
      tokenizer.save_pretrained(self.tmpdirname					)


    def      lowerCAmelCase_  (					self	:  Optional[Any]   ,				__lowerCAmelCase	:  str					):
      _UpperCAmelCase						 =      """this is a test"""
      _UpperCAmelCase						 =      """this is a test"""
      return input_text, output_text


    def      lowerCAmelCase_  (					self	:  Dict					):
      _UpperCAmelCase						 =      """<pad>"""
      _UpperCAmelCase						 =      0

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(A_					)   ,				A_					)
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(A_					)   ,				A_					)


    def      lowerCAmelCase_  (					self	:  Dict					):
      _UpperCAmelCase						 =      list(self.get_tokenizer().get_vocab().keys()					)
      self.assertEqual(vocab_keys[0]   ,				"""<pad>"""					)
      self.assertEqual(vocab_keys[1]   ,				"""<unk>"""					)
      self.assertEqual(vocab_keys[-1]   ,				"""[PAD]"""					)
      self.assertEqual(len(A_					)   ,				3_0001					)


    def      lowerCAmelCase_  (					self	:  str					):
      self.assertEqual(self.get_tokenizer().vocab_size   ,				3_0000					)


    def      lowerCAmelCase_  (					self	:  List[str]					):
      # fmt: off
      _UpperCAmelCase						 =      """ \tHeLLo!how  \n Are yoU?  """
      _UpperCAmelCase						 =      ["""▁hello""", """!""", """how""", """▁are""", """▁you""", """?"""]
      # fmt: on

      _UpperCAmelCase						 =      DebertaVaTokenizer(A_   ,				do_lower_case=A_					)
      _UpperCAmelCase						 =      tokenizer.convert_ids_to_tokens(tokenizer.encode(A_   ,				add_special_tokens=A_					)					)

      self.assertListEqual(A_   ,				A_					)

      _UpperCAmelCase						 =      DebertaVaTokenizerFast(A_   ,				do_lower_case=A_					)
      _UpperCAmelCase						 =      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_   ,				add_special_tokens=A_					)					)

      self.assertListEqual(A_   ,				A_					)


    @unittest.skip("""There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."""					)
    def      lowerCAmelCase_  (					self	:  str					):
      pass


    @unittest.skip("""There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."""					)
    def      lowerCAmelCase_  (					self	:  Optional[int]					):
      pass


    def      lowerCAmelCase_  (					self	:  List[str]					):
      # fmt: off
      _UpperCAmelCase						 =      """I was born in 92000, and this is falsé."""
      _UpperCAmelCase						 =      ["""▁""", """<unk>""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """▁""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """▁""", """.""", ]
      # fmt: on

      _UpperCAmelCase						 =      DebertaVaTokenizer(A_   ,				split_by_punct=A_					)
      _UpperCAmelCase						 =      tokenizer.convert_ids_to_tokens(tokenizer.encode(A_   ,				add_special_tokens=A_					)					)

      self.assertListEqual(A_   ,				A_					)

      _UpperCAmelCase						 =      DebertaVaTokenizerFast(A_   ,				split_by_punct=A_					)
      _UpperCAmelCase						 =      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_   ,				add_special_tokens=A_					)					)

      self.assertListEqual(A_   ,				A_					)


    def      lowerCAmelCase_  (					self	:  Optional[int]					):
      # fmt: off
      _UpperCAmelCase						 =      """I was born in 92000, and this is falsé."""
      _UpperCAmelCase						 =      ["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """▁""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """▁""", """.""", ]
      # fmt: on

      _UpperCAmelCase						 =      DebertaVaTokenizer(A_   ,				do_lower_case=A_   ,				split_by_punct=A_					)
      _UpperCAmelCase						 =      tokenizer.convert_ids_to_tokens(tokenizer.encode(A_   ,				add_special_tokens=A_					)					)
      self.assertListEqual(A_   ,				A_					)

      _UpperCAmelCase						 =      DebertaVaTokenizerFast(A_   ,				do_lower_case=A_   ,				split_by_punct=A_					)
      _UpperCAmelCase						 =      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_   ,				add_special_tokens=A_					)					)
      self.assertListEqual(A_   ,				A_					)


    def      lowerCAmelCase_  (					self	:  Tuple					):
      # fmt: off
      _UpperCAmelCase						 =      """I was born in 92000, and this is falsé."""
      _UpperCAmelCase						 =      ["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """.""", ]
      # fmt: on

      _UpperCAmelCase						 =      DebertaVaTokenizer(A_   ,				do_lower_case=A_   ,				split_by_punct=A_					)
      _UpperCAmelCase						 =      tokenizer.convert_ids_to_tokens(tokenizer.encode(A_   ,				add_special_tokens=A_					)					)

      self.assertListEqual(A_   ,				A_					)

      _UpperCAmelCase						 =      DebertaVaTokenizerFast(A_   ,				do_lower_case=A_   ,				split_by_punct=A_					)
      _UpperCAmelCase						 =      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_   ,				add_special_tokens=A_					)					)

      self.assertListEqual(A_   ,				A_					)


    def      lowerCAmelCase_  (					self	:  Optional[int]					):
      # fmt: off
      _UpperCAmelCase						 =      """I was born in 92000, and this is falsé."""
      _UpperCAmelCase						 =      ["""▁""", """<unk>""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """▁""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """▁""", """.""", ]
      # fmt: on

      _UpperCAmelCase						 =      DebertaVaTokenizer(A_   ,				do_lower_case=A_   ,				split_by_punct=A_					)
      _UpperCAmelCase						 =      tokenizer.convert_ids_to_tokens(tokenizer.encode(A_   ,				add_special_tokens=A_					)					)

      self.assertListEqual(A_   ,				A_					)

      _UpperCAmelCase						 =      DebertaVaTokenizerFast(A_   ,				do_lower_case=A_   ,				split_by_punct=A_					)
      _UpperCAmelCase						 =      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_   ,				add_special_tokens=A_					)					)

      self.assertListEqual(A_   ,				A_					)


    def      lowerCAmelCase_  (					self	:  Union[str, Any]					):
      # fmt: off
      _UpperCAmelCase						 =      """ \tHeLLo!how  \n Are yoU?  """
      _UpperCAmelCase						 =      ["""▁""", """<unk>""", """e""", """<unk>""", """o""", """!""", """how""", """▁""", """<unk>""", """re""", """▁yo""", """<unk>""", """?"""]
      # fmt: on

      _UpperCAmelCase						 =      DebertaVaTokenizer(A_   ,				do_lower_case=A_   ,				split_by_punct=A_					)
      _UpperCAmelCase						 =      tokenizer.convert_ids_to_tokens(tokenizer.encode(A_   ,				add_special_tokens=A_					)					)

      self.assertListEqual(A_   ,				A_					)

      _UpperCAmelCase						 =      DebertaVaTokenizerFast(A_   ,				do_lower_case=A_   ,				split_by_punct=A_					)
      _UpperCAmelCase						 =      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_   ,				add_special_tokens=A_					)					)

      self.assertListEqual(A_   ,				A_					)


    def      lowerCAmelCase_  (					self	:  List[str]					):
      _UpperCAmelCase						 =      self.get_tokenizer()
      _UpperCAmelCase						 =      self.get_rust_tokenizer()

      _UpperCAmelCase						 =      """I was born in 92000, and this is falsé."""

      _UpperCAmelCase						 =      tokenizer.convert_ids_to_tokens(tokenizer.encode(A_   ,				add_special_tokens=A_					)					)
      _UpperCAmelCase						 =      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_   ,				add_special_tokens=A_					)					)
      self.assertListEqual(A_   ,				A_					)

      _UpperCAmelCase						 =      tokenizer.encode(A_   ,				add_special_tokens=A_					)
      _UpperCAmelCase						 =      rust_tokenizer.encode(A_   ,				add_special_tokens=A_					)
      self.assertListEqual(A_   ,				A_					)

      _UpperCAmelCase						 =      self.get_rust_tokenizer()
      _UpperCAmelCase						 =      tokenizer.encode(A_					)
      _UpperCAmelCase						 =      rust_tokenizer.encode(A_					)
      self.assertListEqual(A_   ,				A_					)


    def      lowerCAmelCase_  (					self	:  Optional[int]					):
      _UpperCAmelCase						 =      """This is a test"""
      _UpperCAmelCase						 =      [13, 1, 4398, 25, 21, 1289]
      _UpperCAmelCase						 =      ["""▁""", """T""", """his""", """▁is""", """▁a""", """▁test"""]
      _UpperCAmelCase						 =      ["""▁""", """<unk>""", """his""", """▁is""", """▁a""", """▁test"""]

      _UpperCAmelCase						 =      DebertaVaTokenizer(A_   ,				keep_accents=A_					)
      _UpperCAmelCase						 =      DebertaVaTokenizerFast(A_   ,				keep_accents=A_					)

      _UpperCAmelCase						 =      tokenizer.encode(A_   ,				add_special_tokens=A_					)
      self.assertListEqual(A_   ,				A_					)
      _UpperCAmelCase						 =      tokenizer.tokenize(A_					)
      self.assertListEqual(A_   ,				A_					)
      _UpperCAmelCase						 =      tokenizer.convert_ids_to_tokens(A_					)
      self.assertListEqual(A_   ,				A_					)

      _UpperCAmelCase						 =      rust_tokenizer.encode(A_   ,				add_special_tokens=A_					)
      self.assertListEqual(A_   ,				A_					)
      _UpperCAmelCase						 =      rust_tokenizer.tokenize(A_					)
      self.assertListEqual(A_   ,				A_					)
      _UpperCAmelCase						 =      rust_tokenizer.convert_ids_to_tokens(A_					)
      self.assertListEqual(A_   ,				A_					)

      # fmt: off
      _UpperCAmelCase						 =      """I was born in 92000, and this is falsé."""
      _UpperCAmelCase						 =      [13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
      _UpperCAmelCase						 =      ["""▁""", """I""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """é""", """.""", ]
      _UpperCAmelCase						 =      ["""▁""", """<unk>""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """.""", ]
      # fmt: on

      _UpperCAmelCase						 =      tokenizer.encode(A_   ,				add_special_tokens=A_					)
      self.assertListEqual(A_   ,				A_					)
      _UpperCAmelCase						 =      tokenizer.tokenize(A_					)
      self.assertListEqual(A_   ,				A_					)
      _UpperCAmelCase						 =      tokenizer.convert_ids_to_tokens(A_					)
      self.assertListEqual(A_   ,				A_					)

      _UpperCAmelCase						 =      rust_tokenizer.encode(A_   ,				add_special_tokens=A_					)
      self.assertListEqual(A_   ,				A_					)
      _UpperCAmelCase						 =      rust_tokenizer.tokenize(A_					)
      self.assertListEqual(A_   ,				A_					)
      _UpperCAmelCase						 =      rust_tokenizer.convert_ids_to_tokens(A_					)
      self.assertListEqual(A_   ,				A_					)


    def      lowerCAmelCase_  (					self	:  Dict					):
      _UpperCAmelCase						 =      DebertaVaTokenizer(A_					)

      _UpperCAmelCase						 =      tokenizer.encode("""sequence builders"""					)
      _UpperCAmelCase						 =      tokenizer.encode("""multi-sequence build"""					)

      _UpperCAmelCase						 =      tokenizer.build_inputs_with_special_tokens(A_					)
      _UpperCAmelCase						 =      tokenizer.build_inputs_with_special_tokens(A_   ,				A_					)

      self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]   ,				A_					)
      self.assertEqual(
          [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]   ,				A_   ,				)


    @slow
    def      lowerCAmelCase_  (					self	:  List[Any]					):
      # fmt: off
      _UpperCAmelCase						 =      {"""input_ids""": [[1, 3_9867, 36, 1_9390, 486, 27, 3_5052, 8_1436, 18, 6_0685, 1225, 7, 3_5052, 8_1436, 18, 9367, 1_6899, 18, 1_5937, 53, 594, 773, 18, 1_6287, 3_0465, 36, 1_5937, 6, 4_1139, 38, 3_6979, 6_0763, 191, 6, 3_4132, 99, 6, 5_0538, 390, 4_3230, 6, 3_4132, 2779, 2_0850, 14, 699, 1072, 1194, 36, 382, 1_0901, 53, 7, 699, 1072, 2084, 36, 2_0422, 630, 53, 19, 105, 3049, 1896, 1053, 1_6899, 1506, 11, 3_7978, 4243, 7, 1237, 3_1869, 200, 1_6566, 654, 6, 3_5052, 8_1436, 7, 5_5630, 1_3593, 4, 2], [1, 26, 1_5011, 13, 667, 8, 1053, 18, 2_3611, 1237, 7_2356, 1_2820, 34, 10_4134, 1209, 35, 1_3313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 1_5785, 1_4951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
      # fmt: on

      self.tokenizer_integration_test_util(
          expected_encoding=A_   ,				model_name="""microsoft/deberta-v2-xlarge"""   ,				revision="""ad6e42c1532ddf3a15c39246b63f5559d558b670"""   ,				)

"""simple docstring"""


from __future__ import annotations






def     __UpperCAmelCase					(			lowercase		):







  """simple docstring"""
  return [ord(lowercase		) - 96 for elem in plain]






def     __UpperCAmelCase					(			lowercase		):







  """simple docstring"""
  return "".join(chr(elem + 96		) for elem in encoded		)






def     __UpperCAmelCase					(			):







  """simple docstring"""
  _UpperCAmelCase						 =      encode(input("""-> """		).strip().lower()		)
  print("""Encoded: """   ,lowercase		)
  print("""Decoded:"""   ,decode(lowercase		)		)


if __name__ == "__main__":
    main()
