import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__  =					get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model''')




@require_sentencepiece
@require_tokenizers
class 					_UpperCAmelCase			(     lowerCAmelCase,			unittest.TestCase  ):



 '''simple docstring'''




 __A			       =					GPTSwaTokenizer
 __A			       =					False
 __A			       =					True
 __A			       =					False







 def 		__UpperCAmelCase				(		self				:   Dict)						->	Dict:


   """simple docstring"""







   super().setUp()

   # We have a SentencePiece fixture for testing
   _UpperCamelCase           =    GPTSwaTokenizer(lowercase_							, eos_token="<unk>"							, bos_token="<unk>"							, pad_token="<unk>")

   tokenizer.save_pretrained(self.tmpdirname)







 def 		__UpperCAmelCase				(		self				:   Union[str, Any]							, lowercase_				:   str)						->	List[str]:


   """simple docstring"""







   _UpperCamelCase           =    "This is a test"
   _UpperCamelCase           =    "This is a test"
   return input_text, output_text







 def 		__UpperCAmelCase				(		self				:   int)						->	List[str]:


   """simple docstring"""







   _UpperCamelCase           =    "<s>"
   _UpperCamelCase           =    1

   self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase_)							, lowercase_)
   self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase_)							, lowercase_)







 def 		__UpperCAmelCase				(		self				:   Dict)						->	Optional[Any]:


   """simple docstring"""







   _UpperCamelCase           =    list(self.get_tokenizer().get_vocab().keys())

   self.assertEqual(vocab_keys[0]							, "<unk>")
   self.assertEqual(vocab_keys[1]							, "<s>")
   self.assertEqual(vocab_keys[-1]							, "j")
   self.assertEqual(len(lowercase_)							, 2000)







 def 		__UpperCAmelCase				(		self				:   Tuple)						->	Any:


   """simple docstring"""







   self.assertEqual(self.get_tokenizer().vocab_size							, 2000)







 def 		__UpperCAmelCase				(		self				:   str)						->	Optional[Any]:


   """simple docstring"""







   _UpperCamelCase           =    GPTSwaTokenizer(lowercase_)

   _UpperCamelCase           =    tokenizer.tokenize("This is a test")
   self.assertListEqual(lowercase_							, ["▁This", "▁is", "▁a", "▁t", "est"])

   self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_)							, [465, 287, 265, 631, 842])

   _UpperCamelCase           =    tokenizer.tokenize("I was born in 92000, and this is falsé.")
   # fmt: off
   self.assertListEqual(
       lowercase_							, ["▁I", "▁was", "▁bor", "n", "▁in", "▁", "<0x39>", "2", "0", "0", "0", ",", "▁and", "▁this", "▁is", "▁f", "al", "s", "<0xC3>", "<0xA9>", "."]							, )
   # fmt: on

   _UpperCamelCase           =    tokenizer.convert_tokens_to_ids(lowercase_)
   self.assertListEqual(
       lowercase_							, [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]							, )

   _UpperCamelCase           =    tokenizer.convert_ids_to_tokens(lowercase_)
   # fmt: off
   self.assertListEqual(
       lowercase_							, ["▁I", "▁was", "▁bor", "n", "▁in", "▁", "<0x39>", "2", "0", "0", "0", ",", "▁and", "▁this", "▁is", "▁f", "al", "s", "<0xC3>", "<0xA9>", "."])
   # fmt: on







 def 		__UpperCAmelCase				(		self				:   int)						->	Any:


   """simple docstring"""







   _UpperCamelCase           =    GPTSwaTokenizer(lowercase_)
   _UpperCamelCase           =    ["This is a test", "I was born in 92000, and this is falsé."]
   _UpperCamelCase           =    [
       [465, 287, 265, 631, 842],
       [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
   ]

   # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
   for text, expected_ids in zip(lowercase_							, lowercase_):
     self.assertListEqual(tokenizer.encode_fast(lowercase_)							, lowercase_)

   # Test that decode_fast returns the input text
   for text, token_ids in zip(lowercase_							, lowercase_):
     self.assertEqual(tokenizer.decode_fast(lowercase_)							, lowercase_)







 @slow
 def 		__UpperCAmelCase				(		self				:   List[str])						->	Any:


   """simple docstring"""







   _UpperCamelCase           =    [
       "<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')",
       "Hey there, how are you doing this fine day?",
       "This is a text with a trailing spaces followed by a dot     .",
       "Häj sväjs lillebrör! =)",
       "Det är inget fel på Mr. Cool",
   ]

   # fmt: off
   _UpperCamelCase           =    {"input_ids": [[63423, 5, 6811, 14954, 282, 816, 3821, 63466, 63425, 63462, 18, 63978, 678, 301, 1320, 63423, 63455, 63458, 18, 63982, 4246, 3940, 1901, 47789, 5547, 18994], [19630, 1100, 63446, 1342, 633, 544, 4488, 593, 5102, 2416, 63495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 58593, 22413, 9106, 546, 268, 33213, 63979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [55130, 63450, 924, 63449, 2249, 4062, 1558, 318, 63504, 21498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 63443, 26801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
   # fmt: on
   self.tokenizer_integration_test_util(
       expected_encoding=lowercase_							, model_name="AI-Sweden/gpt-sw3-126m"							, sequences=lowercase_							, )





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase__  =					{
    '''configuration_deberta''': ['''DEBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''DebertaConfig''', '''DebertaOnnxConfig'''],
    '''tokenization_deberta''': ['''DebertaTokenizer'''],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   lowerCamelCase__  =					['''DebertaTokenizerFast''']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   lowerCamelCase__  =					[
       '''DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''DebertaForMaskedLM''',
       '''DebertaForQuestionAnswering''',
       '''DebertaForSequenceClassification''',
       '''DebertaForTokenClassification''',
       '''DebertaModel''',
       '''DebertaPreTrainedModel''',
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   lowerCamelCase__  =					[
       '''TF_DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''TFDebertaForMaskedLM''',
       '''TFDebertaForQuestionAnswering''',
       '''TFDebertaForSequenceClassification''',
       '''TFDebertaForTokenClassification''',
       '''TFDebertaModel''',
       '''TFDebertaPreTrainedModel''',
   ]


if TYPE_CHECKING:
   from .configuration_deberta import DEBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, DebertaConfig, DebertaOnnxConfig
   from .tokenization_deberta import DebertaTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_deberta_fast import DebertaTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_deberta import (
          DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
          DebertaForMaskedLM,
          DebertaForQuestionAnswering,
          DebertaForSequenceClassification,
          DebertaForTokenClassification,
          DebertaModel,
          DebertaPreTrainedModel,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_deberta import (
          TF_DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFDebertaForMaskedLM,
          TFDebertaForQuestionAnswering,
          TFDebertaForSequenceClassification,
          TFDebertaForTokenClassification,
          TFDebertaModel,
          TFDebertaPreTrainedModel,
      )


else:
   import sys

   lowerCamelCase__  =					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




