



"""simple docstring"""







import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowercase__		:    Dict				   =     get_tests_dir("""fixtures/test_sentencepiece_with_bytefallback.model""")







@require_sentencepiece
@require_tokenizers
class 				UpperCamelCase__       (     lowercase_, unittest.TestCase  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE      =					GPTSwaTokenizer
      _SCREAMING_SNAKE_CASE      =					False
      _SCREAMING_SNAKE_CASE      =					True
      _SCREAMING_SNAKE_CASE      =					False



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Tuple	):
        super().setUp()

        # We have a SentencePiece fixture for testing
        lowerCAmelCase_       :    int								=			GPTSwaTokenizer(SCREAMING_SNAKE_CASE_   ,      eos_token='<unk>'   ,      bos_token='<unk>'   ,      pad_token='<unk>'	)

        tokenizer.save_pretrained(self.tmpdirname	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				List[Any]   ,      SCREAMING_SNAKE_CASE_    :				Dict	):
        lowerCAmelCase_       :    List[str]								=			'This is a test'
        lowerCAmelCase_       :    int								=			'This is a test'
        return input_text, output_text



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Any	):
        lowerCAmelCase_       :    Union[str, Any]								=			'<s>'
        lowerCAmelCase_       :    Dict								=			1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE_	)   ,      SCREAMING_SNAKE_CASE_	)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE_	)   ,      SCREAMING_SNAKE_CASE_	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Union[str, Any]	):
        lowerCAmelCase_       :    List[str]								=			list(self.get_tokenizer().get_vocab().keys()	)

        self.assertEqual(vocab_keys[0]   ,      '<unk>'	)
        self.assertEqual(vocab_keys[1]   ,      '<s>'	)
        self.assertEqual(vocab_keys[-1]   ,      'j'	)
        self.assertEqual(len(SCREAMING_SNAKE_CASE_	)   ,      2_0_0_0	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Tuple	):
        self.assertEqual(self.get_tokenizer().vocab_size   ,      2_0_0_0	)



      def 					SCREAMING_SNAKE_CASE__			(				self    :				Dict	):
        lowerCAmelCase_       :    Optional[int]								=			GPTSwaTokenizer(SCREAMING_SNAKE_CASE_	)

        lowerCAmelCase_       :    List[Any]								=			tokenizer.tokenize('This is a test'	)
        self.assertListEqual(SCREAMING_SNAKE_CASE_   ,      ['▁This', '▁is', '▁a', '▁t', 'est']	)

        self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_	)   ,      [4_6_5, 2_8_7, 2_6_5, 6_3_1, 8_4_2]	)

        lowerCAmelCase_       :    List[str]								=			tokenizer.tokenize('I was born in 92000, and this is falsé.'	)
        # fmt: off
        self.assertListEqual(
            SCREAMING_SNAKE_CASE_   ,      ['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']   ,      )
        # fmt: on

        lowerCAmelCase_       :    Optional[int]								=			tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_	)
        self.assertListEqual(
            SCREAMING_SNAKE_CASE_   ,      [2_6_2, 2_7_2, 1_5_2_5, 2_8_6, 2_7_1, 2_6_8, 6_0, 9_1_6, 6_3_3, 6_3_3, 6_3_3, 2_5_9, 2_6_6, 3_0_1, 2_8_7, 3_8_4, 3_6_7, 2_6_3, 1_9_8, 1_7_2, 2_6_0]   ,      )

        lowerCAmelCase_       :    str								=			tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_	)
        # fmt: off
        self.assertListEqual(
            SCREAMING_SNAKE_CASE_   ,      ['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']	)
        # fmt: on



      def 					SCREAMING_SNAKE_CASE__			(				self    :				str	):
        lowerCAmelCase_       :    List[str]								=			GPTSwaTokenizer(SCREAMING_SNAKE_CASE_	)
        lowerCAmelCase_       :    Optional[int]								=			['This is a test', 'I was born in 92000, and this is falsé.']
        lowerCAmelCase_       :    Union[str, Any]								=			[
            [4_6_5, 2_8_7, 2_6_5, 6_3_1, 8_4_2],
            [2_6_2, 2_7_2, 1_5_2_5, 2_8_6, 2_7_1, 2_6_8, 6_0, 9_1_6, 6_3_3, 6_3_3, 6_3_3, 2_5_9, 2_6_6, 3_0_1, 2_8_7, 3_8_4, 3_6_7, 2_6_3, 1_9_8, 1_7_2, 2_6_0],
        ]

        # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
        for text, expected_ids in zip(SCREAMING_SNAKE_CASE_   ,      SCREAMING_SNAKE_CASE_	):
          self.assertListEqual(tokenizer.encode_fast(SCREAMING_SNAKE_CASE_	)   ,      SCREAMING_SNAKE_CASE_	)

        # Test that decode_fast returns the input text
        for text, token_ids in zip(SCREAMING_SNAKE_CASE_   ,      SCREAMING_SNAKE_CASE_	):
          self.assertEqual(tokenizer.decode_fast(SCREAMING_SNAKE_CASE_	)   ,      SCREAMING_SNAKE_CASE_	)





      @slow
      def 					SCREAMING_SNAKE_CASE__			(				self    :				Tuple	):
        lowerCAmelCase_       :    Optional[Any]								=			[
            '<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')',
            'Hey there, how are you doing this fine day?',
            'This is a text with a trailing spaces followed by a dot     .',
            'Häj sväjs lillebrör! =)',
            'Det är inget fel på Mr. Cool',
        ]

        # fmt: off
        lowerCAmelCase_       :    Union[str, Any]								=			{'input_ids': [[6_3_4_2_3, 5, 6_8_1_1, 1_4_9_5_4, 2_8_2, 8_1_6, 3_8_2_1, 6_3_4_6_6, 6_3_4_2_5, 6_3_4_6_2, 1_8, 6_3_9_7_8, 6_7_8, 3_0_1, 1_3_2_0, 6_3_4_2_3, 6_3_4_5_5, 6_3_4_5_8, 1_8, 6_3_9_8_2, 4_2_4_6, 3_9_4_0, 1_9_0_1, 4_7_7_8_9, 5_5_4_7, 1_8_9_9_4], [1_9_6_3_0, 1_1_0_0, 6_3_4_4_6, 1_3_4_2, 6_3_3, 5_4_4, 4_4_8_8, 5_9_3, 5_1_0_2, 2_4_1_6, 6_3_4_9_5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_6_5_2, 4_2_8, 2_6_8, 1_9_3_6, 5_1_5, 2_6_8, 5_8_5_9_3, 2_2_4_1_3, 9_1_0_6, 5_4_6, 2_6_8, 3_3_2_1_3, 6_3_9_7_9, 6_9_8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5_1_3_0, 6_3_4_5_0, 9_2_4, 6_3_4_4_9, 2_2_4_9, 4_0_6_2, 1_5_5_8, 3_1_8, 6_3_5_0_4, 2_1_4_9_8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_0_9, 3_7_7, 2_8_2_7, 2_5_5_9, 3_3_2, 6_5_7_5, 6_3_4_4_3, 2_6_8_0_1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
        # fmt: on
        self.tokenizer_integration_test_util(
            expected_encoding=SCREAMING_SNAKE_CASE_   ,      model_name='AI-Sweden/gpt-sw3-126m'   ,      sequences=SCREAMING_SNAKE_CASE_   ,      )










"""simple docstring"""







import baseaa

def     UpperCamelCase_   (   lowerCAmelCase__				:    str						)						->       bytes:

  """simple docstring"""




  return baseaa.aaaencode(string.encode('utf-8'						)						)

def     UpperCamelCase_   (   lowerCAmelCase__				:    bytes						)						->       str:

  """simple docstring"""




  return baseaa.aaadecode(lowerCAmelCase__						).decode('utf-8'						)


if __name__ == "__main__":
     import doctest

     doctest.testmod()





