




'''simple docstring'''

import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


a       :						List[Any]								=		get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model''')



@require_sentencepiece
@require_tokenizers
class 		SCREAMING_SNAKE_CASE__  (			_UpperCamelCase				,     unittest.TestCase	):
			__SCREAMING_SNAKE_CASE    =			GPTSwaTokenizer
			__SCREAMING_SNAKE_CASE    =			False
			__SCREAMING_SNAKE_CASE    =			True
			__SCREAMING_SNAKE_CASE    =			False





			def 							A (      self						:    int							):



						"""simple docstring"""




						super().setUp()

						# We have a SentencePiece fixture for testing
						__snake_case			=		GPTSwaTokenizer(a_      ,		eos_token="<unk>"      ,		bos_token="<unk>"      ,		pad_token="<unk>"							)

						tokenizer.save_pretrained(self.tmpdirname							)





			def 							A (      self						:    str      ,		a_						:    List[Any]							):



						"""simple docstring"""




						__snake_case			=		"This is a test"
						__snake_case			=		"This is a test"
						return input_text, output_text





			def 							A (      self						:    Union[str, Any]							):



						"""simple docstring"""




						__snake_case			=		"<s>"
						__snake_case			=		1

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(a_							)      ,		a_							)
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(a_							)      ,		a_							)





			def 							A (      self						:    Tuple							):



						"""simple docstring"""




						__snake_case			=		list(self.get_tokenizer().get_vocab().keys()							)

						self.assertEqual(vocab_keys[0]      ,		"<unk>"							)
						self.assertEqual(vocab_keys[1]      ,		"<s>"							)
						self.assertEqual(vocab_keys[-1]      ,		"j"							)
						self.assertEqual(len(a_							)      ,		2_000							)





			def 							A (      self						:    Optional[int]							):



						"""simple docstring"""




						self.assertEqual(self.get_tokenizer().vocab_size      ,		2_000							)





			def 							A (      self						:    Dict							):



						"""simple docstring"""




						__snake_case			=		GPTSwaTokenizer(a_							)

						__snake_case			=		tokenizer.tokenize("This is a test"							)
						self.assertListEqual(a_      ,		["▁This", "▁is", "▁a", "▁t", "est"]							)

						self.assertListEqual(tokenizer.convert_tokens_to_ids(a_							)      ,		[465, 287, 265, 631, 842]							)

						__snake_case			=		tokenizer.tokenize("I was born in 92000, and this is falsé."							)
						# fmt: off
						self.assertListEqual(
						    a_      ,		["▁I", "▁was", "▁bor", "n", "▁in", "▁", "<0x39>", "2", "0", "0", "0", ",", "▁and", "▁this", "▁is", "▁f", "al", "s", "<0xC3>", "<0xA9>", "."]      ,		)
						# fmt: on

						__snake_case			=		tokenizer.convert_tokens_to_ids(a_							)
						self.assertListEqual(
						    a_      ,		[262, 272, 1_525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]      ,		)

						__snake_case			=		tokenizer.convert_ids_to_tokens(a_							)
						# fmt: off
						self.assertListEqual(
						    a_      ,		["▁I", "▁was", "▁bor", "n", "▁in", "▁", "<0x39>", "2", "0", "0", "0", ",", "▁and", "▁this", "▁is", "▁f", "al", "s", "<0xC3>", "<0xA9>", "."]							)
						# fmt: on





			def 							A (      self						:    List[str]							):



						"""simple docstring"""




						__snake_case			=		GPTSwaTokenizer(a_							)
						__snake_case			=		["This is a test", "I was born in 92000, and this is falsé."]
						__snake_case			=		[
						    [465, 287, 265, 631, 842],
						    [262, 272, 1_525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
						]

						# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
						for text, expected_ids in zip(a_      ,		a_							):
									self.assertListEqual(tokenizer.encode_fast(a_							)      ,		a_							)

						# Test that decode_fast returns the input text
						for text, token_ids in zip(a_      ,		a_							):
									self.assertEqual(tokenizer.decode_fast(a_							)      ,		a_							)






			@slow
			def 							A (      self						:    Any							):



						"""simple docstring"""




						__snake_case			=		[
						    "<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')",
						    "Hey there, how are you doing this fine day?",
						    "This is a text with a trailing spaces followed by a dot     .",
						    "Häj sväjs lillebrör! =)",
						    "Det är inget fel på Mr. Cool",
						]

						# fmt: off
						__snake_case			=		{"input_ids": [[63_423, 5, 6_811, 14_954, 282, 816, 3_821, 63_466, 63_425, 63_462, 18, 63_978, 678, 301, 1_320, 63_423, 63_455, 63_458, 18, 63_982, 4_246, 3_940, 1_901, 47_789, 5_547, 18_994], [19_630, 1_100, 63_446, 1_342, 633, 544, 4_488, 593, 5_102, 2_416, 63_495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_652, 428, 268, 1_936, 515, 268, 58_593, 22_413, 9_106, 546, 268, 33_213, 63_979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [55_130, 63_450, 924, 63_449, 2_249, 4_062, 1_558, 318, 63_504, 21_498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2_827, 2_559, 332, 6_575, 63_443, 26_801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
						# fmt: on
						self.tokenizer_integration_test_util(
						    expected_encoding=a_      ,		model_name="AI-Sweden/gpt-sw3-126m"      ,		sequences=a_      ,		)








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__A			:					Dict						=			{
    'configuration_bigbird_pegasus': [
        'BIGBIRD_PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'BigBirdPegasusConfig',
        'BigBirdPegasusOnnxConfig',
    ],
}

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __A			:					int						=			[
        'BIGBIRD_PEGASUS_PRETRAINED_MODEL_ARCHIVE_LIST',
        'BigBirdPegasusForCausalLM',
        'BigBirdPegasusForConditionalGeneration',
        'BigBirdPegasusForQuestionAnswering',
        'BigBirdPegasusForSequenceClassification',
        'BigBirdPegasusModel',
        'BigBirdPegasusPreTrainedModel',
    ]


if TYPE_CHECKING:
    from .configuration_bigbird_pegasus import (
        BIGBIRD_PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP,
        BigBirdPegasusConfig,
        BigBirdPegasusOnnxConfig,
    )

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_bigbird_pegasus import (
            BIGBIRD_PEGASUS_PRETRAINED_MODEL_ARCHIVE_LIST,
            BigBirdPegasusForCausalLM,
            BigBirdPegasusForConditionalGeneration,
            BigBirdPegasusForQuestionAnswering,
            BigBirdPegasusForSequenceClassification,
            BigBirdPegasusModel,
            BigBirdPegasusPreTrainedModel,
        )


else:
    import sys

    __A			:					List[Any]						=			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)