


import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCamelCase     		=							get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model')







@require_sentencepiece
@require_tokenizers
class 							lowerCamelCase__						(				UpperCAmelCase       ,						unittest.TestCase   ):






			"""simple docstring"""






			_UpperCamelCase		:       Dict 			=		GPTSwaTokenizer
			_UpperCamelCase		:       Any 			=		False
			_UpperCamelCase		:       Optional[int] 			=		True
			_UpperCamelCase		:       Any 			=		False




			def 					snake_case__					(				self  ):

						'''simple docstring'''


						super().setUp()

						# We have a SentencePiece fixture for testing
						UpperCamelCase__       =							GPTSwaTokenizer(snake_case   ,    eos_token="<unk>"   ,    bos_token="<unk>"   ,    pad_token="<unk>"  )

						tokenizer.save_pretrained(self.tmpdirname  )




			def 					snake_case__					(				self   ,    snake_case  ):

						'''simple docstring'''


						UpperCamelCase__       =							"This is a test"
						UpperCamelCase__       =							"This is a test"
						return input_text, output_text




			def 					snake_case__					(				self  ):

						'''simple docstring'''


						UpperCamelCase__       =							"<s>"
						UpperCamelCase__       =							1

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case  )   ,    snake_case  )
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case  )   ,    snake_case  )




			def 					snake_case__					(				self  ):

						'''simple docstring'''


						UpperCamelCase__       =							list(self.get_tokenizer().get_vocab().keys()  )

						self.assertEqual(vocab_keys[0]   ,    "<unk>"  )
						self.assertEqual(vocab_keys[1]   ,    "<s>"  )
						self.assertEqual(vocab_keys[-1]   ,    "j"  )
						self.assertEqual(len(snake_case  )   ,    2000  )




			def 					snake_case__					(				self  ):

						'''simple docstring'''


						self.assertEqual(self.get_tokenizer().vocab_size   ,    2000  )




			def 					snake_case__					(				self  ):

						'''simple docstring'''


						UpperCamelCase__       =							GPTSwaTokenizer(snake_case  )

						UpperCamelCase__       =							tokenizer.tokenize("This is a test"  )
						self.assertListEqual(snake_case   ,    ["▁This", "▁is", "▁a", "▁t", "est"]  )

						self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case  )   ,    [465, 287, 265, 631, 842]  )

						UpperCamelCase__       =							tokenizer.tokenize("I was born in 92000, and this is falsé."  )
						# fmt: off
						self.assertListEqual(
						    snake_case   ,    ["▁I", "▁was", "▁bor", "n", "▁in", "▁", "<0x39>", "2", "0", "0", "0", ",", "▁and", "▁this", "▁is", "▁f", "al", "s", "<0xC3>", "<0xA9>", "."]   ,    )
						# fmt: on

						UpperCamelCase__       =							tokenizer.convert_tokens_to_ids(snake_case  )
						self.assertListEqual(
						    snake_case   ,    [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]   ,    )

						UpperCamelCase__       =							tokenizer.convert_ids_to_tokens(snake_case  )
						# fmt: off
						self.assertListEqual(
						    snake_case   ,    ["▁I", "▁was", "▁bor", "n", "▁in", "▁", "<0x39>", "2", "0", "0", "0", ",", "▁and", "▁this", "▁is", "▁f", "al", "s", "<0xC3>", "<0xA9>", "."]  )
						# fmt: on




			def 					snake_case__					(				self  ):

						'''simple docstring'''


						UpperCamelCase__       =							GPTSwaTokenizer(snake_case  )
						UpperCamelCase__       =							["This is a test", "I was born in 92000, and this is falsé."]
						UpperCamelCase__       =							[
						    [465, 287, 265, 631, 842],
						    [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
						]

						# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
						for text, expected_ids in zip(snake_case   ,    snake_case  ):
									self.assertListEqual(tokenizer.encode_fast(snake_case  )   ,    snake_case  )

						# Test that decode_fast returns the input text
						for text, token_ids in zip(snake_case   ,    snake_case  ):
									self.assertEqual(tokenizer.decode_fast(snake_case  )   ,    snake_case  )







			@slow
			def 					snake_case__					(				self  ):

						'''simple docstring'''


						UpperCamelCase__       =							[
						    "<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')",
						    "Hey there, how are you doing this fine day?",
						    "This is a text with a trailing spaces followed by a dot     .",
						    "Häj sväjs lillebrör! =)",
						    "Det är inget fel på Mr. Cool",
						]

						# fmt: off
						UpperCamelCase__       =							{"input_ids": [[63423, 5, 6811, 14954, 282, 816, 3821, 63466, 63425, 63462, 18, 63978, 678, 301, 1320, 63423, 63455, 63458, 18, 63982, 4246, 3940, 1901, 47789, 5547, 18994], [19630, 1100, 63446, 1342, 633, 544, 4488, 593, 5102, 2416, 63495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 58593, 22413, 9106, 546, 268, 33213, 63979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [55130, 63450, 924, 63449, 2249, 4062, 1558, 318, 63504, 21498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 63443, 26801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
						# fmt: on
						self.tokenizer_integration_test_util(
						    expected_encoding=snake_case   ,    model_name="AI-Sweden/gpt-sw3-126m"   ,    sequences=snake_case   ,    )






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__UpperCamelCase     		=							{
    'configuration_xlm_roberta_xl': [
        'XLM_ROBERTA_XL_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'XLMRobertaXLConfig',
        'XLMRobertaXLOnnxConfig',
    ],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__UpperCamelCase     		=							[
						    'XLM_ROBERTA_XL_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'XLMRobertaXLForCausalLM',
						    'XLMRobertaXLForMaskedLM',
						    'XLMRobertaXLForMultipleChoice',
						    'XLMRobertaXLForQuestionAnswering',
						    'XLMRobertaXLForSequenceClassification',
						    'XLMRobertaXLForTokenClassification',
						    'XLMRobertaXLModel',
						    'XLMRobertaXLPreTrainedModel',
						]

if TYPE_CHECKING:
						from .configuration_xlm_roberta_xl import (
						    XLM_ROBERTA_XL_PRETRAINED_CONFIG_ARCHIVE_MAP,
						    XLMRobertaXLConfig,
						    XLMRobertaXLOnnxConfig,
						)

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_xlm_roberta_xl import (
												    XLM_ROBERTA_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
												    XLMRobertaXLForCausalLM,
												    XLMRobertaXLForMaskedLM,
												    XLMRobertaXLForMultipleChoice,
												    XLMRobertaXLForQuestionAnswering,
												    XLMRobertaXLForSequenceClassification,
												    XLMRobertaXLForTokenClassification,
												    XLMRobertaXLModel,
												    XLMRobertaXLPreTrainedModel,
												)

else:
						import sys

						__UpperCamelCase     		=							_LazyModule(__name__, globals()['__file__'], _import_structure)


