






"""simple docstring"""






import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


A_      	=     get_tests_dir("fixtures/test_sentencepiece_with_bytefallback.model")
@require_sentencepiece
@require_tokenizers
class     __lowerCAmelCase (      UpperCAmelCase__							, unittest.TestCase							):







  '''simple docstring'''


  __lowerCamelCase							:     Union[str, Any]  						=      GPTSwaTokenizer
  __lowerCamelCase							:     Union[str, Any]  						=      False
  __lowerCamelCase							:     Any  						=      True
  __lowerCamelCase							:     Any  						=      False
  def        UpperCamelCase__ (   self:		List[Any]       ):
     super().setUp()

     # We have a SentencePiece fixture for testing
     UpperCamelCase_			     =GPTSwaTokenizer(_UpperCAmelCase					,      eos_token="<unk>"					,      bos_token="<unk>"					,      pad_token="<unk>"       )

     tokenizer.save_pretrained(self.tmpdirname       )
  def        UpperCamelCase__ (   self:		Union[str, Any]					,      UpperCamelCase_:		Any       ):
     UpperCamelCase_			     ="This is a test"
     UpperCamelCase_			     ="This is a test"
     return input_text, output_text
  def        UpperCamelCase__ (   self:		int       ):
     UpperCamelCase_			     ="<s>"
     UpperCamelCase_			     =1

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCAmelCase       )					,      _UpperCAmelCase       )
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCAmelCase       )					,      _UpperCAmelCase       )
  def        UpperCamelCase__ (   self:		Optional[Any]       ):
     UpperCamelCase_			     =list(self.get_tokenizer().get_vocab().keys()       )

     self.assertEqual(vocab_keys[0]					,      "<unk>"       )
     self.assertEqual(vocab_keys[1]					,      "<s>"       )
     self.assertEqual(vocab_keys[-1]					,      "j"       )
     self.assertEqual(len(_UpperCAmelCase       )					,      2000       )
  def        UpperCamelCase__ (   self:		Tuple       ):
     self.assertEqual(self.get_tokenizer().vocab_size					,      2000       )
  def        UpperCamelCase__ (   self:		Dict       ):
     UpperCamelCase_			     =GPTSwaTokenizer(_UpperCAmelCase       )

     UpperCamelCase_			     =tokenizer.tokenize("This is a test"       )
     self.assertListEqual(_UpperCAmelCase					,      ["▁This", "▁is", "▁a", "▁t", "est"]       )

     self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase       )					,      [465, 287, 265, 631, 842]       )

     UpperCamelCase_			     =tokenizer.tokenize("I was born in 92000, and this is falsé."       )
     # fmt: off
     self.assertListEqual(
         _UpperCAmelCase					,      ["▁I", "▁was", "▁bor", "n", "▁in", "▁", "<0x39>", "2", "0", "0", "0", ",", "▁and", "▁this", "▁is", "▁f", "al", "s", "<0xC3>", "<0xA9>", "."]					,      )
     # fmt: on

     UpperCamelCase_			     =tokenizer.convert_tokens_to_ids(_UpperCAmelCase       )
     self.assertListEqual(
         _UpperCAmelCase					,      [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]					,      )

     UpperCamelCase_			     =tokenizer.convert_ids_to_tokens(_UpperCAmelCase       )
     # fmt: off
     self.assertListEqual(
         _UpperCAmelCase					,      ["▁I", "▁was", "▁bor", "n", "▁in", "▁", "<0x39>", "2", "0", "0", "0", ",", "▁and", "▁this", "▁is", "▁f", "al", "s", "<0xC3>", "<0xA9>", "."]       )
     # fmt: on
  def        UpperCamelCase__ (   self:		Tuple       ):
     UpperCamelCase_			     =GPTSwaTokenizer(_UpperCAmelCase       )
     UpperCamelCase_			     =["This is a test", "I was born in 92000, and this is falsé."]
     UpperCamelCase_			     =[
         [465, 287, 265, 631, 842],
         [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
     ]

     # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
     for text, expected_ids in zip(_UpperCAmelCase					,      _UpperCAmelCase       ):
        self.assertListEqual(tokenizer.encode_fast(_UpperCAmelCase       )					,      _UpperCAmelCase       )

     # Test that decode_fast returns the input text
     for text, token_ids in zip(_UpperCAmelCase					,      _UpperCAmelCase       ):
        self.assertEqual(tokenizer.decode_fast(_UpperCAmelCase       )					,      _UpperCAmelCase       )





  @slow
  def        UpperCamelCase__ (   self:		int       ):
     UpperCamelCase_			     =[
         "<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')",
         "Hey there, how are you doing this fine day?",
         "This is a text with a trailing spaces followed by a dot     .",
         "Häj sväjs lillebrör! =)",
         "Det är inget fel på Mr. Cool",
     ]

     # fmt: off
     UpperCamelCase_			     ={"input_ids": [[6_3423, 5, 6811, 1_4954, 282, 816, 3821, 6_3466, 6_3425, 6_3462, 18, 6_3978, 678, 301, 1320, 6_3423, 6_3455, 6_3458, 18, 6_3982, 4246, 3940, 1901, 4_7789, 5547, 1_8994], [1_9630, 1100, 6_3446, 1342, 633, 544, 4488, 593, 5102, 2416, 6_3495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 5_8593, 2_2413, 9106, 546, 268, 3_3213, 6_3979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5130, 6_3450, 924, 6_3449, 2249, 4062, 1558, 318, 6_3504, 2_1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 6_3443, 2_6801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
     # fmt: on
     self.tokenizer_integration_test_util(
         expected_encoding=_UpperCAmelCase					,      model_name="AI-Sweden/gpt-sw3-126m"					,      sequences=_UpperCAmelCase					,      )




import os
import sys

A			:						Tuple        =  os.path.join(os.path.dirname(__file__), 'src')
sys.path.append(SRC_DIR)


from transformers import (
    AutoConfig,
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForMaskedLM,
    AutoModelForQuestionAnswering,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    add_start_docstrings,
)


A			:						Dict        =  [
    'torch',
    'numpy',
    'tokenizers',
    'filelock',
    'requests',
    'tqdm',
    'regex',
    'sentencepiece',
    'sacremoses',
    'importlib_metadata',
    'huggingface_hub',
]






@add_start_docstrings(AutoConfig.__doc__	)
def 					UpperCamelCase   (      *__magic_name__	:				str       ,       **__magic_name__	:				List[Any]	)  ->    str:
		"""simple docstring"""




		return AutoConfig.from_pretrained(*__magic_name__       ,       **__magic_name__	)






@add_start_docstrings(AutoTokenizer.__doc__	)
def 					UpperCamelCase   (      *__magic_name__	:				Any       ,       **__magic_name__	:				Optional[Any]	)  ->    Optional[Any]:
		"""simple docstring"""




		return AutoTokenizer.from_pretrained(*__magic_name__       ,       **__magic_name__	)






@add_start_docstrings(AutoModel.__doc__	)
def 					UpperCamelCase   (      *__magic_name__	:				str       ,       **__magic_name__	:				Optional[Any]	)  ->    Tuple:
		"""simple docstring"""




		return AutoModel.from_pretrained(*__magic_name__       ,       **__magic_name__	)






@add_start_docstrings(AutoModelForCausalLM.__doc__	)
def 					UpperCamelCase   (      *__magic_name__	:				str       ,       **__magic_name__	:				Tuple	)  ->    List[str]:
		"""simple docstring"""




		return AutoModelForCausalLM.from_pretrained(*__magic_name__       ,       **__magic_name__	)






@add_start_docstrings(AutoModelForMaskedLM.__doc__	)
def 					UpperCamelCase   (      *__magic_name__	:				Optional[Any]       ,       **__magic_name__	:				Any	)  ->    Optional[int]:
		"""simple docstring"""




		return AutoModelForMaskedLM.from_pretrained(*__magic_name__       ,       **__magic_name__	)






@add_start_docstrings(AutoModelForSequenceClassification.__doc__	)
def 					UpperCamelCase   (      *__magic_name__	:				Any       ,       **__magic_name__	:				List[str]	)  ->    Union[str, Any]:
		"""simple docstring"""




		return AutoModelForSequenceClassification.from_pretrained(*__magic_name__       ,       **__magic_name__	)







@add_start_docstrings(AutoModelForQuestionAnswering.__doc__	)
def 					UpperCamelCase   (      *__magic_name__	:				Dict       ,       **__magic_name__	:				List[Any]	)  ->    int:
		"""simple docstring"""




		return AutoModelForQuestionAnswering.from_pretrained(*__magic_name__       ,       **__magic_name__	)

