






"""simple docstring"""







import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_	:    str            =      get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model''')





@require_sentencepiece
@require_tokenizers
class   UpperCamelCase_ (       a_   ,					unittest.TestCase  ):
  _A       :					Optional[int]				=	GPTSwaTokenizer
  _A       :					List[str]				=	False
  _A       :					Union[str, Any]				=	True
  _A       :					str				=	False



  def 		UpperCamelCase_				(      self				)			->			Optional[Any]:



         """simple docstring"""





         super().setUp()

         # We have a SentencePiece fixture for testing
         UpperCAmelCase	    =			GPTSwaTokenizer(snake_case__ ,		eos_token="""<unk>""" ,		bos_token="""<unk>""" ,		pad_token="""<unk>"""				)

         tokenizer.save_pretrained(self.tmpdirname				)



  def 		UpperCamelCase_				(      self ,		snake_case__				)			->			Any:



         """simple docstring"""





         UpperCAmelCase	    =			"""This is a test"""
         UpperCAmelCase	    =			"""This is a test"""
         return input_text, output_text



  def 		UpperCamelCase_				(      self				)			->			Dict:



         """simple docstring"""





         UpperCAmelCase	    =			"""<s>"""
         UpperCAmelCase	    =			1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__				) ,		snake_case__				)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__				) ,		snake_case__				)



  def 		UpperCamelCase_				(      self				)			->			Optional[Any]:



         """simple docstring"""





         UpperCAmelCase	    =			list(self.get_tokenizer().get_vocab().keys()				)

         self.assertEqual(vocab_keys[0] ,		"""<unk>"""				)
         self.assertEqual(vocab_keys[1] ,		"""<s>"""				)
         self.assertEqual(vocab_keys[-1] ,		"""j"""				)
         self.assertEqual(len(snake_case__				) ,		20_00				)



  def 		UpperCamelCase_				(      self				)			->			Union[str, Any]:



         """simple docstring"""





         self.assertEqual(self.get_tokenizer().vocab_size ,		20_00				)



  def 		UpperCamelCase_				(      self				)			->			int:



         """simple docstring"""





         UpperCAmelCase	    =			GPTSwaTokenizer(snake_case__				)

         UpperCAmelCase	    =			tokenizer.tokenize("""This is a test"""				)
         self.assertListEqual(snake_case__ ,		["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]				)

         self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__				) ,		[4_65, 2_87, 2_65, 6_31, 8_42]				)

         UpperCAmelCase	    =			tokenizer.tokenize("""I was born in 92000, and this is falsé."""				)
         # fmt: off
         self.assertListEqual(
             snake_case__ ,		["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""] ,		)
         # fmt: on

         UpperCAmelCase	    =			tokenizer.convert_tokens_to_ids(snake_case__				)
         self.assertListEqual(
             snake_case__ ,		[2_62, 2_72, 15_25, 2_86, 2_71, 2_68, 60, 9_16, 6_33, 6_33, 6_33, 2_59, 2_66, 3_01, 2_87, 3_84, 3_67, 2_63, 1_98, 1_72, 2_60] ,		)

         UpperCAmelCase	    =			tokenizer.convert_ids_to_tokens(snake_case__				)
         # fmt: off
         self.assertListEqual(
             snake_case__ ,		["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]				)
         # fmt: on



  def 		UpperCamelCase_				(      self				)			->			Dict:



         """simple docstring"""





         UpperCAmelCase	    =			GPTSwaTokenizer(snake_case__				)
         UpperCAmelCase	    =			["""This is a test""", """I was born in 92000, and this is falsé."""]
         UpperCAmelCase	    =			[
             [4_65, 2_87, 2_65, 6_31, 8_42],
             [2_62, 2_72, 15_25, 2_86, 2_71, 2_68, 60, 9_16, 6_33, 6_33, 6_33, 2_59, 2_66, 3_01, 2_87, 3_84, 3_67, 2_63, 1_98, 1_72, 2_60],
         ]

         # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
         for text, expected_ids in zip(snake_case__ ,		snake_case__				):
                self.assertListEqual(tokenizer.encode_fast(snake_case__				) ,		snake_case__				)

         # Test that decode_fast returns the input text
         for text, token_ids in zip(snake_case__ ,		snake_case__				):
                self.assertEqual(tokenizer.decode_fast(snake_case__				) ,		snake_case__				)



  @slow
  def 		UpperCamelCase_				(      self				)			->			Optional[int]:



         """simple docstring"""





         UpperCAmelCase	    =			[
             """<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')""",
             """Hey there, how are you doing this fine day?""",
             """This is a text with a trailing spaces followed by a dot     .""",
             """Häj sväjs lillebrör! =)""",
             """Det är inget fel på Mr. Cool""",
         ]

         # fmt: off
         UpperCAmelCase	    =			{"""input_ids""": [[6_34_23, 5, 68_11, 1_49_54, 2_82, 8_16, 38_21, 6_34_66, 6_34_25, 6_34_62, 18, 6_39_78, 6_78, 3_01, 13_20, 6_34_23, 6_34_55, 6_34_58, 18, 6_39_82, 42_46, 39_40, 19_01, 4_77_89, 55_47, 1_89_94], [1_96_30, 11_00, 6_34_46, 13_42, 6_33, 5_44, 44_88, 5_93, 51_02, 24_16, 6_34_95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [16_52, 4_28, 2_68, 19_36, 5_15, 2_68, 5_85_93, 2_24_13, 91_06, 5_46, 2_68, 3_32_13, 6_39_79, 6_98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_51_30, 6_34_50, 9_24, 6_34_49, 22_49, 40_62, 15_58, 3_18, 6_35_04, 2_14_98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_09, 3_77, 28_27, 25_59, 3_32, 65_75, 6_34_43, 2_68_01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
         # fmt: on
         self.tokenizer_integration_test_util(
             expected_encoding=snake_case__ ,		model_name="""AI-Sweden/gpt-sw3-126m""" ,		sequences=snake_case__ ,		)








"""simple docstring"""







from . import __version__

# Backward compatibility imports, to make sure all those objects can be found in file_utils
from .utils import (
    CLOUDFRONT_DISTRIB_PREFIX,
    CONFIG_NAME,
    DISABLE_TELEMETRY,
    DUMMY_INPUTS,
    DUMMY_MASK,
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    FEATURE_EXTRACTOR_NAME,
    FLAX_WEIGHTS_NAME,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_PREFIX,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    MODEL_CARD_NAME,
    MULTIPLE_CHOICE_DUMMY_INPUTS,
    PYTORCH_PRETRAINED_BERT_CACHE,
    PYTORCH_TRANSFORMERS_CACHE,
    S3_BUCKET_PREFIX,
    SENTENCEPIECE_UNDERLINE,
    SPIECE_UNDERLINE,
    TF2_WEIGHTS_NAME,
    TF_WEIGHTS_NAME,
    TORCH_FX_REQUIRED_VERSION,
    TRANSFORMERS_CACHE,
    TRANSFORMERS_DYNAMIC_MODULE_NAME,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    WEIGHTS_INDEX_NAME,
    WEIGHTS_NAME,
    ContextManagers,
    DummyObject,
    EntryNotFoundError,
    ExplicitEnum,
    ModelOutput,
    PaddingStrategy,
    PushToHubMixin,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    TensorType,
    _LazyModule,
    add_code_sample_docstrings,
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    cached_property,
    copy_func,
    default_cache_path,
    define_sagemaker_information,
    get_cached_models,
    get_file_from_repo,
    get_full_repo_name,
    get_torch_version,
    has_file,
    http_user_agent,
    is_apex_available,
    is_bsa_available,
    is_coloredlogs_available,
    is_datasets_available,
    is_detectrona_available,
    is_faiss_available,
    is_flax_available,
    is_ftfy_available,
    is_in_notebook,
    is_ipex_available,
    is_librosa_available,
    is_offline_mode,
    is_onnx_available,
    is_pandas_available,
    is_phonemizer_available,
    is_protobuf_available,
    is_psutil_available,
    is_pyanvml_available,
    is_pyctcdecode_available,
    is_pytesseract_available,
    is_pytorch_quantization_available,
    is_rjieba_available,
    is_sagemaker_dp_enabled,
    is_sagemaker_mp_enabled,
    is_scipy_available,
    is_sentencepiece_available,
    is_seqio_available,
    is_sklearn_available,
    is_soundfile_availble,
    is_spacy_available,
    is_speech_available,
    is_tensor,
    is_tensorflow_probability_available,
    is_tfaonnx_available,
    is_tf_available,
    is_timm_available,
    is_tokenizers_available,
    is_torch_available,
    is_torch_bfaa_available,
    is_torch_cuda_available,
    is_torch_fx_available,
    is_torch_fx_proxy,
    is_torch_mps_available,
    is_torch_tfaa_available,
    is_torch_tpu_available,
    is_torchaudio_available,
    is_training_run_on_sagemaker,
    is_vision_available,
    replace_return_docstrings,
    requires_backends,
    to_numpy,
    to_py_obj,
    torch_only_method,
)
