
"""simple docstring"""


import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE      =     get_tests_dir("""fixtures/test_sentencepiece_with_bytefallback.model""")



@require_sentencepiece
@require_tokenizers
class   __a  (      _lowerCAmelCase    ,     unittest.TestCase	):
       UpperCamelCase_		:					Union[str, Any]   =			GPTSwaTokenizer
       UpperCamelCase_		:					Union[str, Any]   =			False
       UpperCamelCase_		:					int   =			True
       UpperCamelCase_		:					int   =			False






       def 	_SCREAMING_SNAKE_CASE       (  self :							List[Any]			)->  Any:







              """simple docstring"""
              super().setUp()

              # We have a SentencePiece fixture for testing
              UpperCamelCase					 =		GPTSwaTokenizer(UpperCAmelCase_       ,      eos_token="<unk>"       ,      bos_token="<unk>"       ,      pad_token="<unk>"			)

              tokenizer.save_pretrained(self.tmpdirname			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							Union[str, Any]       ,      UpperCAmelCase_ :							Any			)->  Dict:







              """simple docstring"""
              UpperCamelCase					 =		"This is a test"
              UpperCamelCase					 =		"This is a test"
              return input_text, output_text






       def 	_SCREAMING_SNAKE_CASE       (  self :							int			)->  Union[str, Any]:







              """simple docstring"""
              UpperCamelCase					 =		"<s>"
              UpperCamelCase					 =		1

              self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase_			)       ,      UpperCAmelCase_			)
              self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase_			)       ,      UpperCAmelCase_			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							Dict			)->  Dict:







              """simple docstring"""
              UpperCamelCase					 =		list(self.get_tokenizer().get_vocab().keys()			)

              self.assertEqual(vocab_keys[0]       ,      "<unk>"			)
              self.assertEqual(vocab_keys[1]       ,      "<s>"			)
              self.assertEqual(vocab_keys[-1]       ,      "j"			)
              self.assertEqual(len(UpperCAmelCase_			)       ,      2_000			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							Optional[Any]			)->  str:







              """simple docstring"""
              self.assertEqual(self.get_tokenizer().vocab_size       ,      2_000			)






       def 	_SCREAMING_SNAKE_CASE       (  self :							Any			)->  int:







              """simple docstring"""
              UpperCamelCase					 =		GPTSwaTokenizer(UpperCAmelCase_			)

              UpperCamelCase					 =		tokenizer.tokenize("This is a test"			)
              self.assertListEqual(UpperCAmelCase_       ,      ["▁This", "▁is", "▁a", "▁t", "est"]			)

              self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_			)       ,      [465, 287, 265, 631, 842]			)

              UpperCamelCase					 =		tokenizer.tokenize("I was born in 92000, and this is falsé."			)
              # fmt: off
              self.assertListEqual(
                  UpperCAmelCase_       ,      ["▁I", "▁was", "▁bor", "n", "▁in", "▁", "<0x39>", "2", "0", "0", "0", ",", "▁and", "▁this", "▁is", "▁f", "al", "s", "<0xC3>", "<0xA9>", "."]       ,      )
              # fmt: on

              UpperCamelCase					 =		tokenizer.convert_tokens_to_ids(UpperCAmelCase_			)
              self.assertListEqual(
                  UpperCAmelCase_       ,      [262, 272, 1_525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]       ,      )

              UpperCamelCase					 =		tokenizer.convert_ids_to_tokens(UpperCAmelCase_			)
              # fmt: off
              self.assertListEqual(
                  UpperCAmelCase_       ,      ["▁I", "▁was", "▁bor", "n", "▁in", "▁", "<0x39>", "2", "0", "0", "0", ",", "▁and", "▁this", "▁is", "▁f", "al", "s", "<0xC3>", "<0xA9>", "."]			)
              # fmt: on






       def 	_SCREAMING_SNAKE_CASE       (  self :							Union[str, Any]			)->  int:







              """simple docstring"""
              UpperCamelCase					 =		GPTSwaTokenizer(UpperCAmelCase_			)
              UpperCamelCase					 =		["This is a test", "I was born in 92000, and this is falsé."]
              UpperCamelCase					 =		[
                  [465, 287, 265, 631, 842],
                  [262, 272, 1_525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
              ]

              # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
              for text, expected_ids in zip(UpperCAmelCase_       ,      UpperCAmelCase_			):
                     self.assertListEqual(tokenizer.encode_fast(UpperCAmelCase_			)       ,      UpperCAmelCase_			)

              # Test that decode_fast returns the input text
              for text, token_ids in zip(UpperCAmelCase_       ,      UpperCAmelCase_			):
                     self.assertEqual(tokenizer.decode_fast(UpperCAmelCase_			)       ,      UpperCAmelCase_			)






       @slow
       def 	_SCREAMING_SNAKE_CASE       (  self :							Optional[Any]			)->  Dict:







              """simple docstring"""
              UpperCamelCase					 =		[
                  "<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')",
                  "Hey there, how are you doing this fine day?",
                  "This is a text with a trailing spaces followed by a dot     .",
                  "Häj sväjs lillebrör! =)",
                  "Det är inget fel på Mr. Cool",
              ]

              # fmt: off
              UpperCamelCase					 =		{"input_ids": [[63_423, 5, 6_811, 14_954, 282, 816, 3_821, 63_466, 63_425, 63_462, 18, 63_978, 678, 301, 1_320, 63_423, 63_455, 63_458, 18, 63_982, 4_246, 3_940, 1_901, 47_789, 5_547, 18_994], [19_630, 1_100, 63_446, 1_342, 633, 544, 4_488, 593, 5_102, 2_416, 63_495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_652, 428, 268, 1_936, 515, 268, 58_593, 22_413, 9_106, 546, 268, 33_213, 63_979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [55_130, 63_450, 924, 63_449, 2_249, 4_062, 1_558, 318, 63_504, 21_498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2_827, 2_559, 332, 6_575, 63_443, 26_801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
              # fmt: on
              self.tokenizer_integration_test_util(
                  expected_encoding=UpperCAmelCase_       ,      model_name="AI-Sweden/gpt-sw3-126m"       ,      sequences=UpperCAmelCase_       ,      )







"""simple docstring"""


from dataclasses import dataclass
from typing import List, Optional, Union

import numpy as np
import PIL
from PIL import Image

from ...utils import (
    BaseOutput,
    OptionalDependencyNotAvailable,
    is_flax_available,
    is_k_diffusion_available,
    is_k_diffusion_version,
    is_onnx_available,
    is_torch_available,
    is_transformers_available,
    is_transformers_version,
)



@dataclass
class   __a  (      _lowerCAmelCase	):
       UpperCamelCase_		:					Union[List[PIL.Image.Image], np.ndarray]
       UpperCamelCase_		:					Optional[List[bool]]


try:
      if not (is_transformers_available() and is_torch_available()):
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      from ...utils.dummy_torch_and_transformers_objects import *  # noqa F403
else:
      from .pipeline_cycle_diffusion import CycleDiffusionPipeline
      from .pipeline_stable_diffusion import StableDiffusionPipeline
      from .pipeline_stable_diffusion_attend_and_excite import StableDiffusionAttendAndExcitePipeline
      from .pipeline_stable_diffusion_imgaimg import StableDiffusionImgaImgPipeline
      from .pipeline_stable_diffusion_inpaint import StableDiffusionInpaintPipeline
      from .pipeline_stable_diffusion_inpaint_legacy import StableDiffusionInpaintPipelineLegacy
      from .pipeline_stable_diffusion_instruct_pixapix import StableDiffusionInstructPixaPixPipeline
      from .pipeline_stable_diffusion_latent_upscale import StableDiffusionLatentUpscalePipeline
      from .pipeline_stable_diffusion_ldmad import StableDiffusionLDMaDPipeline
      from .pipeline_stable_diffusion_model_editing import StableDiffusionModelEditingPipeline
      from .pipeline_stable_diffusion_panorama import StableDiffusionPanoramaPipeline
      from .pipeline_stable_diffusion_paradigms import StableDiffusionParadigmsPipeline
      from .pipeline_stable_diffusion_sag import StableDiffusionSAGPipeline
      from .pipeline_stable_diffusion_upscale import StableDiffusionUpscalePipeline
      from .pipeline_stable_unclip import StableUnCLIPPipeline
      from .pipeline_stable_unclip_imgaimg import StableUnCLIPImgaImgPipeline
      from .safety_checker import StableDiffusionSafetyChecker
      from .stable_unclip_image_normalizer import StableUnCLIPImageNormalizer

try:
      if not (is_transformers_available() and is_torch_available() and is_transformers_version(""">=""", """4.25.0""")):
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      from ...utils.dummy_torch_and_transformers_objects import StableDiffusionImageVariationPipeline
else:
      from .pipeline_stable_diffusion_image_variation import StableDiffusionImageVariationPipeline


try:
      if not (is_transformers_available() and is_torch_available() and is_transformers_version(""">=""", """4.26.0""")):
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      from ...utils.dummy_torch_and_transformers_objects import (
          StableDiffusionDepthaImgPipeline,
          StableDiffusionDiffEditPipeline,
          StableDiffusionPixaPixZeroPipeline,
      )
else:
      from .pipeline_stable_diffusion_depthaimg import StableDiffusionDepthaImgPipeline
      from .pipeline_stable_diffusion_diffedit import StableDiffusionDiffEditPipeline
      from .pipeline_stable_diffusion_pixapix_zero import StableDiffusionPixaPixZeroPipeline


try:
      if not (
          is_torch_available()
          and is_transformers_available()
          and is_k_diffusion_available()
          and is_k_diffusion_version(""">=""", """0.0.12""")
      ):
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      from ...utils.dummy_torch_and_transformers_and_k_diffusion_objects import *  # noqa F403
else:
      from .pipeline_stable_diffusion_k_diffusion import StableDiffusionKDiffusionPipeline

try:
      if not (is_transformers_available() and is_onnx_available()):
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      from ...utils.dummy_onnx_objects import *  # noqa F403
else:
      from .pipeline_onnx_stable_diffusion import OnnxStableDiffusionPipeline, StableDiffusionOnnxPipeline
      from .pipeline_onnx_stable_diffusion_imgaimg import OnnxStableDiffusionImgaImgPipeline
      from .pipeline_onnx_stable_diffusion_inpaint import OnnxStableDiffusionInpaintPipeline
      from .pipeline_onnx_stable_diffusion_inpaint_legacy import OnnxStableDiffusionInpaintPipelineLegacy
      from .pipeline_onnx_stable_diffusion_upscale import OnnxStableDiffusionUpscalePipeline

if is_transformers_available() and is_flax_available():
      import flax



      @flax.struct.dataclass
      class   __a  (      _lowerCAmelCase	):
             UpperCamelCase_		:					np.ndarray
             UpperCamelCase_		:					List[bool]





      from ...schedulers.scheduling_pndm_flax import PNDMSchedulerState
      from .pipeline_flax_stable_diffusion import FlaxStableDiffusionPipeline
      from .pipeline_flax_stable_diffusion_imgaimg import FlaxStableDiffusionImgaImgPipeline
      from .pipeline_flax_stable_diffusion_inpaint import FlaxStableDiffusionInpaintPipeline
      from .safety_checker_flax import FlaxStableDiffusionSafetyChecker





