"""simple docstring"""


import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
      from test_module.custom_tokenization_fast import CustomTokenizerFast


class        a				(		unittest.TestCase   ):


    def      lowerCAmelCase_  (					self	:  List[str]					):
      # A mock response for an HTTP head request to emulate server down
      _UpperCAmelCase						 =      mock.Mock()
      _UpperCAmelCase						 =      500
      _UpperCAmelCase						 =      {}
      _UpperCAmelCase						 =      HTTPError
      _UpperCAmelCase						 =      {}

      # Download this model to make sure it's in the cache.
      _UpperCAmelCase						 =      BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""					)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch("""requests.Session.request"""   ,				return_value=_lowercase					) as mock_head:
        _UpperCAmelCase						 =      BertTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""					)
        # This check we did call the fake head request
        mock_head.assert_called()


    @require_tokenizers
    def      lowerCAmelCase_  (					self	:  List[str]					):
      # A mock response for an HTTP head request to emulate server down
      _UpperCAmelCase						 =      mock.Mock()
      _UpperCAmelCase						 =      500
      _UpperCAmelCase						 =      {}
      _UpperCAmelCase						 =      HTTPError
      _UpperCAmelCase						 =      {}

      # Download this model to make sure it's in the cache.
      _UpperCAmelCase						 =      GPTaTokenizerFast.from_pretrained("""gpt2"""					)

      # Under the mock environment we get a 500 error when trying to reach the tokenizer.
      with mock.patch("""requests.Session.request"""   ,				return_value=_lowercase					) as mock_head:
        _UpperCAmelCase						 =      GPTaTokenizerFast.from_pretrained("""gpt2"""					)
        # This check we did call the fake head request
        mock_head.assert_called()


    def      lowerCAmelCase_  (					self	:  str					):
      # This test is for deprecated behavior and can be removed in v5
      try:
        _UpperCAmelCase						 =      tempfile.mktemp()
        with open(_lowercase   ,				"""wb"""					) as f:
          http_get("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""   ,				_lowercase					)

        _UpperCAmelCase						 =      AlbertTokenizer.from_pretrained(_lowercase					)
      finally:
        os.remove(_lowercase					)

      # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
      # the current folder and have the right name.
      if os.path.isfile("""tokenizer.json"""					):
        # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
        return
      try:
        with open("""tokenizer.json"""   ,				"""wb"""					) as f:
          http_get("""https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"""   ,				_lowercase					)
        _UpperCAmelCase						 =      AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""					)
        # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
        self.assertEqual(tokenizer.vocab_size   ,				1000					)
        # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

      finally:
        os.remove("""tokenizer.json"""					)


    def      lowerCAmelCase_  (					self	:  Dict					):
      # This test is for deprecated behavior and can be removed in v5
      _UpperCAmelCase						 =      AlbertTokenizer.from_pretrained("""https://huggingface.co/albert-base-v1/resolve/main/spiece.model"""					)



@is_staging_test
class        a				(		unittest.TestCase   ):
    _snake_case     :  str     							=							['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']


    @classmethod
    def      lowerCAmelCase_  (					cls	:  Union[str, Any]					):
      _UpperCAmelCase						 =      TOKEN
      HfFolder.save_token(_lowercase					)


    @classmethod
    def      lowerCAmelCase_  (					cls	:  List[Any]					):
      try:
        delete_repo(token=cls._token   ,				repo_id="""test-tokenizer"""					)
      except HTTPError:
        pass

      try:
        delete_repo(token=cls._token   ,				repo_id="""valid_org/test-tokenizer-org"""					)
      except HTTPError:
        pass

      try:
        delete_repo(token=cls._token   ,				repo_id="""test-dynamic-tokenizer"""					)
      except HTTPError:
        pass


    def      lowerCAmelCase_  (					self	:  Any					):
      with tempfile.TemporaryDirectory() as tmp_dir:
        _UpperCAmelCase						 =      os.path.join(_lowercase   ,				"""vocab.txt"""					)
        with open(_lowercase   ,				"""w"""   ,				encoding="""utf-8"""					) as vocab_writer:
          vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]					)					)
        _UpperCAmelCase						 =      BertTokenizer(_lowercase					)

      tokenizer.push_to_hub("""test-tokenizer"""   ,				use_auth_token=self._token					)
      _UpperCAmelCase						 =      BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''					)
      self.assertDictEqual(new_tokenizer.vocab   ,				tokenizer.vocab					)

      # Reset repo
      delete_repo(token=self._token   ,				repo_id="""test-tokenizer"""					)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(_lowercase   ,				repo_id="""test-tokenizer"""   ,				push_to_hub=_lowercase   ,				use_auth_token=self._token					)

      _UpperCAmelCase						 =      BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''					)
      self.assertDictEqual(new_tokenizer.vocab   ,				tokenizer.vocab					)


    def      lowerCAmelCase_  (					self	:  List[Any]					):
      with tempfile.TemporaryDirectory() as tmp_dir:
        _UpperCAmelCase						 =      os.path.join(_lowercase   ,				"""vocab.txt"""					)
        with open(_lowercase   ,				"""w"""   ,				encoding="""utf-8"""					) as vocab_writer:
          vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]					)					)
        _UpperCAmelCase						 =      BertTokenizer(_lowercase					)

      tokenizer.push_to_hub("""valid_org/test-tokenizer-org"""   ,				use_auth_token=self._token					)
      _UpperCAmelCase						 =      BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""					)
      self.assertDictEqual(new_tokenizer.vocab   ,				tokenizer.vocab					)

      # Reset repo
      delete_repo(token=self._token   ,				repo_id="""valid_org/test-tokenizer-org"""					)

      # Push to hub via save_pretrained
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(
            _lowercase   ,				repo_id="""valid_org/test-tokenizer-org"""   ,				push_to_hub=_lowercase   ,				use_auth_token=self._token					)

      _UpperCAmelCase						 =      BertTokenizer.from_pretrained("""valid_org/test-tokenizer-org"""					)
      self.assertDictEqual(new_tokenizer.vocab   ,				tokenizer.vocab					)


    @require_tokenizers
    def      lowerCAmelCase_  (					self	:  int					):
      CustomTokenizer.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
        _UpperCAmelCase						 =      os.path.join(_lowercase   ,				"""vocab.txt"""					)
        with open(_lowercase   ,				"""w"""   ,				encoding="""utf-8"""					) as vocab_writer:
          vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]					)					)
        _UpperCAmelCase						 =      CustomTokenizer(_lowercase					)

      # No fast custom tokenizer
      tokenizer.push_to_hub("""test-dynamic-tokenizer"""   ,				use_auth_token=self._token					)

      _UpperCAmelCase						 =      AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''   ,				trust_remote_code=_lowercase					)
      # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__   ,				"""CustomTokenizer"""					)

      # Fast and slow custom tokenizer
      CustomTokenizerFast.register_for_auto_class()
      with tempfile.TemporaryDirectory() as tmp_dir:
        _UpperCAmelCase						 =      os.path.join(_lowercase   ,				"""vocab.txt"""					)
        with open(_lowercase   ,				"""w"""   ,				encoding="""utf-8"""					) as vocab_writer:
          vocab_writer.write("""""".join([x + """\n""" for x in self.vocab_tokens]					)					)

        _UpperCAmelCase						 =      BertTokenizerFast.from_pretrained(_lowercase					)
        bert_tokenizer.save_pretrained(_lowercase					)
        _UpperCAmelCase						 =      CustomTokenizerFast.from_pretrained(_lowercase					)

      tokenizer.push_to_hub("""test-dynamic-tokenizer"""   ,				use_auth_token=self._token					)

      _UpperCAmelCase						 =      AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''   ,				trust_remote_code=_lowercase					)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__   ,				"""CustomTokenizerFast"""					)
      _UpperCAmelCase						 =      AutoTokenizer.from_pretrained(
          f'''{USER}/test-dynamic-tokenizer'''   ,				use_fast=_lowercase   ,				trust_remote_code=_lowercase					)
      # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
      self.assertEqual(tokenizer.__class__.__name__   ,				"""CustomTokenizer"""					)



class        a				(		unittest.TestCase   ):


    def      lowerCAmelCase_  (					self	:  Union[str, Any]					):
      _UpperCAmelCase						 =      Trie()
      trie.add("""Hello 友達"""					)
      self.assertEqual(trie.data   ,				{"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {""" """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}					)
      trie.add("""Hello"""					)
      trie.data
      self.assertEqual(trie.data   ,				{"""H""": {"""e""": {"""l""": {"""l""": {"""o""": {"""""": 1, """ """: {"""友""": {"""達""": {"""""": 1}}}}}}}}}					)


    def      lowerCAmelCase_  (					self	:  str					):
      _UpperCAmelCase						 =      Trie()
      self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""					)   ,				["""[CLS] This is a extra_id_100"""]					)
      trie.add("""[CLS]"""					)
      trie.add("""extra_id_1"""					)
      trie.add("""extra_id_100"""					)
      self.assertEqual(trie.split("""[CLS] This is a extra_id_100"""					)   ,				["""[CLS]""", """ This is a """, """extra_id_100"""]					)


    def      lowerCAmelCase_  (					self	:  Dict					):
      _UpperCAmelCase						 =      Trie()
      trie.add("""A"""					)
      self.assertEqual(trie.split("""ABC"""					)   ,				["""A""", """BC"""]					)
      self.assertEqual(trie.split("""BCA"""					)   ,				["""BC""", """A"""]					)


    def      lowerCAmelCase_  (					self	:  List[str]					):
      _UpperCAmelCase						 =      Trie()
      trie.add("""TOKEN]"""					)
      trie.add("""[SPECIAL_TOKEN]"""					)
      self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""					)   ,				["""This is something """, """[SPECIAL_TOKEN]"""]					)


    def      lowerCAmelCase_  (					self	:  Dict					):
      _UpperCAmelCase						 =      Trie()
      trie.add("""A"""					)
      trie.add("""P"""					)
      trie.add("""[SPECIAL_TOKEN]"""					)
      self.assertEqual(trie.split("""This is something [SPECIAL_TOKEN]"""					)   ,				["""This is something """, """[SPECIAL_TOKEN]"""]					)


    def      lowerCAmelCase_  (					self	:  List[str]					):
      _UpperCAmelCase						 =      Trie()
      trie.add("""AB"""					)
      trie.add("""B"""					)
      trie.add("""C"""					)
      self.assertEqual(trie.split("""ABC"""					)   ,				["""AB""", """C"""]					)


    def      lowerCAmelCase_  (					self	:  List[Any]					):
      _UpperCAmelCase						 =      Trie()
      trie.add("""ABC"""					)
      trie.add("""B"""					)
      trie.add("""CD"""					)
      self.assertEqual(trie.split("""ABCD"""					)   ,				["""ABC""", """D"""]					)


    def      lowerCAmelCase_  (					self	:  Union[str, Any]					):
      # Even if the offsets are wrong, we necessarily output correct string
      # parts.
      _UpperCAmelCase						 =      Trie()
      _UpperCAmelCase						 =      trie.cut_text("""ABC"""   ,				[0, 0, 2, 1, 2, 3]					)
      self.assertEqual(_lowercase   ,				["""AB""", """C"""]					)

"""simple docstring"""


import gc
import unittest

import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    DDPMScheduler,
    PriorTransformer,
    StableUnCLIPPipeline,
    UNetaDConditionModel,
)
from diffusers.pipelines.stable_diffusion.stable_unclip_image_normalizer import StableUnCLIPImageNormalizer
from diffusers.utils.testing_utils import enable_full_determinism, load_numpy, require_torch_gpu, slow, torch_device

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import (
    PipelineKarrasSchedulerTesterMixin,
    PipelineLatentTesterMixin,
    PipelineTesterMixin,
    assert_mean_pixel_difference,
)


enable_full_determinism()


class        a				(		lowerCAmelCase_	,							lowerCAmelCase_	,							lowerCAmelCase_	,							unittest.TestCase   ):
    _snake_case     :  Optional[int]     							=							StableUnCLIPPipeline
    _snake_case     :  List[str]     							=							TEXT_TO_IMAGE_PARAMS
    _snake_case     :  Tuple     							=							TEXT_TO_IMAGE_BATCH_PARAMS
    _snake_case     :  List[Any]     							=							TEXT_TO_IMAGE_IMAGE_PARAMS
    _snake_case     :  str     							=							TEXT_TO_IMAGE_IMAGE_PARAMS

    # TODO(will) Expected attn_bias.stride(1) == 0 to be true, but got false
    _snake_case     :  str     							=							False


    def      lowerCAmelCase_  (					self	:  Dict					):
      _UpperCAmelCase						 =      32
      _UpperCAmelCase						 =      embedder_hidden_size

      # prior components

      torch.manual_seed(0					)
      _UpperCAmelCase						 =      CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""					)

      torch.manual_seed(0					)
      _UpperCAmelCase						 =      CLIPTextModelWithProjection(
          CLIPTextConfig(
              bos_token_id=0   ,				eos_token_id=2   ,				hidden_size=__lowerCAmelCase   ,				projection_dim=__lowerCAmelCase   ,				intermediate_size=37   ,				layer_norm_eps=1e-0_5   ,				num_attention_heads=4   ,				num_hidden_layers=5   ,				pad_token_id=1   ,				vocab_size=1000   ,				)					)

      torch.manual_seed(0					)
      _UpperCAmelCase						 =      PriorTransformer(
          num_attention_heads=2   ,				attention_head_dim=12   ,				embedding_dim=__lowerCAmelCase   ,				num_layers=1   ,				)

      torch.manual_seed(0					)
      _UpperCAmelCase						 =      DDPMScheduler(
          variance_type="""fixed_small_log"""   ,				prediction_type="""sample"""   ,				num_train_timesteps=1000   ,				clip_sample=__lowerCAmelCase   ,				clip_sample_range=5.0   ,				beta_schedule="""squaredcos_cap_v2"""   ,				)

      # regular denoising components

      torch.manual_seed(0					)
      _UpperCAmelCase						 =      StableUnCLIPImageNormalizer(embedding_dim=__lowerCAmelCase					)
      _UpperCAmelCase						 =      DDPMScheduler(beta_schedule="""squaredcos_cap_v2"""					)

      torch.manual_seed(0					)
      _UpperCAmelCase						 =      CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""					)

      torch.manual_seed(0					)
      _UpperCAmelCase						 =      CLIPTextModel(
          CLIPTextConfig(
              bos_token_id=0   ,				eos_token_id=2   ,				hidden_size=__lowerCAmelCase   ,				projection_dim=32   ,				intermediate_size=37   ,				layer_norm_eps=1e-0_5   ,				num_attention_heads=4   ,				num_hidden_layers=5   ,				pad_token_id=1   ,				vocab_size=1000   ,				)					)

      torch.manual_seed(0					)
      _UpperCAmelCase						 =      UNetaDConditionModel(
          sample_size=32   ,				in_channels=4   ,				out_channels=4   ,				down_block_types=("""CrossAttnDownBlock2D""", """DownBlock2D""")   ,				up_block_types=("""UpBlock2D""", """CrossAttnUpBlock2D""")   ,				block_out_channels=(32, 64)   ,				attention_head_dim=(2, 4)   ,				class_embed_type="""projection"""   ,				projection_class_embeddings_input_dim=embedder_projection_dim * 2   ,				cross_attention_dim=__lowerCAmelCase   ,				layers_per_block=1   ,				upcast_attention=__lowerCAmelCase   ,				use_linear_projection=__lowerCAmelCase   ,				)

      torch.manual_seed(0					)
      _UpperCAmelCase						 =      DDIMScheduler(
          beta_schedule="""scaled_linear"""   ,				beta_start=0.00_085   ,				beta_end=0.012   ,				prediction_type="""v_prediction"""   ,				set_alpha_to_one=__lowerCAmelCase   ,				steps_offset=1   ,				)

      torch.manual_seed(0					)
      _UpperCAmelCase						 =      AutoencoderKL()

      _UpperCAmelCase						 =      {
          # prior components
          """prior_tokenizer""": prior_tokenizer,
          """prior_text_encoder""": prior_text_encoder,
          """prior""": prior,
          """prior_scheduler""": prior_scheduler,
          # image noising components
          """image_normalizer""": image_normalizer,
          """image_noising_scheduler""": image_noising_scheduler,
          # regular denoising components
          """tokenizer""": tokenizer,
          """text_encoder""": text_encoder,
          """unet""": unet,
          """scheduler""": scheduler,
          """vae""": vae,
      }

      return components


    def      lowerCAmelCase_  (					self	:  List[Any]   ,				__lowerCAmelCase	:  Any   ,				__lowerCAmelCase	:  int=0					):
      if str(__lowerCAmelCase					).startswith("""mps"""					):
        _UpperCAmelCase						 =      torch.manual_seed(__lowerCAmelCase					)
      else:
        _UpperCAmelCase						 =      torch.Generator(device=__lowerCAmelCase					).manual_seed(__lowerCAmelCase					)
      _UpperCAmelCase						 =      {
          """prompt""": """A painting of a squirrel eating a burger""",
          """generator""": generator,
          """num_inference_steps""": 2,
          """prior_num_inference_steps""": 2,
          """output_type""": """numpy""",
      }
      return inputs


    def      lowerCAmelCase_  (					self	:  List[str]					):
      _UpperCAmelCase						 =      torch_device == """cpu"""

      self._test_attention_slicing_forward_pass(test_max_difference=__lowerCAmelCase					)


    def      lowerCAmelCase_  (					self	:  Any					):
      _UpperCAmelCase						 =      torch_device in ["""cpu""", """mps"""]

      self._test_inference_batch_single_identical(test_max_difference=__lowerCAmelCase					)



@slow
@require_torch_gpu
class        a				(		unittest.TestCase   ):


    def      lowerCAmelCase_  (					self	:  List[str]					):
      # clean up the VRAM after each test
      super().tearDown()
      gc.collect()
      torch.cuda.empty_cache()


    def      lowerCAmelCase_  (					self	:  List[str]					):
      _UpperCAmelCase						 =      load_numpy(
          """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_unclip/stable_unclip_2_1_l_anime_turtle_fp16.npy"""					)

      _UpperCAmelCase						 =      StableUnCLIPPipeline.from_pretrained("""fusing/stable-unclip-2-1-l"""   ,				torch_dtype=torch.floataa					)
      pipe.to(__lowerCAmelCase					)
      pipe.set_progress_bar_config(disable=__lowerCAmelCase					)
      # stable unclip will oom when integration tests are run on a V100,
      # so turn on memory savings
      pipe.enable_attention_slicing()
      pipe.enable_sequential_cpu_offload()

      _UpperCAmelCase						 =      torch.Generator(device="""cpu"""					).manual_seed(0					)
      _UpperCAmelCase						 =      pipe("""anime turle"""   ,				generator=__lowerCAmelCase   ,				output_type="""np"""					)

      _UpperCAmelCase						 =      output.images[0]

      assert image.shape == (768, 768, 3)

      assert_mean_pixel_difference(__lowerCAmelCase   ,				__lowerCAmelCase					)


    def      lowerCAmelCase_  (					self	:  List[str]					):
      torch.cuda.empty_cache()
      torch.cuda.reset_max_memory_allocated()
      torch.cuda.reset_peak_memory_stats()

      _UpperCAmelCase						 =      StableUnCLIPPipeline.from_pretrained("""fusing/stable-unclip-2-1-l"""   ,				torch_dtype=torch.floataa					)
      _UpperCAmelCase						 =      pipe.to(__lowerCAmelCase					)
      pipe.set_progress_bar_config(disable=__lowerCAmelCase					)
      pipe.enable_attention_slicing()
      pipe.enable_sequential_cpu_offload()

      _UpperCAmelCase						 =      pipe(
          """anime turtle"""   ,				prior_num_inference_steps=2   ,				num_inference_steps=2   ,				output_type="""np"""   ,				)

      _UpperCAmelCase						 =      torch.cuda.max_memory_allocated()
      # make sure that less than 7 GB is allocated
      assert mem_bytes < 7 * 10**9
