
"""simple docstring"""

import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
      UpperCamelCase     :    int       =			'pt'
elif is_tf_available():
      UpperCamelCase     :    Optional[Any]       =			'tf'
else:
      UpperCamelCase     :    List[Any]       =			'jax'

class __lowerCAmelCase		(				_lowerCamelCase      ,  unittest.TestCase      ):






     """simple docstring"""




     lowercase						       =						PerceiverTokenizer
     lowercase						       =						False





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      super().setUp()
      __UpperCamelCase      =	PerceiverTokenizer()
      tokenizer.save_pretrained(self.tmpdirname				)





     @cached_property
     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      return PerceiverTokenizer.from_pretrained('deepmind/language-perceiver'				)





     def 						UpperCAmelCase (     self     ,					**__UpperCAmelCase				):

      '''simple docstring'''




      return self.tokenizer_class.from_pretrained(self.tmpdirname     ,					**__UpperCAmelCase				)





     def 						UpperCAmelCase (     self     ,					__UpperCAmelCase     ,					__UpperCAmelCase=False     ,					__UpperCAmelCase=20     ,					__UpperCAmelCase=5				):

      '''simple docstring'''




      __UpperCamelCase      =	[]
      for i in range(len(__UpperCAmelCase				)				):
       try:
        __UpperCamelCase      =	tokenizer.decode([i]     ,					clean_up_tokenization_spaces=__UpperCAmelCase				)
       except UnicodeDecodeError:
        pass
       toks.append((i, tok)				)

      __UpperCamelCase      =	list(filter(lambda __UpperCAmelCase				: re.match(R'^[ a-zA-Z]+$'     ,					t[1]				)     ,					__UpperCAmelCase				)				)
      __UpperCamelCase      =	list(filter(lambda __UpperCAmelCase				: [t[0]] == tokenizer.encode(t[1]     ,					add_special_tokens=__UpperCAmelCase				)     ,					__UpperCAmelCase				)				)
      if max_length is not None and len(__UpperCAmelCase				) > max_length:
       __UpperCamelCase      =	toks[:max_length]
      if min_length is not None and len(__UpperCAmelCase				) < min_length and len(__UpperCAmelCase				) > 0:
       while len(__UpperCAmelCase				) < min_length:
        __UpperCamelCase      =	toks + toks
        # toks_str = [t[1] for t in toks]
      __UpperCamelCase      =	[t[0] for t in toks]

      # Ensure consistency
      __UpperCamelCase      =	tokenizer.decode(__UpperCAmelCase     ,					clean_up_tokenization_spaces=__UpperCAmelCase				)
      if " " not in output_txt and len(__UpperCAmelCase				) > 1:
       __UpperCamelCase      =	(
           tokenizer.decode([toks_ids[0]]     ,					clean_up_tokenization_spaces=__UpperCAmelCase				)
           + ' '
           + tokenizer.decode(toks_ids[1:]     ,					clean_up_tokenization_spaces=__UpperCAmelCase				)
       )
      if with_prefix_space:
       __UpperCamelCase      =	' ' + output_txt
      __UpperCamelCase      =	tokenizer.encode(__UpperCAmelCase     ,					add_special_tokens=__UpperCAmelCase				)
      return output_txt, output_ids





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      __UpperCamelCase      =	self.perceiver_tokenizer
      __UpperCamelCase      =	'Unicode €.'
      __UpperCamelCase      =	tokenizer(__UpperCAmelCase				)
      __UpperCamelCase      =	[4, 91, 116, 111, 105, 117, 106, 107, 38, 232, 136, 178, 52, 5]
      self.assertEqual(encoded['input_ids']     ,					__UpperCAmelCase				)

      # decoding
      __UpperCamelCase      =	tokenizer.decode(__UpperCAmelCase				)
      self.assertEqual(__UpperCAmelCase     ,					'[CLS]Unicode €.[SEP]'				)

      __UpperCamelCase      =	tokenizer('e è é ê ë'				)
      __UpperCamelCase      =	[4, 107, 38, 201, 174, 38, 201, 175, 38, 201, 176, 38, 201, 177, 5]
      self.assertEqual(encoded['input_ids']     ,					__UpperCAmelCase				)
      # decoding
      __UpperCamelCase      =	tokenizer.decode(__UpperCAmelCase				)
      self.assertEqual(__UpperCAmelCase     ,					'[CLS]e è é ê ë[SEP]'				)

      # encode/decode, but with `encode` instead of `__call__`
      self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'				)				)     ,					'[CLS]e è é ê ë[SEP]'				)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      __UpperCamelCase      =	self.perceiver_tokenizer
      __UpperCamelCase      =	['A long paragraph for summarization.', 'Another paragraph for summarization.']
      # fmt: off
      __UpperCamelCase      =	[4, 71, 38, 114, 117, 116, 109, 38, 118, 103, 120, 103, 109, 120, 103, 118, 110, 38, 108, 117, 120, 38, 121, 123, 115, 115, 103, 120, 111, 128, 103, 122, 111, 117, 116, 52, 5, 0]
      # fmt: on
      __UpperCamelCase      =	tokenizer(__UpperCAmelCase     ,					padding=__UpperCAmelCase     ,					return_tensors=__UpperCAmelCase				)
      self.assertIsInstance(__UpperCAmelCase     ,					__UpperCAmelCase				)

      if FRAMEWORK != "jax":
       __UpperCamelCase      =	list(batch.input_ids.numpy()[0]				)
      else:
       __UpperCamelCase      =	list(batch.input_ids.tolist()[0]				)

      self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase				)

      self.assertEqual((2, 38)     ,					batch.input_ids.shape				)
      self.assertEqual((2, 38)     ,					batch.attention_mask.shape				)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      __UpperCamelCase      =	self.perceiver_tokenizer
      __UpperCamelCase      =	['A long paragraph for summarization.', 'Another paragraph for summarization.']
      __UpperCamelCase      =	tokenizer(__UpperCAmelCase     ,					padding=__UpperCAmelCase     ,					return_tensors=__UpperCAmelCase				)
      # check if input_ids are returned and no decoder_input_ids
      self.assertIn('input_ids'     ,					__UpperCAmelCase				)
      self.assertIn('attention_mask'     ,					__UpperCAmelCase				)
      self.assertNotIn('decoder_input_ids'     ,					__UpperCAmelCase				)
      self.assertNotIn('decoder_attention_mask'     ,					__UpperCAmelCase				)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      __UpperCamelCase      =	self.perceiver_tokenizer
      __UpperCamelCase      =	[
          'Summary of the text.',
          'Another summary.',
      ]
      __UpperCamelCase      =	tokenizer(
          text_target=__UpperCAmelCase     ,					max_length=32     ,					padding='max_length'     ,					truncation=__UpperCAmelCase     ,					return_tensors=__UpperCAmelCase				)
      self.assertEqual(32     ,					targets['input_ids'].shape[1]				)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      __UpperCamelCase      =	self.get_tokenizers()
      for tokenizer in tokenizers:
       with self.subTest(F'{tokenizer.__class__.__name__}'				):
        self.assertNotEqual(tokenizer.model_max_length     ,					42				)

        # Now let's start the test
      __UpperCamelCase      =	self.get_tokenizers()
      for tokenizer in tokenizers:
       with self.subTest(F'{tokenizer.__class__.__name__}'				):
        # Isolate this from the other tests because we save additional tokens/etc
        __UpperCamelCase      =	tempfile.mkdtemp()

        __UpperCamelCase      =	' He is very happy, UNwant\u00E9d,running'
        __UpperCamelCase      =	tokenizer.encode(__UpperCAmelCase     ,					add_special_tokens=__UpperCAmelCase				)
        tokenizer.save_pretrained(__UpperCAmelCase				)

        __UpperCamelCase      =	tokenizer.__class__.from_pretrained(__UpperCAmelCase				)
        __UpperCamelCase      =	after_tokenizer.encode(__UpperCAmelCase     ,					add_special_tokens=__UpperCAmelCase				)
        self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase				)

        shutil.rmtree(__UpperCAmelCase				)

      __UpperCamelCase      =	self.get_tokenizers(model_max_length=42				)
      for tokenizer in tokenizers:
       with self.subTest(F'{tokenizer.__class__.__name__}'				):
        # Isolate this from the other tests because we save additional tokens/etc
        __UpperCamelCase      =	tempfile.mkdtemp()

        __UpperCamelCase      =	' He is very happy, UNwant\u00E9d,running'
        tokenizer.add_tokens(['bim', 'bambam']				)
        __UpperCamelCase      =	tokenizer.additional_special_tokens
        additional_special_tokens.append('new_additional_special_token'				)
        tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}				)
        __UpperCamelCase      =	tokenizer.encode(__UpperCAmelCase     ,					add_special_tokens=__UpperCAmelCase				)
        tokenizer.save_pretrained(__UpperCAmelCase				)

        __UpperCamelCase      =	tokenizer.__class__.from_pretrained(__UpperCAmelCase				)
        __UpperCamelCase      =	after_tokenizer.encode(__UpperCAmelCase     ,					add_special_tokens=__UpperCAmelCase				)
        self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase				)
        self.assertIn('new_additional_special_token'     ,					after_tokenizer.additional_special_tokens				)
        self.assertEqual(after_tokenizer.model_max_length     ,					42				)

        __UpperCamelCase      =	tokenizer.__class__.from_pretrained(__UpperCAmelCase     ,					model_max_length=43				)
        self.assertEqual(tokenizer.model_max_length     ,					43				)

        shutil.rmtree(__UpperCAmelCase				)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      __UpperCamelCase      =	[]
      if self.test_slow_tokenizer:
       tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())				)

      if self.test_rust_tokenizer:
       tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())				)

      for tokenizer_class, tokenizer_utils in tokenizer_list:
       with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer_utils.save_pretrained(__UpperCAmelCase				)

        with open(os.path.join(__UpperCAmelCase     ,					'special_tokens_map.json'				)     ,					encoding='utf-8'				) as json_file:
         __UpperCamelCase      =	json.load(__UpperCAmelCase				)

        with open(os.path.join(__UpperCAmelCase     ,					'tokenizer_config.json'				)     ,					encoding='utf-8'				) as json_file:
         __UpperCamelCase      =	json.load(__UpperCAmelCase				)

        __UpperCamelCase      =	[F'<extra_id_{i}>' for i in range(125				)]

        __UpperCamelCase      =	added_tokens_extra_ids + [
            'an_additional_special_token'
        ]
        __UpperCamelCase      =	added_tokens_extra_ids + [
            'an_additional_special_token'
        ]

        with open(os.path.join(__UpperCAmelCase     ,					'special_tokens_map.json'				)     ,					'w'     ,					encoding='utf-8'				) as outfile:
         json.dump(__UpperCAmelCase     ,					__UpperCAmelCase				)
        with open(os.path.join(__UpperCAmelCase     ,					'tokenizer_config.json'				)     ,					'w'     ,					encoding='utf-8'				) as outfile:
         json.dump(__UpperCAmelCase     ,					__UpperCAmelCase				)

        # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
        # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
        # "special_tokens_map.json" files
        __UpperCamelCase      =	tokenizer_class.from_pretrained(
            __UpperCAmelCase     ,					)
        self.assertIn(
            'an_additional_special_token'     ,					tokenizer_without_change_in_init.additional_special_tokens				)
        self.assertEqual(
            ['an_additional_special_token']     ,					tokenizer_without_change_in_init.convert_ids_to_tokens(
                tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']				)				)     ,					)

        # Now we test that we can change the value of additional_special_tokens in the from_pretrained
        __UpperCamelCase      =	added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'     ,					lstrip=__UpperCAmelCase				)]
        __UpperCamelCase      =	tokenizer_class.from_pretrained(
            __UpperCAmelCase     ,					additional_special_tokens=__UpperCAmelCase     ,					)

        self.assertIn('a_new_additional_special_token'     ,					tokenizer.additional_special_tokens				)
        self.assertEqual(
            ['a_new_additional_special_token']     ,					tokenizer.convert_ids_to_tokens(
                tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']				)				)     ,					)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      __UpperCamelCase      =	self.perceiver_tokenizer
      self.assertEqual(tokenizer.decode([178]				)     ,					'�'				)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      pass





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      pass





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      pass





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      pass







     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      __UpperCamelCase      =	self.get_tokenizers(fast=__UpperCAmelCase     ,					do_lower_case=__UpperCAmelCase				)
      for tokenizer in tokenizers:
       with self.subTest(F'{tokenizer.__class__.__name__}'				):
        __UpperCamelCase      =	['[CLS]', 't', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 's', 't', '[SEP]']
        __UpperCamelCase      =	tokenizer.convert_tokens_to_string(__UpperCAmelCase				)

        self.assertIsInstance(__UpperCAmelCase     ,					__UpperCAmelCase				)





"""simple docstring"""

import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    StableDiffusionAttendAndExcitePipeline,
    UNetaDConditionModel,
)
from diffusers.utils import load_numpy, skip_mps, slow
from diffusers.utils.testing_utils import require_torch_gpu

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin


UpperCamelCase     :    List[Any]       =			False

@skip_mps
class __lowerCAmelCase		(				__SCREAMING_SNAKE_CASE      ,  __SCREAMING_SNAKE_CASE      ,  __SCREAMING_SNAKE_CASE      ,  unittest.TestCase      ):
     lowercase						       =						StableDiffusionAttendAndExcitePipeline
     lowercase						       =						False
     lowercase						       =						TEXT_TO_IMAGE_PARAMS
     lowercase						       =						TEXT_TO_IMAGE_BATCH_PARAMS.union({"token_indices"}      )
     lowercase						       =						TEXT_TO_IMAGE_IMAGE_PARAMS
     lowercase						       =						TEXT_TO_IMAGE_IMAGE_PARAMS





     @classmethod
     def 						UpperCAmelCase (     cls				):

      '''simple docstring'''




      super().setUpClass()
      torch.use_deterministic_algorithms(__UpperCAmelCase				)





     @classmethod
     def 						UpperCAmelCase (     cls				):

      '''simple docstring'''




      super().tearDownClass()
      torch.use_deterministic_algorithms(__UpperCAmelCase				)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      torch.manual_seed(0				)
      __UpperCamelCase      =	UNetaDConditionModel(
          block_out_channels=(32, 64)     ,					layers_per_block=1     ,					sample_size=32     ,					in_channels=4     ,					out_channels=4     ,					down_block_types=('DownBlock2D', 'CrossAttnDownBlock2D')     ,					up_block_types=('CrossAttnUpBlock2D', 'UpBlock2D')     ,					cross_attention_dim=32     ,					attention_head_dim=(2, 4)     ,					use_linear_projection=__UpperCAmelCase     ,					)
      __UpperCamelCase      =	DDIMScheduler(
          beta_start=0.0_0_0_8_5     ,					beta_end=0.0_1_2     ,					beta_schedule='scaled_linear'     ,					clip_sample=__UpperCAmelCase     ,					set_alpha_to_one=__UpperCAmelCase     ,					)
      torch.manual_seed(0				)
      __UpperCamelCase      =	AutoencoderKL(
          block_out_channels=[32, 64]     ,					in_channels=3     ,					out_channels=3     ,					down_block_types=['DownEncoderBlock2D', 'DownEncoderBlock2D']     ,					up_block_types=['UpDecoderBlock2D', 'UpDecoderBlock2D']     ,					latent_channels=4     ,					sample_size=128     ,					)
      torch.manual_seed(0				)
      __UpperCamelCase      =	CLIPTextConfig(
          bos_token_id=0     ,					eos_token_id=2     ,					hidden_size=32     ,					intermediate_size=37     ,					layer_norm_eps=1E-05     ,					num_attention_heads=4     ,					num_hidden_layers=5     ,					pad_token_id=1     ,					vocab_size=1000     ,					hidden_act='gelu'     ,					projection_dim=512     ,					)
      __UpperCamelCase      =	CLIPTextModel(__UpperCAmelCase				)
      __UpperCamelCase      =	CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip'				)

      __UpperCamelCase      =	{
          'unet': unet,
          'scheduler': scheduler,
          'vae': vae,
          'text_encoder': text_encoder,
          'tokenizer': tokenizer,
          'safety_checker': None,
          'feature_extractor': None,
      }

      return components





     def 						UpperCAmelCase (     self     ,					__UpperCAmelCase     ,					__UpperCAmelCase=0				):

      '''simple docstring'''




      if str(__UpperCAmelCase				).startswith('mps'				):
       __UpperCamelCase      =	torch.manual_seed(__UpperCAmelCase				)
      else:
       __UpperCamelCase      =	torch.Generator(device=__UpperCAmelCase				).manual_seed(__UpperCAmelCase				)
      __UpperCamelCase      =	__UpperCamelCase      =	{
          'prompt': 'a cat and a frog',
          'token_indices': [2, 5],
          'generator': generator,
          'num_inference_steps': 1,
          'guidance_scale': 6.0,
          'output_type': 'numpy',
          'max_iter_to_alter': 2,
          'thresholds': {0: 0.7},
      }
      return inputs





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      __UpperCamelCase      =	'cpu'

      __UpperCamelCase      =	self.get_dummy_components()
      __UpperCamelCase      =	self.pipeline_class(**__UpperCAmelCase				)
      pipe.to(__UpperCAmelCase				)
      pipe.set_progress_bar_config(disable=__UpperCAmelCase				)

      __UpperCamelCase      =	self.get_dummy_inputs(__UpperCAmelCase				)
      __UpperCamelCase      =	pipe(**__UpperCAmelCase				).images
      __UpperCamelCase      =	image[0, -3:, -3:, -1]

      self.assertEqual(image.shape     ,					(1, 64, 64, 3)				)
      __UpperCamelCase      =	np.array(
          [0.6_3_9_0_5_3_6_4, 0.6_2_8_9_7_3_0_7, 0.4_8_5_9_9_0_1_7, 0.5_1_3_3_6_2_4, 0.5_5_5_0_0_4_8, 0.4_5_7_6_9_5_1_6, 0.5_0_3_2_6_9_7_3, 0.5_0_2_3_1_3_9, 0.4_5_3_8_4_4_9_6]				)
      __UpperCamelCase      =	np.abs(image_slice.flatten() - expected_slice				).max()
      self.assertLessEqual(__UpperCAmelCase     ,					1E-3				)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      super().test_cpu_offload_forward_pass(expected_max_diff=5E-4				)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      self._test_inference_batch_consistent(batch_sizes=[1, 2]				)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      self._test_inference_batch_single_identical(batch_size=2     ,					expected_max_diff=7E-4				)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      super().test_dict_tuple_outputs_equivalent(expected_max_difference=3E-3				)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      super().test_pt_np_pil_outputs_equivalent(expected_max_diff=5E-4				)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      super().test_save_load_local(expected_max_difference=5E-4				)







     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      super().test_save_load_optional_components(expected_max_difference=4E-4				)


@require_torch_gpu
@slow
class __lowerCAmelCase		(				unittest.TestCase      ):





     @classmethod
     def 						UpperCAmelCase (     cls				):

      '''simple docstring'''




      super().setUpClass()
      torch.use_deterministic_algorithms(__UpperCAmelCase				)





     @classmethod
     def 						UpperCAmelCase (     cls				):

      '''simple docstring'''




      super().tearDownClass()
      torch.use_deterministic_algorithms(__UpperCAmelCase				)





     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      super().tearDown()
      gc.collect()
      torch.cuda.empty_cache()







     def 						UpperCAmelCase (     self				):

      '''simple docstring'''




      __UpperCamelCase      =	torch.manual_seed(51				)

      __UpperCamelCase      =	StableDiffusionAttendAndExcitePipeline.from_pretrained(
          'CompVis/stable-diffusion-v1-4'     ,					safety_checker=__UpperCAmelCase     ,					torch_dtype=torch.floataa				)
      pipe.to('cuda'				)

      __UpperCamelCase      =	'a painting of an elephant with glasses'
      __UpperCamelCase      =	[5, 7]

      __UpperCamelCase      =	pipe(
          prompt=__UpperCAmelCase     ,					token_indices=__UpperCAmelCase     ,					guidance_scale=7.5     ,					generator=__UpperCAmelCase     ,					num_inference_steps=5     ,					max_iter_to_alter=5     ,					output_type='numpy'     ,					).images[0]

      __UpperCamelCase      =	load_numpy(
          'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/attend-and-excite/elephant_glasses.npy'				)
      assert np.abs((expected_image - image).max()				) < 5E-1



