



import gc
import unittest

import numpy as np
import torch

from diffusers import AutoencoderKL, DDIMScheduler, DiTPipeline, DPMSolverMultistepScheduler, TransformeraDModel
from diffusers.utils import is_xformers_available, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import (
    CLASS_CONDITIONED_IMAGE_GENERATION_BATCH_PARAMS,
    CLASS_CONDITIONED_IMAGE_GENERATION_PARAMS,
)
from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()


class   __lowerCAmelCase  (		UpperCAmelCase_				,  unittest.TestCase   ):
					"""simple docstring"""






					A__					:		int      							=     DiTPipeline
					A__					:		Dict      							=     CLASS_CONDITIONED_IMAGE_GENERATION_PARAMS
					A__					:		Optional[Any]      							=     PipelineTesterMixin.required_optional_params - {
					    "latents",
					    "num_images_per_prompt",
					    "callback",
					    "callback_steps",
					}
					A__					:		Any      							=     CLASS_CONDITIONED_IMAGE_GENERATION_BATCH_PARAMS
					A__					:		List[str]      							=     False



					def      _a				(  self       :							Any       ):






								"""simple docstring"""


								torch.manual_seed(0       )
								A__	  =     TransformeraDModel(
								    sample_size=16					,       num_layers=2					,       patch_size=4					,       attention_head_dim=8					,       num_attention_heads=2					,       in_channels=4					,       out_channels=8					,       attention_bias=_snake_case					,       activation_fn='gelu-approximate'					,       num_embeds_ada_norm=10_00					,       norm_type='ada_norm_zero'					,       norm_elementwise_affine=_snake_case					,       )
								A__	  =     AutoencoderKL()
								A__	  =     DDIMScheduler()
								A__	  =     {'transformer': transformer.eval(), 'vae': vae.eval(), 'scheduler': scheduler}
								return components



					def      _a				(  self       :							Any					,       _snake_case       :							Optional[Any]					,       _snake_case       :							Tuple=0       ):






								"""simple docstring"""


								if str(_snake_case       ).startswith('mps'       ):
											A__	  =     torch.manual_seed(_snake_case       )
								else:
											A__	  =     torch.Generator(device=_snake_case       ).manual_seed(_snake_case       )
								A__	  =     {
								    'class_labels': [1],
								    'generator': generator,
								    'num_inference_steps': 2,
								    'output_type': 'numpy',
								}
								return inputs



					def      _a				(  self       :							Union[str, Any]       ):






								"""simple docstring"""


								A__	  =     'cpu'

								A__	  =     self.get_dummy_components()
								A__	  =     self.pipeline_class(**_snake_case       )
								pipe.to(_snake_case       )
								pipe.set_progress_bar_config(disable=_snake_case       )

								A__	  =     self.get_dummy_inputs(_snake_case       )
								A__	  =     pipe(**_snake_case       ).images
								A__	  =     image[0, -3:, -3:, -1]

								self.assertEqual(image.shape					,       (1, 16, 16, 3)       )
								A__	  =     np.array([0.2946, 0.6601, 0.4329, 0.3296, 0.4144, 0.5319, 0.7273, 0.5013, 0.4457]       )
								A__	  =     np.abs(image_slice.flatten() - expected_slice       ).max()
								self.assertLessEqual(_snake_case					,       1E-3       )



					def      _a				(  self       :							Optional[int]       ):






								"""simple docstring"""


								self._test_inference_batch_single_identical(relax_max_difference=_snake_case					,       expected_max_diff=1E-3       )



					@unittest.skipIf(
					    torch_device != 'cuda' or not is_xformers_available()					,       reason='XFormers attention is only available with CUDA and `xformers` installed'					,       )
					def      _a				(  self       :							Tuple       ):






								"""simple docstring"""


								self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1E-3       )







@require_torch_gpu
@slow
class   __lowerCAmelCase  (		unittest.TestCase   ):
					"""simple docstring"""






					def      _a				(  self       :							Optional[Any]       ):






								"""simple docstring"""


								super().tearDown()
								gc.collect()
								torch.cuda.empty_cache()



					def      _a				(  self       :							Optional[int]       ):






								"""simple docstring"""


								A__	  =     torch.manual_seed(0       )

								A__	  =     DiTPipeline.from_pretrained('facebook/DiT-XL-2-256'       )
								pipe.to('cuda'       )

								A__	  =     ['vase', 'umbrella', 'white shark', 'white wolf']
								A__	  =     pipe.get_label_ids(_snake_case       )

								A__	  =     pipe(_snake_case					,       generator=_snake_case					,       num_inference_steps=40					,       output_type='np'       ).images

								for word, image in zip(_snake_case					,       _snake_case       ):
											A__	  =     load_numpy(
											    F'''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/dit/{word}.npy'''       )
											assert np.abs((expected_image - image).max()       ) < 1E-2



					def      _a				(  self       :							Any       ):






								"""simple docstring"""


								A__	  =     DiTPipeline.from_pretrained('facebook/DiT-XL-2-512'       )
								A__	  =     DPMSolverMultistepScheduler.from_config(pipe.scheduler.config       )
								pipe.to('cuda'       )

								A__	  =     ['vase', 'umbrella']
								A__	  =     pipe.get_label_ids(_snake_case       )

								A__	  =     torch.manual_seed(0       )
								A__	  =     pipe(_snake_case					,       generator=_snake_case					,       num_inference_steps=25					,       output_type='np'       ).images

								for word, image in zip(_snake_case					,       _snake_case       ):
											A__	  =     load_numpy(
											    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
											    F'''/dit/{word}_512.npy'''       )

											assert np.abs((expected_image - image).max()       ) < 1E-1








'''simple docstring'''






import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class    __magic_name__				(						lowerCAmelCase   ,unittest.TestCase							):
       UpperCAmelCase		 =CodeGenTokenizer
       UpperCAmelCase		 =CodeGenTokenizerFast
       UpperCAmelCase		 =True
       UpperCAmelCase		 ={"add_prefix_space": True}
       UpperCAmelCase		 =False






       def      lowerCAmelCase						(     self)		->    Optional[int]:




          '''simple docstring'''
          super().setUp()

          # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
          _UpperCAmelCase  :  Optional[Any]				       =[
              'l',
              'o',
              'w',
              'e',
              'r',
              's',
              't',
              'i',
              'd',
              'n',
              '\u0120',
              '\u0120l',
              '\u0120n',
              '\u0120lo',
              '\u0120low',
              'er',
              '\u0120lowest',
              '\u0120newer',
              '\u0120wider',
              '<unk>',
              '<|endoftext|>',
          ]
          _UpperCAmelCase  :  Dict				       =dict(zip(snake_case    ,				range(len(snake_case))))
          _UpperCAmelCase  :  List[Any]				       =['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
          _UpperCAmelCase  :  List[Any]				       ={'unk_token': '<unk>'}

          _UpperCAmelCase  :  List[Any]				       =os.path.join(self.tmpdirname    ,				VOCAB_FILES_NAMES['vocab_file'])
          _UpperCAmelCase  :  List[str]				       =os.path.join(self.tmpdirname    ,				VOCAB_FILES_NAMES['merges_file'])
          with open(self.vocab_file    ,				'w'    ,				encoding='utf-8') as fp:
             fp.write(json.dumps(snake_case) + '\n')
          with open(self.merges_file    ,				'w'    ,				encoding='utf-8') as fp:
             fp.write('\n'.join(snake_case))






       def      lowerCAmelCase						(     self    ,				**snake_case)		->    str:




          '''simple docstring'''
          kwargs.update(self.special_tokens_map)
          return CodeGenTokenizer.from_pretrained(self.tmpdirname    ,				**snake_case)






       def      lowerCAmelCase						(     self    ,				**snake_case)		->    Optional[Any]:




          '''simple docstring'''
          kwargs.update(self.special_tokens_map)
          return CodeGenTokenizerFast.from_pretrained(self.tmpdirname    ,				**snake_case)






       def      lowerCAmelCase						(     self    ,				snake_case)		->    Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  Union[str, Any]				       ='lower newer'
          _UpperCAmelCase  :  int				       ='lower newer'
          return input_text, output_text






       def      lowerCAmelCase						(     self)		->    int:




          '''simple docstring'''
          _UpperCAmelCase  :  Optional[int]				       =CodeGenTokenizer(self.vocab_file    ,				self.merges_file    ,				**self.special_tokens_map)
          _UpperCAmelCase  :  str				       ='lower newer'
          _UpperCAmelCase  :  Union[str, Any]				       =['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
          _UpperCAmelCase  :  Any				       =tokenizer.tokenize(snake_case    ,				add_prefix_space=snake_case)
          self.assertListEqual(snake_case    ,				snake_case)

          _UpperCAmelCase  :  Any				       =tokens + [tokenizer.unk_token]
          _UpperCAmelCase  :  List[str]				       =[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case)    ,				snake_case)






       def      lowerCAmelCase						(     self)		->    Union[str, Any]:




          '''simple docstring'''
          if not self.test_rust_tokenizer:
             return

          _UpperCAmelCase  :  Tuple				       =self.get_tokenizer()
          _UpperCAmelCase  :  Dict				       =self.get_rust_tokenizer(add_prefix_space=snake_case)

          _UpperCAmelCase  :  Union[str, Any]				       ='lower newer'

          # Testing tokenization
          _UpperCAmelCase  :  List[str]				       =tokenizer.tokenize(snake_case    ,				add_prefix_space=snake_case)
          _UpperCAmelCase  :  Optional[int]				       =rust_tokenizer.tokenize(snake_case)
          self.assertListEqual(snake_case    ,				snake_case)

          # Testing conversion to ids without special tokens
          _UpperCAmelCase  :  str				       =tokenizer.encode(snake_case    ,				add_special_tokens=snake_case    ,				add_prefix_space=snake_case)
          _UpperCAmelCase  :  Optional[int]				       =rust_tokenizer.encode(snake_case    ,				add_special_tokens=snake_case)
          self.assertListEqual(snake_case    ,				snake_case)

          # Testing conversion to ids with special tokens
          _UpperCAmelCase  :  Dict				       =self.get_rust_tokenizer(add_prefix_space=snake_case)
          _UpperCAmelCase  :  Tuple				       =tokenizer.encode(snake_case    ,				add_prefix_space=snake_case)
          _UpperCAmelCase  :  int				       =rust_tokenizer.encode(snake_case)
          self.assertListEqual(snake_case    ,				snake_case)

          # Testing the unknown token
          _UpperCAmelCase  :  List[str]				       =tokens + [rust_tokenizer.unk_token]
          _UpperCAmelCase  :  Union[str, Any]				       =[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
          self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(snake_case)    ,				snake_case)






       def      lowerCAmelCase						(     self    ,				*snake_case    ,				**snake_case)		->    Any:




          '''simple docstring'''
          # It's very difficult to mix/test pretokenization with byte-level
          # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
          pass






       def      lowerCAmelCase						(     self    ,				snake_case=1_5)		->    Optional[int]:




          '''simple docstring'''
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                _UpperCAmelCase  :  Optional[Any]				       =self.rust_tokenizer_class.from_pretrained(snake_case    ,				**snake_case)

                # Simple input
                _UpperCAmelCase  :  List[str]				       ='This is a simple input'
                _UpperCAmelCase  :  Optional[Any]				       =['This is a simple input 1', 'This is a simple input 2']
                _UpperCAmelCase  :  int				       =('This is a simple input', 'This is a pair')
                _UpperCAmelCase  :  Union[str, Any]				       =[
                    ('This is a simple input 1', 'This is a simple input 2'),
                    ('This is a simple pair 1', 'This is a simple pair 2'),
                ]

                # Simple input tests
                self.assertRaises(snake_case    ,				tokenizer_r.encode    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length')

                # Simple input
                self.assertRaises(snake_case    ,				tokenizer_r.encode_plus    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length')

                # Simple input
                self.assertRaises(
                    snake_case    ,				tokenizer_r.batch_encode_plus    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length'    ,				)

                # Pair input
                self.assertRaises(snake_case    ,				tokenizer_r.encode    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length')

                # Pair input
                self.assertRaises(snake_case    ,				tokenizer_r.encode_plus    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length')

                # Pair input
                self.assertRaises(
                    snake_case    ,				tokenizer_r.batch_encode_plus    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length'    ,				)






       def      lowerCAmelCase						(     self)		->    Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  Any				       =CodeGenTokenizer.from_pretrained(self.tmpdirname    ,				pad_token='<pad>')

          # Simple input
          _UpperCAmelCase  :  List[Any]				       ='This is a simple input'
          _UpperCAmelCase  :  Any				       =['This is a simple input looooooooong', 'This is a simple input']
          _UpperCAmelCase  :  List[str]				       =('This is a simple input', 'This is a pair')
          _UpperCAmelCase  :  Optional[int]				       =[
              ('This is a simple input loooooong', 'This is a simple input'),
              ('This is a simple pair loooooong', 'This is a simple pair'),
          ]

          _UpperCAmelCase  :  List[Any]				       =tokenizer.pad_token_id

          _UpperCAmelCase  :  Union[str, Any]				       =tokenizer(snake_case    ,				padding='max_length'    ,				max_length=3_0    ,				return_tensors='np')
          _UpperCAmelCase  :  List[Any]				       =tokenizer(snake_case    ,				padding=snake_case    ,				truncate=snake_case    ,				return_tensors='np')
          _UpperCAmelCase  :  Optional[Any]				       =tokenizer(*snake_case    ,				padding='max_length'    ,				max_length=6_0    ,				return_tensors='np')
          _UpperCAmelCase  :  List[Any]				       =tokenizer(snake_case    ,				padding=snake_case    ,				truncate=snake_case    ,				return_tensors='np')

          # s
          # test single string max_length padding
          self.assertEqual(out_s['input_ids'].shape[-1]    ,				3_0)
          self.assertTrue(pad_token_id in out_s['input_ids'])
          self.assertTrue(0 in out_s['attention_mask'])

          # s2
          # test automatic padding
          self.assertEqual(out_sa['input_ids'].shape[-1]    ,				3_3)
          # long slice doesn't have padding
          self.assertFalse(pad_token_id in out_sa['input_ids'][0])
          self.assertFalse(0 in out_sa['attention_mask'][0])
          # short slice does have padding
          self.assertTrue(pad_token_id in out_sa['input_ids'][1])
          self.assertTrue(0 in out_sa['attention_mask'][1])

          # p
          # test single pair max_length padding
          self.assertEqual(out_p['input_ids'].shape[-1]    ,				6_0)
          self.assertTrue(pad_token_id in out_p['input_ids'])
          self.assertTrue(0 in out_p['attention_mask'])

          # p2
          # test automatic padding pair
          self.assertEqual(out_pa['input_ids'].shape[-1]    ,				5_2)
          # long slice pair doesn't have padding
          self.assertFalse(pad_token_id in out_pa['input_ids'][0])
          self.assertFalse(0 in out_pa['attention_mask'][0])
          # short slice pair does have padding
          self.assertTrue(pad_token_id in out_pa['input_ids'][1])
          self.assertTrue(0 in out_pa['attention_mask'][1])






       def      lowerCAmelCase						(     self)		->    Optional[int]:




          '''simple docstring'''
          _UpperCAmelCase  :  Any				       ='$$$'
          _UpperCAmelCase  :  Any				       =CodeGenTokenizer.from_pretrained(self.tmpdirname    ,				bos_token=snake_case    ,				add_bos_token=snake_case)

          _UpperCAmelCase  :  Optional[Any]				       ='This is a simple input'
          _UpperCAmelCase  :  Any				       =['This is a simple input 1', 'This is a simple input 2']

          _UpperCAmelCase  :  int				       =tokenizer.bos_token_id

          _UpperCAmelCase  :  Optional[Any]				       =tokenizer(snake_case)
          _UpperCAmelCase  :  Tuple				       =tokenizer(snake_case)

          self.assertEqual(out_s.input_ids[0]    ,				snake_case)
          self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids))

          _UpperCAmelCase  :  Optional[int]				       =tokenizer.decode(out_s.input_ids)
          _UpperCAmelCase  :  Any				       =tokenizer.batch_decode(out_sa.input_ids)

          self.assertEqual(decode_s.split()[0]    ,				snake_case)
          self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa))






       @slow
       def      lowerCAmelCase						(     self)		->    Optional[int]:




          '''simple docstring'''
          _UpperCAmelCase  :  Optional[Any]				       =CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono')

          _UpperCAmelCase  :  Optional[Any]				       ='\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
          _UpperCAmelCase  :  Tuple				       ='\nif len_a > len_b:      result = a\nelse:      result = b'

          _UpperCAmelCase  :  Optional[int]				       =tokenizer.encode(snake_case)
          _UpperCAmelCase  :  List[str]				       =['^#', re.escape('<|endoftext|>'), '^\'\'\'', '^"""', '\n\n\n']
          _UpperCAmelCase  :  Dict				       =tokenizer.decode(snake_case    ,				truncate_before_pattern=snake_case)
          self.assertEqual(snake_case    ,				snake_case)






       def      lowerCAmelCase						(     self)		->    Optional[Any]:




          '''simple docstring'''
          pass

