




import argparse
import logging
import math
import time

import torch

from transformers import TransfoXLCorpus, TransfoXLLMHeadModel


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
_lowerCamelCase					:     Any					  =  logging.getLogger(__name__)




def 							_UpperCAmelCase     ():
				'''simple docstring'''
				_lowerCAmelCase			:			Any        =					argparse.ArgumentParser(description="""PyTorch Transformer Language Model"""				)
				parser.add_argument("""--model_name"""					,			type=UpperCamelCase_					,			default="""transfo-xl-wt103"""					,			help="""pretrained model name"""				)
				parser.add_argument(
				    """--split"""					,			type=UpperCamelCase_					,			default="""test"""					,			choices=["""all""", """valid""", """test"""]					,			help="""which split to evaluate"""				)
				parser.add_argument("""--batch_size"""					,			type=UpperCamelCase_					,			default=10					,			help="""batch size"""				)
				parser.add_argument("""--tgt_len"""					,			type=UpperCamelCase_					,			default=128					,			help="""number of tokens to predict"""				)
				parser.add_argument("""--ext_len"""					,			type=UpperCamelCase_					,			default=0					,			help="""length of the extended context"""				)
				parser.add_argument("""--mem_len"""					,			type=UpperCamelCase_					,			default=1600					,			help="""length of the retained previous heads"""				)
				parser.add_argument("""--clamp_len"""					,			type=UpperCamelCase_					,			default=1000					,			help="""max positional embedding index"""				)
				parser.add_argument("""--no_cuda"""					,			action="""store_true"""					,			help="""Do not use CUDA even though CUA is available"""				)
				parser.add_argument("""--work_dir"""					,			type=UpperCamelCase_					,			required=UpperCamelCase_					,			help="""path to the work_dir"""				)
				parser.add_argument("""--no_log"""					,			action="""store_true"""					,			help="""do not log the eval result"""				)
				parser.add_argument("""--same_length"""					,			action="""store_true"""					,			help="""set same length attention with masking"""				)
				parser.add_argument("""--server_ip"""					,			type=UpperCamelCase_					,			default=""""""					,			help="""Can be used for distant debugging."""				)
				parser.add_argument("""--server_port"""					,			type=UpperCamelCase_					,			default=""""""					,			help="""Can be used for distant debugging."""				)
				_lowerCAmelCase			:			Optional[Any]        =					parser.parse_args()
				assert args.ext_len >= 0, "extended context length must be non-negative"

				if args.server_ip and args.server_port:
								# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
								import ptvsd

								print("""Waiting for debugger attach"""				)
								ptvsd.enable_attach(address=(args.server_ip, args.server_port)					,			redirect_output=UpperCamelCase_				)
								ptvsd.wait_for_attach()

				_lowerCAmelCase			:			int        =					torch.device("""cuda""" if torch.cuda.is_available() and not args.no_cuda else """cpu"""				)
				logger.info("""device: {}""".format(UpperCamelCase_				)				)

				# Load a pre-processed dataset
				# You can also build the corpus yourself using TransfoXLCorpus methods
				# The pre-processing involve computing word frequencies to prepare the Adaptive input and SoftMax
				# and tokenizing the dataset
				# The pre-processed corpus is a convertion (using the conversion script )
				_lowerCAmelCase			:			str        =					TransfoXLCorpus.from_pretrained(args.model_name				)

				_lowerCAmelCase			:			List[str]        =					corpus.get_iterator("""valid"""					,			args.batch_size					,			args.tgt_len					,			device=UpperCamelCase_					,			ext_len=args.ext_len				)
				_lowerCAmelCase			:			int        =					corpus.get_iterator("""test"""					,			args.batch_size					,			args.tgt_len					,			device=UpperCamelCase_					,			ext_len=args.ext_len				)

				# Load a pre-trained model
				_lowerCAmelCase			:			List[str]        =					TransfoXLLMHeadModel.from_pretrained(args.model_name				)
				model.to(UpperCamelCase_				)

				logger.info(
				    """Evaluating with bsz {} tgt_len {} ext_len {} mem_len {} clamp_len {}""".format(
				        args.batch_size					,			args.tgt_len					,			args.ext_len					,			args.mem_len					,			args.clamp_len				)				)

				model.reset_memory_length(args.mem_len				)
				if args.clamp_len > 0:
								_lowerCAmelCase			:			Any        =					args.clamp_len
				if args.same_length:
								_lowerCAmelCase			:			int        =					True

				###############################################################################
				# Evaluation code
				###############################################################################
				def evaluate(UpperCamelCase_       :       Optional[int]				):
								# Turn on evaluation mode which disables dropout.
								model.eval()
								_lowerCAmelCase    ,			_lowerCAmelCase			:			Tuple        =					0, 0.0
								_lowerCAmelCase			:			str        =					time.time()
								with torch.no_grad():
												_lowerCAmelCase			:			Optional[int]        =					None
												for idx, (data, target, seq_len) in enumerate(UpperCamelCase_				):
																_lowerCAmelCase			:			Any        =					model(UpperCamelCase_					,			lm_labels=UpperCamelCase_					,			mems=UpperCamelCase_				)
																_lowerCAmelCase    ,			_lowerCAmelCase    ,			_lowerCAmelCase			:			Dict        =					ret
																_lowerCAmelCase			:			Optional[Any]        =					loss.mean()
																total_loss += seq_len * loss.item()
																total_len += seq_len
												_lowerCAmelCase			:			Optional[Any]        =					time.time() - start_time
								logger.info("""Time : {:.2f}s, {:.2f}ms/segment""".format(UpperCamelCase_					,			1000 * total_time / (idx + 1)				)				)
								return total_loss / total_len

				# Run on test data.
				if args.split == "all":
								_lowerCAmelCase			:			List[Any]        =					evaluate(UpperCamelCase_				)
								_lowerCAmelCase			:			Tuple        =					evaluate(UpperCamelCase_				)
				elif args.split == "valid":
								_lowerCAmelCase			:			Tuple        =					evaluate(UpperCamelCase_				)
								_lowerCAmelCase			:			Optional[int]        =					None
				elif args.split == "test":
								_lowerCAmelCase			:			Any        =					evaluate(UpperCamelCase_				)
								_lowerCAmelCase			:			List[Any]        =					None

				def format_log(UpperCamelCase_       :       Optional[Any]					,			UpperCamelCase_       :       Dict				):
								_lowerCAmelCase			:			Dict        =					"""| {0} loss {1:5.2f} | {0} ppl {2:9.3f} """.format(UpperCamelCase_					,			UpperCamelCase_					,			math.exp(UpperCamelCase_				)				)
								return log_str

				_lowerCAmelCase			:			int        =					""""""
				if valid_loss is not None:
								log_str += format_log(UpperCamelCase_					,			"""valid"""				)
				if test_loss is not None:
								log_str += format_log(UpperCamelCase_					,			"""test"""				)

				logger.info("""=""" * 100				)
				logger.info(UpperCamelCase_				)
				logger.info("""=""" * 100				)


if __name__ == "__main__":
	main()








'''simple docstring'''



import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    EulerAncestralDiscreteScheduler,
    PNDMScheduler,
    StableDiffusionModelEditingPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, skip_mps

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()






@skip_mps
class 							__lowercase				(					lowerCAmelCase__       ,					lowerCAmelCase__       ,					lowerCAmelCase__       ,					unittest.TestCase  ):







	'''simple docstring'''

	a			: str      =				StableDiffusionModelEditingPipeline
	a			: int      =				TEXT_TO_IMAGE_PARAMS
	a			: List[Any]      =				TEXT_TO_IMAGE_BATCH_PARAMS
	a			: Tuple      =				TEXT_TO_IMAGE_IMAGE_PARAMS
	a			: Any      =				TEXT_TO_IMAGE_IMAGE_PARAMS


	def   _UpperCAmelCase   (self       )							->							Dict:







				'''simple docstring'''





				torch.manual_seed(0       )
				__lowercase   =							UNetaDConditionModel(
				    block_out_channels=(32, 64)   ,layers_per_block=2   ,sample_size=32   ,in_channels=4   ,out_channels=4   ,down_block_types=('''DownBlock2D''', '''CrossAttnDownBlock2D''')   ,up_block_types=('''CrossAttnUpBlock2D''', '''UpBlock2D''')   ,cross_attention_dim=32   ,)
				__lowercase   =							DDIMScheduler()
				torch.manual_seed(0       )
				__lowercase   =							AutoencoderKL(
				    block_out_channels=[32, 64]   ,in_channels=3   ,out_channels=3   ,down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''']   ,up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''']   ,latent_channels=4   ,)
				torch.manual_seed(0       )
				__lowercase   =							CLIPTextConfig(
				    bos_token_id=0   ,eos_token_id=2   ,hidden_size=32   ,intermediate_size=37   ,layer_norm_eps=1E-0_5   ,num_attention_heads=4   ,num_hidden_layers=5   ,pad_token_id=1   ,vocab_size=1000   ,)
				__lowercase   =							CLIPTextModel(_lowerCamelCase       )
				__lowercase   =							CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''       )

				__lowercase   =							{
				    '''unet''': unet,
				    '''scheduler''': scheduler,
				    '''vae''': vae,
				    '''text_encoder''': text_encoder,
				    '''tokenizer''': tokenizer,
				    '''safety_checker''': None,
				    '''feature_extractor''': None,
				}
				return components


	def   _UpperCAmelCase   (self   ,_lowerCamelCase   ,_lowerCamelCase=0       )							->							Any:







				'''simple docstring'''





				__lowercase   =							torch.manual_seed(_lowerCamelCase       )
				__lowercase   =							{
				    '''prompt''': '''A field of roses''',
				    '''generator''': generator,
				    # Setting height and width to None to prevent OOMs on CPU.
				    '''height''': None,
				    '''width''': None,
				    '''num_inference_steps''': 2,
				    '''guidance_scale''': 6.0,
				    '''output_type''': '''numpy''',
				}
				return inputs


	def   _UpperCAmelCase   (self       )							->							Tuple:







				'''simple docstring'''





				__lowercase   =							'''cpu'''  # ensure determinism for the device-dependent torch.Generator
				__lowercase   =							self.get_dummy_components()
				__lowercase   =							StableDiffusionModelEditingPipeline(**_lowerCamelCase       )
				__lowercase   =							sd_pipe.to(_lowerCamelCase       )
				sd_pipe.set_progress_bar_config(disable=_lowerCamelCase       )

				__lowercase   =							self.get_dummy_inputs(_lowerCamelCase       )
				__lowercase   =							sd_pipe(**_lowerCamelCase       ).images
				__lowercase   =							image[0, -3:, -3:, -1]
				assert image.shape == (1, 64, 64, 3)

				__lowercase   =							np.array([0.4_7_5_5, 0.5_1_3_2, 0.4_9_7_6, 0.3_9_0_4, 0.3_5_5_4, 0.4_7_6_5, 0.5_1_3_9, 0.5_1_5_8, 0.4_8_8_9]       )

				assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-2


	def   _UpperCAmelCase   (self       )							->							Optional[int]:







				'''simple docstring'''





				__lowercase   =							'''cpu'''  # ensure determinism for the device-dependent torch.Generator
				__lowercase   =							self.get_dummy_components()
				__lowercase   =							StableDiffusionModelEditingPipeline(**_lowerCamelCase       )
				__lowercase   =							sd_pipe.to(_lowerCamelCase       )
				sd_pipe.set_progress_bar_config(disable=_lowerCamelCase       )

				__lowercase   =							self.get_dummy_inputs(_lowerCamelCase       )
				__lowercase   =							'''french fries'''
				__lowercase   =							sd_pipe(**_lowerCamelCase   ,negative_prompt=_lowerCamelCase       )
				__lowercase   =							output.images
				__lowercase   =							image[0, -3:, -3:, -1]

				assert image.shape == (1, 64, 64, 3)

				__lowercase   =							np.array([0.4_9_9_2, 0.5_1_0_1, 0.5_0_0_4, 0.3_9_4_9, 0.3_6_0_4, 0.4_7_3_5, 0.5_2_1_6, 0.5_2_0_4, 0.4_9_1_3]       )

				assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-2


	def   _UpperCAmelCase   (self       )							->							Tuple:







				'''simple docstring'''





				__lowercase   =							'''cpu'''  # ensure determinism for the device-dependent torch.Generator
				__lowercase   =							self.get_dummy_components()
				__lowercase   =							EulerAncestralDiscreteScheduler(
				    beta_start=0.0_0_0_8_5   ,beta_end=0.0_1_2   ,beta_schedule='''scaled_linear'''       )
				__lowercase   =							StableDiffusionModelEditingPipeline(**_lowerCamelCase       )
				__lowercase   =							sd_pipe.to(_lowerCamelCase       )
				sd_pipe.set_progress_bar_config(disable=_lowerCamelCase       )

				__lowercase   =							self.get_dummy_inputs(_lowerCamelCase       )
				__lowercase   =							sd_pipe(**_lowerCamelCase       ).images
				__lowercase   =							image[0, -3:, -3:, -1]

				assert image.shape == (1, 64, 64, 3)

				__lowercase   =							np.array([0.4_7_4_7, 0.5_3_7_2, 0.4_7_7_9, 0.4_9_8_2, 0.5_5_4_3, 0.4_8_1_6, 0.5_2_3_8, 0.4_9_0_4, 0.5_0_2_7]       )

				assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-2


	def   _UpperCAmelCase   (self       )							->							Optional[int]:







				'''simple docstring'''





				__lowercase   =							'''cpu'''  # ensure determinism for the device-dependent torch.Generator
				__lowercase   =							self.get_dummy_components()
				__lowercase   =							PNDMScheduler()
				__lowercase   =							StableDiffusionModelEditingPipeline(**_lowerCamelCase       )
				__lowercase   =							sd_pipe.to(_lowerCamelCase       )
				sd_pipe.set_progress_bar_config(disable=_lowerCamelCase       )

				__lowercase   =							self.get_dummy_inputs(_lowerCamelCase       )
				# the pipeline does not expect pndm so test if it raises error.
				with self.assertRaises(_lowerCamelCase       ):
							__lowercase   =							sd_pipe(**_lowerCamelCase       ).images


	def   _UpperCAmelCase   (self       )							->							int:







				'''simple docstring'''





				super().test_inference_batch_single_identical(expected_max_diff=5E-3       )


	def   _UpperCAmelCase   (self       )							->							str:







				'''simple docstring'''





				super().test_attention_slicing_forward_pass(expected_max_diff=5E-3       )






@slow
@require_torch_gpu
class 							__lowercase				(					unittest.TestCase  ):







	'''simple docstring'''

	def   _UpperCAmelCase   (self       )							->							List[str]:







				'''simple docstring'''





				super().tearDown()
				gc.collect()
				torch.cuda.empty_cache()


	def   _UpperCAmelCase   (self   ,_lowerCamelCase=0       )							->							Any:







				'''simple docstring'''





				__lowercase   =							torch.manual_seed(_lowerCamelCase       )
				__lowercase   =							{
				    '''prompt''': '''A field of roses''',
				    '''generator''': generator,
				    '''num_inference_steps''': 3,
				    '''guidance_scale''': 7.5,
				    '''output_type''': '''numpy''',
				}
				return inputs


	def   _UpperCAmelCase   (self       )							->							str:







				'''simple docstring'''





				__lowercase   =							'''CompVis/stable-diffusion-v1-4'''
				__lowercase   =							StableDiffusionModelEditingPipeline.from_pretrained(_lowerCamelCase   ,safety_checker=_lowerCamelCase       )
				pipe.to(_lowerCamelCase       )
				pipe.set_progress_bar_config(disable=_lowerCamelCase       )
				pipe.enable_attention_slicing()

				__lowercase   =							self.get_inputs()
				__lowercase   =							pipe(**_lowerCamelCase       ).images
				__lowercase   =							image[0, -3:, -3:, -1].flatten()

				assert image.shape == (1, 512, 512, 3)

				__lowercase   =							np.array(
				    [0.6_7_4_9_4_9_6, 0.6_3_8_6_4_5_3, 0.5_1_4_4_3_2_6_7, 0.6_6_0_9_4_9_0_5, 0.6_1_9_2_1_2_1_5, 0.5_4_9_1_3_3_2, 0.5_7_4_4_4_1_7, 0.5_8_0_7_5_1_0_6, 0.5_1_7_4_6_5_8]       )

				assert np.abs(expected_slice - image_slice       ).max() < 1E-2

				# make sure image changes after editing
				pipe.edit_model('''A pack of roses'''   ,'''A pack of blue roses'''       )

				__lowercase   =							pipe(**_lowerCamelCase       ).images
				__lowercase   =							image[0, -3:, -3:, -1].flatten()

				assert image.shape == (1, 512, 512, 3)

				assert np.abs(expected_slice - image_slice       ).max() > 1E-1


	def   _UpperCAmelCase   (self       )							->							List[str]:







				'''simple docstring'''





				torch.cuda.empty_cache()
				torch.cuda.reset_max_memory_allocated()
				torch.cuda.reset_peak_memory_stats()

				__lowercase   =							'''CompVis/stable-diffusion-v1-4'''
				__lowercase   =							DDIMScheduler.from_pretrained(_lowerCamelCase   ,subfolder='''scheduler'''       )
				__lowercase   =							StableDiffusionModelEditingPipeline.from_pretrained(
				    _lowerCamelCase   ,scheduler=_lowerCamelCase   ,safety_checker=_lowerCamelCase       )
				__lowercase   =							pipe.to(_lowerCamelCase       )
				pipe.set_progress_bar_config(disable=_lowerCamelCase       )
				pipe.enable_attention_slicing(1       )
				pipe.enable_sequential_cpu_offload()

				__lowercase   =							self.get_inputs()
				__lowercase   =							pipe(**_lowerCamelCase       )

				__lowercase   =							torch.cuda.max_memory_allocated()
				# make sure that less than 4.4 GB is allocated
				assert mem_bytes < 4.4 * 10**9





