import os import torch import gradio as gr from tqdm import tqdm from PIL import Image import torch.nn.functional as F from torchvision import transforms as tfms from transformers import CLIPTextModel, CLIPTokenizer, logging from diffusers import AutoencoderKL, LMSDiscreteScheduler, UNet2DConditionModel, DiffusionPipeline HTML_TEMPLATE = """

Generative Art with Textual Inversion and Guidance

Generate unique art using different styles and guidance methods.

Illustration Style
Illustration Style
Line Art
Line Art
""" torch_device = "cuda" if torch.cuda.is_available() else "mps" if torch.backends.mps.is_available() else "cpu" if "mps" == torch_device: os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = "1" # Load the pipeline model_path = "CompVis/stable-diffusion-v1-4" sd_pipeline = DiffusionPipeline.from_pretrained( model_path, low_cpu_mem_usage=True, torch_dtype=torch.float16 ).to(torch_device) # Load textual inversions sd_pipeline.load_textual_inversion("sd-concepts-library/illustration-style") sd_pipeline.load_textual_inversion("sd-concepts-library/line-art") sd_pipeline.load_textual_inversion("sd-concepts-library/hitokomoru-style-nao") sd_pipeline.load_textual_inversion("sd-concepts-library/style-of-marc-allante") sd_pipeline.load_textual_inversion("sd-concepts-library/midjourney-style") sd_pipeline.load_textual_inversion("sd-concepts-library/hanfu-anime-style") sd_pipeline.load_textual_inversion("sd-concepts-library/birb-style") # Update style token dictionary style_token_dict = { "Illustration Style": '', "Line Art":'', "Hitokomoru Style":'', "Marc Allante": '', "Midjourney":'', "Hanfu Anime": '', "Birb Style": '' } def apply_guidance(image, guidance_method, loss_scale): # Convert PIL Image to tensor img_tensor = tfms.ToTensor()(image).unsqueeze(0).to(torch_device) if guidance_method == 'Grayscale': gray = tfms.Grayscale(3)(img_tensor) guided = img_tensor + (gray - img_tensor) * (loss_scale / 10000) elif guidance_method == 'Bright': bright = F.relu(img_tensor) # Simple brightness increase guided = img_tensor + (bright - img_tensor) * (loss_scale / 10000) elif guidance_method == 'Contrast': mean = img_tensor.mean() contrast = (img_tensor - mean) * 2 + mean guided = img_tensor + (contrast - img_tensor) * (loss_scale / 10000) elif guidance_method == 'Symmetry': flipped = torch.flip(img_tensor, [3]) # Flip horizontally guided = img_tensor + (flipped - img_tensor) * (loss_scale / 10000) elif guidance_method == 'Saturation': saturated = tfms.functional.adjust_saturation(img_tensor, 2) guided = img_tensor + (saturated - img_tensor) * (loss_scale / 10000) else: return image # Convert back to PIL Image guided = guided.squeeze(0).clamp(0, 1) guided = (guided * 255).byte().cpu().permute(1, 2, 0).numpy() return Image.fromarray(guided) def generate_with_guidance(prompt, num_inference_steps, guidance_scale, seed, guidance_method, loss_scale): # Generate image with pipeline generator = torch.Generator(device=torch_device).manual_seed(seed) image = sd_pipeline( prompt, num_inference_steps=num_inference_steps, guidance_scale=guidance_scale, generator=generator ).images[0] # Apply guidance guided_image = apply_guidance(image, guidance_method, loss_scale) return guided_image def inference(text, style, inference_step, guidance_scale, seed, guidance_method, loss_scale): prompt = text + " " + style_token_dict[style] # Generate image with pipeline image_pipeline = sd_pipeline( prompt, num_inference_steps=inference_step, guidance_scale=guidance_scale, generator=torch.Generator(device=torch_device).manual_seed(seed) ).images[0] # Generate image with guidance image_guide = generate_with_guidance(prompt, inference_step, guidance_scale, seed, guidance_method, loss_scale) return image_pipeline, image_guide title = "Generative with Textual Inversion and Guidance" description = "A Gradio interface to infer Stable Diffusion and generate images with different art styles and guidance methods" examples = [ ["A majestic castle on a floating island", 'Illustration Style', 10, 7.5, 42, 'Grayscale', 200] ] title = "Generative Art with Textual Inversion and Guidance" description = "Create unique artworks using Stable Diffusion with various styles and guidance methods." with gr.Blocks(css=HTML_TEMPLATE) as demo: gr.HTML(HTML_TEMPLATE) # This adds the styled header to your app with gr.Row(): text = gr.Textbox(label="Prompt", placeholder="Enter your creative prompt here...") style = gr.Dropdown(label="Style", choices=list(style_token_dict.keys()), value="Illustration Style") with gr.Row(): inference_step = gr.Slider(1, 50, 10, step=1, label="Inference steps") guidance_scale = gr.Slider(1, 10, 7.5, step=0.1, label="Guidance scale") seed = gr.Slider(0, 10000, 42, step=1, label="Seed") with gr.Row(): guidance_method = gr.Dropdown(label="Guidance method", choices=['Grayscale', 'Bright', 'Contrast', 'Symmetry', 'Saturation'], value="Grayscale") loss_scale = gr.Slider(100, 10000, 200, step=100, label="Loss scale") with gr.Row(): generate_button = gr.Button("Generate Art") with gr.Row(): output_image = gr.Image(width=512, height=512, label="Generated art") output_image_guided = gr.Image(width=512, height=512, label="Generated art with guidance") generate_button.click( inference, inputs=[text, style, inference_step, guidance_scale, seed, guidance_method, loss_scale], outputs=[output_image, output_image_guided] ) gr.Examples( examples=[ ["A majestic castle on a floating island", 'Illustration Style', 10, 7.5, 42, 'Grayscale', 200] ], inputs=[text, style, inference_step, guidance_scale, seed, guidance_method, loss_scale], outputs=[output_image, output_image_guided], fn=inference, cache_examples=True, ) demo.launch()