import os import random import uuid import json import time import asyncio import re from threading import Thread import gradio as gr import spaces import torch import numpy as np from PIL import Image import edge_tts import subprocess # Install flash-attn with our environment flag (if needed) subprocess.run( 'pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True ) # Set torch backend configurations for Flux RealismLora torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.backends.cuda.matmul.allow_tf32 = True # ------------------------------- # CONFIGURATION & UTILITY FUNCTIONS # ------------------------------- MAX_SEED = 2**32 - 1 def save_image(img: Image.Image) -> str: """Save a PIL image with a unique filename and return its path.""" unique_name = str(uuid.uuid4()) + ".png" img.save(unique_name) return unique_name def randomize_seed_fn(seed: int, randomize_seed: bool) -> int: if randomize_seed: seed = random.randint(0, MAX_SEED) return seed def progress_bar_html(label: str) -> str: """ Returns an HTML snippet for an animated progress bar with a given label. """ return f'''
{label}
''' # ------------------------------- # FLUX REALISMLORA IMAGE GENERATION SETUP (New Implementation) # ------------------------------- from diffusers import DiffusionPipeline base_model = "black-forest-labs/FLUX.1-dev" pipe = DiffusionPipeline.from_pretrained(base_model, torch_dtype=torch.bfloat16) lora_repo = "XLabs-AI/flux-RealismLora" trigger_word = "" # No trigger word used. pipe.load_lora_weights(lora_repo) pipe.to("cuda") @spaces.GPU() def run_lora(prompt, cfg_scale, steps, randomize_seed, seed, width, height, lora_scale, progress=gr.Progress(track_tqdm=True)): # Set random seed for reproducibility if randomize_seed: seed = random.randint(0, MAX_SEED) generator = torch.Generator(device="cuda").manual_seed(seed) # Update progress bar (0% at start) progress(0, "Starting image generation...") # Simulate progress updates during the steps for i in range(1, steps + 1): if steps >= 10 and i % (steps // 10) == 0: progress(i / steps * 100, f"Processing step {i} of {steps}...") # Generate image using the pipeline image = pipe( prompt=f"{prompt} {trigger_word}", num_inference_steps=steps, guidance_scale=cfg_scale, width=width, height=height, generator=generator, joint_attention_kwargs={"scale": lora_scale}, ).images[0] # Final progress update (100%) progress(100, "Completed!") yield image, seed # ------------------------------- # SMOLVLM2 SETUP (Default Text/Multimodal Model) # ------------------------------- from transformers import AutoProcessor, AutoModelForImageTextToText, TextIteratorStreamer smol_processor = AutoProcessor.from_pretrained("HuggingFaceTB/SmolVLM2-2.2B-Instruct") smol_model = AutoModelForImageTextToText.from_pretrained( "HuggingFaceTB/SmolVLM2-2.2B-Instruct", _attn_implementation="flash_attention_2", torch_dtype=torch.float16 ).to("cuda:0") # ------------------------------- # TTS UTILITY FUNCTIONS # ------------------------------- TTS_VOICES = [ "en-US-JennyNeural", # @tts1 "en-US-GuyNeural", # @tts2 ] async def text_to_speech(text: str, voice: str, output_file="output.mp3"): """Convert text to speech using Edge TTS and save the output as MP3.""" communicate = edge_tts.Communicate(text, voice) await communicate.save(output_file) return output_file # ------------------------------- # CHAT / MULTIMODAL GENERATION FUNCTION # ------------------------------- @spaces.GPU def generate(input_dict: dict, chat_history: list[dict], max_tokens: int = 200): """ Generates chatbot responses using SmolVLM2 with support for multimodal inputs and TTS. Special commands: - "@image": triggers image generation using the RealismLora flux implementation. - "@tts1" or "@tts2": triggers text-to-speech after generation. """ torch.cuda.empty_cache() text = input_dict["text"] files = input_dict.get("files", []) # If the query starts with "@image", use RealismLora to generate an image. if text.strip().lower().startswith("@image"): prompt = text[len("@image"):].strip() yield progress_bar_html("Hold Tight Generating Flux RealismLora Image") # Default parameters for RealismLora generation default_cfg_scale = 3.2 default_steps = 32 default_width = 1152 default_height = 896 default_seed = 3981632454 default_lora_scale = 0.85 # Call the new run_lora function and yield its final result for result in run_lora(prompt, default_cfg_scale, default_steps, True, default_seed, default_width, default_height, default_lora_scale, progress=gr.Progress(track_tqdm=True)): final_result = result yield gr.Image(final_result[0]) return # Handle TTS commands if present. tts_prefix = "@tts" is_tts = any(text.strip().lower().startswith(f"{tts_prefix}{i}") for i in range(1, 3)) voice = None if is_tts: voice_index = next((i for i in range(1, 3) if text.strip().lower().startswith(f"{tts_prefix}{i}")), None) if voice_index: voice = TTS_VOICES[voice_index - 1] text = text.replace(f"{tts_prefix}{voice_index}", "").strip() yield "Processing with SmolVLM2" # Build conversation messages based on input and history. user_content = [] media_queue = [] if chat_history == []: text = text.strip() for file in files: if file.endswith((".png", ".jpg", ".jpeg", ".gif", ".bmp")): media_queue.append({"type": "image", "path": file}) elif file.endswith((".mp4", ".mov", ".avi", ".mkv", ".flv")): media_queue.append({"type": "video", "path": file}) if "" in text or "