prompt submssion

frontend/package.json

@@ -22,6 +22,7 @@
 		"@typescript-eslint/eslint-plugin": "^5.38.0",
 		"@typescript-eslint/parser": "^5.38.0",
 		"autoprefixer": "^10.4.12",
+		"d3-scale": "^4.0.2",
 		"eslint": "^8.24.0",
 		"eslint-config-prettier": "^8.3.0",
 		"eslint-plugin-svelte3": "^4.0.0",

frontend/src/lib/App.svelte

@@ -3,16 +3,103 @@
 	import Frame from '$lib/Frame.svelte';
 	import Canvas from '$lib/Canvas.svelte';
 	import Menu from '$lib/Menu.svelte';
+	import PromptModal from '$lib/PromptModal.svelte';
 	import type { Room } from '@liveblocks/client';
 	import { COLORS, EMOJIS } from '$lib/constants';
-	import { currZoomTransform, myPresence, others } from '$lib/store';
-
+	import { PUBLIC_WS_ENDPOINT } from '$env/static/public';
+	import {
+		isLoading,
+		loadingState,
+		currZoomTransform,
+		myPresence,
+		others,
+		isPrompting,
+		clickedPosition
+	} from '$lib/store';
+	import { base64ToBlob, uploadImage } from '$lib/utils';
 	/**
 	 * The main Liveblocks code for the example.
 	 * Check in src/routes/index.svelte to see the setup code.
 	 */
 
 	export let room: Room;
+
+	async function onClose(e: CustomEvent) {
+		$isPrompting = false;
+	}
+	async function onPrompt(e: CustomEvent) {
+		const prompt = e.detail.prompt;
+		const imgURLs = await generateImage(prompt);
+		$isPrompting = false;
+		console.log('prompt', prompt, imgURLs);
+	}
+	async function generateImage(_prompt: string) {
+		if (!_prompt || $isLoading == true) return;
+		$loadingState = 'Pending';
+		$isLoading = true;
+		const sessionHash = crypto.randomUUID();
+
+		const payload = {
+			fn_index: 2,
+			data: [_prompt],
+			session_hash: sessionHash
+		};
+		const websocket = new WebSocket(PUBLIC_WS_ENDPOINT);
+		// websocket.onopen = async function (event) {
+		// 	websocket.send(JSON.stringify({ hash: sessionHash }));
+		// };
+		websocket.onclose = (evt) => {
+			if (!evt.wasClean) {
+				$loadingState = 'Error';
+				$isLoading = false;
+			}
+		};
+		websocket.onmessage = async function (event) {
+			try {
+				const data = JSON.parse(event.data);
+				$loadingState = '';
+				switch (data.msg) {
+					case 'send_data':
+						$loadingState = 'Sending Data';
+						websocket.send(JSON.stringify(payload));
+						break;
+					case 'queue_full':
+						$loadingState = 'Queue full';
+						websocket.close();
+						$isLoading = false;
+						return;
+					case 'estimation':
+						const { msg, rank, queue_size } = data;
+						$loadingState = `On queue ${rank}/${queue_size}`;
+						break;
+					case 'process_generating':
+						$loadingState = data.success ? 'Generating' : 'Error';
+						break;
+					case 'process_completed':
+						try {
+							const imgsBase64 = data.output.data[0] as string[];
+							const imgBlobs = await Promise.all(imgsBase64.map((base64) => base64ToBlob(base64)));
+							const imgURLs = await Promise.all(imgBlobs.map((blob) => uploadImage(blob, _prompt)));
+							console.log(imgURLs);
+							$loadingState = data.success ? 'Complete' : 'Error';
+						} catch (e) {
+							$loadingState = e.message;
+						}
+						websocket.close();
+						$isLoading = false;
+						return;
+					case 'process_starts':
+						$loadingState = 'Processing';
+						break;
+				}
+			} catch (e) {
+				console.error(e);
+				$isLoading = false;
+				$loadingState = 'Error';
+			}
+		};
+	}
+	let modal = false;
 </script>
 
 <!-- Show the current user's cursor location -->
@@ -20,11 +107,19 @@
 	{$myPresence?.cursor
 		? `${$myPresence.cursor.x} × ${$myPresence.cursor.y}`
 		: 'Move your cursor to broadcast its position to other people in the room.'}
+	{$loadingState}
+	{$isLoading}
 </div>
+{#if $isPrompting}
+	<PromptModal on:prompt={onPrompt} on:close={onClose} />
+{/if}
 <div class="fixed left-0 z-0 w-screen h-screen cursor-none">
 	<Canvas />
 
 	<main class="z-10 relative">
+		{#if $clickedPosition}
+			<Frame color={COLORS[0]} position={$clickedPosition} transform={$currZoomTransform} />
+		{/if}
 		{#if $myPresence?.cursor}
 			<Frame color={COLORS[0]} position={$myPresence.cursor} transform={$currZoomTransform} />
 			<Cursor
@@ -56,7 +151,7 @@
 		{/if}
 	</main>
 </div>
-<div class="fixed bottom-0 left-0 right-0 z-50 my-2">
+<div class="fixed bottom-0 left-0 right-0 z-10 my-2">
 	<Menu />
 </div>
 

frontend/src/lib/Canvas.svelte

@@ -3,7 +3,7 @@
 	import { select } from 'd3-selection';
 	import { scaleLinear } from 'd3-scale';
 	import { onMount } from 'svelte';
-	import { currZoomTransform, myPresence, others } from '$lib/store';
+	import { currZoomTransform, myPresence, isPrompting, clickedPosition } from '$lib/store';
 
 	const height = 512 * 5;
 	const width = 512 * 5;
@@ -26,7 +26,7 @@
 
 		const scale = width / containerEl.clientWidth;
 		const zoomHandler = zoom()
-			.scaleExtent([1/scale,1])
+			.scaleExtent([1 / scale, 1])
 			// .translateExtent([
 			// 	[0, 0],
 			// 	[width, height]
@@ -39,7 +39,13 @@
 			.call(zoomHandler as any)
 			// .call(zoomHandler.scaleTo as any, 1 / scale)
 			.on('pointermove', handlePointerMove)
-			.on('pointerleave', handlePointerLeave);
+			.on('pointerleave', handlePointerLeave)
+			.on('dblclick.zoom', null)
+			.on('click', () => {
+				$isPrompting = true;
+				$clickedPosition = $myPresence.cursor;
+				console.log($clickedPosition);
+			});
 
 		canvasCtx = canvasEl.getContext('2d') as CanvasRenderingContext2D;
 		canvasCtx.fillStyle = 'red';

frontend/src/lib/PromptModal.svelte

@@ -0,0 +1,42 @@
+<script lang="ts">
+	import { createEventDispatcher, onMount } from 'svelte';
+	const dispatch = createEventDispatcher();
+	let prompt: string;
+
+	const onKeyup = (e: KeyboardEvent) => {
+		if (e.key === 'Escape') {
+			dispatch('close');
+		}
+	};
+	onMount(() => {
+		window.addEventListener('keyup', onKeyup);
+		return () => {
+			window.removeEventListener('keyup', onKeyup);
+		};
+	});
+</script>
+
+<form
+	class="fixed w-screen h-screen top-0 left-0 z-50 flex items-center justify-center bg-black bg-opacity-80 px-3"
+	on:submit|preventDefault={() => dispatch('prompt', { prompt })}
+	on:click={() => dispatch('close')}
+>
+	<input
+		on:click|stopPropagation
+		class="input"
+		placeholder="Type a prompt..."
+		title="Input prompt to generate image and obtain palette"
+		type="text"
+		name="prompt"
+		bind:value={prompt}
+	/>
+</form>
+
+<style lang="postcss" scoped>
+	.link {
+		@apply text-xs underline font-bold hover:no-underline hover:text-gray-500 visited:text-gray-500;
+	}
+	.input {
+		@apply w-full max-w-sm text-sm disabled:opacity-50 italic placeholder:text-white text-white placeholder:text-opacity-50 bg-slate-900 border-2 border-white rounded-2xl px-2 shadow-sm focus:outline-none focus:border-gray-400 focus:ring-1;
+	}
+</style>

frontend/src/lib/store.ts

@@ -5,6 +5,8 @@ import { type ZoomTransform, zoomIdentity } from 'd3-zoom';
 
 export const loadingState = writable<string>('');
 export const isLoading = writable<boolean>(false);
+export const isPrompting = writable<boolean>(false);
+export const clickedPosition = writable<{ x: number; y: number }>();
 
 export const currZoomTransform = writable<ZoomTransform>(zoomIdentity);
 

frontend/src/lib/utils.ts

@@ -3,6 +3,27 @@ import { dev } from '$app/environment';
 export function randomSeed() {
 	return BigInt(13248873089935215612 & (((1 << 63) - 1) * Math.random()));
 }
+
+export function base64ToBlob(base64image: string): Promise<Blob> {
+	return new Promise((resolve) => {
+		const img = new Image();
+		img.onload = async () => {
+			const w = img.width;
+			const h = img.height;
+			const canvas = document.createElement('canvas');
+			canvas.width = w;
+			canvas.height = h;
+			const ctx = canvas.getContext('2d') as CanvasRenderingContext2D;
+			ctx.drawImage(img, 0, 0, w, h);
+
+			const imgBlob: Blob = await new Promise((_resolve) =>
+				canvas.toBlob(_resolve, 'image/jpeg', 0.95)
+			);
+			resolve(imgBlob);
+		};
+		img.src = base64image;
+	});
+}
 export async function uploadImage(imagBlob: Blob, prompt: string): string {
 	// simple regex slugify string	for file name
 	const promptSlug = slugify(prompt);

frontend/src/routes/+page.svelte

@@ -46,40 +46,12 @@
 </script>
 
 <div class="max-w-screen-md mx-auto px-3 py-8 relative">
-	<div class="relative z-10">
+	<div class="relative">
 		<h1 class="text-3xl font-bold leading-normal">Stable Diffussion Outpainting Multiplayer</h1>
-		<p class="text-sm" />
-		<div class="relative bg-white dark:bg-black py-3">
-			<form class="grid grid-cols-6">
-				<input
-					class="input"
-					placeholder="A photo of a beautiful sunset in San Francisco"
-					title="Input prompt to generate image and obtain palette"
-					type="text"
-					name="prompt"
-					disabled={$isLoading}
-				/>
-				<button class="button" disabled={$isLoading} title="Generate Palette">
-					Create Palette
-				</button>
-			</form>
-		</div>
 	</div>
-	<div class="relative z-0">
+	<div class="relative">
 		{#if room}
 			<App {room} />
 		{/if}
 	</div>
 </div>
-
-<style lang="postcss" scoped>
-	.link {
-		@apply text-xs underline font-bold hover:no-underline hover:text-gray-500 visited:text-gray-500;
-	}
-	.input {
-		@apply text-sm disabled:opacity-50 col-span-4 md:col-span-5 italic dark:placeholder:text-black placeholder:text-white text-white dark:text-black placeholder:text-opacity-30 dark:placeholder:text-opacity-10 dark:bg-white bg-slate-900 border-2 border-black rounded-2xl px-2 shadow-sm focus:outline-none focus:border-gray-400 focus:ring-1;
-	}
-	.button {
-		@apply disabled:opacity-50 col-span-2 md:col-span-1 dark:bg-white dark:text-black border-2 border-black rounded-2xl ml-2 px-2 py-2  text-xs shadow-sm font-bold focus:outline-none focus:border-gray-400;
-	}
-</style>