get imgt2t models; render eye icon on them; add upload image icon

src/lib/components/debug-menu.svelte

@@ -8,6 +8,7 @@
 	import { showQuotaModal } from "./quota-modal.svelte";
 	import type { ToastData } from "./toaster.svelte.js";
 	import { addToast } from "./toaster.svelte.js";
+	import { models } from "$lib/state/models.svelte";
 
 	let innerWidth = $state<number>();
 	let innerHeight = $state<number>();
@@ -31,6 +32,12 @@
 				console.log(session.$);
 			},
 		},
+		{
+			label: "Log models to console",
+			cb: () => {
+				console.log(models.all);
+			},
+		},
 		{
 			label: "Test prompt",
 			cb: async () => {

src/lib/components/inference-playground/message.svelte

@@ -1,6 +1,8 @@
 <script lang="ts">
 	import { TextareaAutosize } from "$lib/spells/textarea-autosize.svelte.js";
 	import type { ConversationMessage } from "$lib/types.js";
+	import Tooltip from "$lib/components/tooltip.svelte";
+	import IconImage from "~icons/carbon/image-reference";
 
 	type Props = {
 		content: ConversationMessage["content"];
@@ -20,28 +22,62 @@
 </script>
 
 <div
-	class=" group/message group grid grid-cols-[1fr_2.5rem] items-start gap-2 border-b px-3.5 pt-4 pb-6 hover:bg-gray-100/70 @-2xl:grid-cols-[130px_1fr_2.5rem] @2xl:grid-rows-1 @2xl:gap-4 @2xl:px-6 dark:border-gray-800 dark:hover:bg-gray-800/30"
+	class="group/message group flex flex-col items-start gap-x-4 gap-y-2 border-b px-3.5 pt-4 pb-6 hover:bg-gray-100/70
+	@2xl:flex-row @2xl:px-6 dark:border-gray-800 dark:hover:bg-gray-800/30"
 	class:pointer-events-none={loading}
 >
-	<div class="col-span-2 pt-3 pb-1 text-sm font-semibold uppercase @2xl:col-span-1 @2xl:pb-2">
+	<div class="pt-3 text-sm font-semibold uppercase @2xl:basis-[130px]">
 		{role}
 	</div>
-	<!-- svelte-ignore a11y_autofocus -->
-	<!-- svelte-ignore a11y_positive_tabindex -->
-	<textarea
-		bind:this={element}
-		{autofocus}
-		bind:value={content}
-		placeholder="Enter {role} message"
-		class="resize-none overflow-hidden rounded-sm bg-transparent px-2 py-2.5 ring-gray-100 outline-none group-hover/message:ring-3 hover:bg-white focus:bg-white focus:ring-3 @2xl:px-3 dark:ring-gray-600 dark:hover:bg-gray-900 dark:focus:bg-gray-900"
-		rows="1"
-		tabindex="2"
-	></textarea>
-	<button
-		tabindex="0"
-		onclick={onDelete}
-		type="button"
-		class="mt-1.5 size-8 rounded-lg border border-gray-200 bg-white text-xs font-medium text-gray-900 group-hover/message:block hover:bg-gray-100 hover:text-blue-700 focus:z-10 focus:ring-4 focus:ring-gray-100 focus:outline-hidden sm:hidden dark:border-gray-600 dark:bg-gray-800 dark:text-gray-400 dark:hover:bg-gray-700 dark:hover:text-white dark:focus:ring-gray-700"
-		>✕</button
-	>
+	<div class="flex w-full items-center gap-4">
+		<!-- svelte-ignore a11y_autofocus -->
+		<!-- svelte-ignore a11y_positive_tabindex -->
+		<textarea
+			bind:this={element}
+			{autofocus}
+			bind:value={content}
+			placeholder="Enter {role} message"
+			class="grow resize-none overflow-hidden rounded-lg bg-transparent px-2 py-2.5 ring-gray-100 outline-none group-hover/message:ring-3 hover:bg-white focus:bg-white focus:ring-3 @2xl:px-3 dark:ring-gray-600 dark:hover:bg-gray-900 dark:focus:bg-gray-900"
+			rows="1"
+			tabindex="2"
+		></textarea>
+
+		<Tooltip openDelay={250}>
+			{#snippet trigger(tooltip)}
+				<button
+					tabindex="0"
+					onclick={onDelete}
+					type="button"
+					class="grid size-8 place-items-center rounded-lg border border-gray-200 bg-white text-xs font-medium text-gray-900
+			group-focus-within/message:visible group-hover/message:visible hover:bg-gray-100
+			hover:text-blue-700 focus:z-10 focus:ring-4
+			focus:ring-gray-100 focus:outline-hidden sm:invisible dark:border-gray-600 dark:bg-gray-800
+			dark:text-gray-400 dark:hover:bg-gray-700 dark:hover:text-white dark:focus:ring-gray-700"
+					{...tooltip.trigger}
+				>
+					<IconImage />
+				</button>
+			{/snippet}
+			Add image
+		</Tooltip>
+
+		<Tooltip>
+			{#snippet trigger(tooltip)}
+				<button
+					tabindex="0"
+					onclick={onDelete}
+					type="button"
+					class="size-8 rounded-lg border border-gray-200 bg-white text-xs font-medium text-gray-900
+			group-focus-within/message:visible group-hover/message:visible hover:bg-gray-100
+			hover:text-blue-700 focus:z-10 focus:ring-4
+			focus:ring-gray-100 focus:outline-hidden sm:invisible dark:border-gray-600 dark:bg-gray-800
+			dark:text-gray-400 dark:hover:bg-gray-700 dark:hover:text-white dark:focus:ring-gray-700"
+					{...tooltip.trigger}
+				>
+					✕
+				</button>
+			{/snippet}
+			Delete
+		</Tooltip>
+	</div>
 </div>

src/lib/components/inference-playground/model-selector-modal.svelte

@@ -9,6 +9,7 @@
 	import { watch } from "runed";
 	import IconSearch from "~icons/carbon/search";
 	import IconStar from "~icons/carbon/star";
+	import IconEye from "~icons/carbon/view";
 
 	interface Props {
 		onModelSelect?: (model: string) => void;
@@ -145,6 +146,11 @@
 								class="mx-1 text-gray-300 dark:text-gray-700">/</span
 							><span class="text-black dark:text-white">{modelName}</span></span
 						>
+						{#if model.pipeline_tag === "image-text-to-text"}
+							<div class="lucide lucide-star ml-auto size-4 text-white">
+								<IconEye />
+							</div>
+						{/if}
 					</button>
 				{/snippet}
 				{#if trending.length > 0}

src/lib/components/tooltip.svelte

@@ -0,0 +1,48 @@
+<script lang="ts">
+	import { Tooltip, type TooltipProps } from "melt/builders";
+	import { type ComponentProps, type Extracted } from "melt";
+	import type { Snippet } from "svelte";
+
+	interface Props {
+		children: Snippet;
+		trigger: Snippet<[Tooltip]>;
+		placement?: NonNullable<Extracted<TooltipProps["computePositionOptions"]>>["placement"];
+		openDelay?: ComponentProps<TooltipProps>["openDelay"];
+	}
+	const { children, trigger, placement = "top", openDelay }: Props = $props();
+
+	const tooltip = new Tooltip({
+		forceVisible: true,
+		computePositionOptions: () => ({ placement }),
+		openDelay: () => openDelay,
+	});
+</script>
+
+{@render trigger(tooltip)}
+
+<div {...tooltip.content} class="rounded-xl bg-white p-0 shadow-xl dark:bg-gray-700">
+	<div {...tooltip.arrow} class="rounded-tl"></div>
+	<p class="px-4 py-1 text-gray-700 dark:text-white">{@render children()}</p>
+</div>
+
+<style>
+	[data-melt-tooltip-content] {
+		border: 0;
+
+		position: absolute;
+		pointer-events: none;
+		opacity: 0;
+
+		transform: scale(0.9);
+
+		transition: 0.3s;
+		transition-property: opacity, transform;
+	}
+
+	[data-melt-tooltip-content][data-open] {
+		pointer-events: auto;
+		opacity: 1;
+
+		transform: scale(1);
+	}
+</style>

src/lib/types.ts

@@ -169,6 +169,7 @@ export enum LibraryName {
 
 export enum PipelineTag {
 	TextGeneration = "text-generation",
+	ImageTextToText = "image-text-to-text",
 }
 
 export type MaybeGetter<T> = T | (() => T);

src/routes/+layout.svelte

@@ -3,6 +3,7 @@
 	import Prompts from "$lib/components/prompts.svelte";
 	import QuotaModal from "$lib/components/quota-modal.svelte";
 	import "../app.css";
+
 	interface Props {
 		children?: import("svelte").Snippet;
 	}

src/routes/api/models/+server.ts

@@ -5,68 +5,117 @@ import { dev } from "$app/environment";
 
 let cache: ModelWithTokenizer[] | undefined;
 
+const headers: HeadersInit = {
+	"Upgrade-Insecure-Requests": "1",
+	"Sec-Fetch-Dest": "document",
+	"Sec-Fetch-Mode": "navigate",
+	"Sec-Fetch-Site": "none",
+	"Sec-Fetch-User": "?1",
+	"Priority": "u=0, i",
+	"Pragma": "no-cache",
+	"Cache-Control": "no-cache",
+};
+
+const requestInit: RequestInit = {
+	credentials: "include",
+	headers,
+	method: "GET",
+	mode: "cors",
+};
+
+interface ApiQueryParams {
+	pipeline_tag?: "text-generation" | "image-text-to-text";
+	filter: string;
+	inference_provider: string;
+	limit: number;
+	expand: string[];
+}
+
+const queryParams: ApiQueryParams = {
+	filter: "conversational",
+	inference_provider: "all",
+	limit: 100,
+	expand: ["inferenceProviderMapping", "config", "library_name", "pipeline_tag", "tags", "mask_token", "trendingScore"],
+};
+
+const baseUrl = "https://huggingface.co/api/models";
+
+function buildApiUrl(params: ApiQueryParams): string {
+	const url = new URL(baseUrl);
+	// Add simple params
+	Object.entries(params).forEach(([key, value]) => {
+		if (!Array.isArray(value)) {
+			url.searchParams.append(key, String(value));
+		}
+	});
+	// Handle array params specially
+	params.expand.forEach(item => {
+		url.searchParams.append("expand[]", item);
+	});
+	return url.toString();
+}
+
 export const GET: RequestHandler = async ({ fetch }) => {
 	if (cache?.length && dev) {
 		console.log("Skipping load, using in memory cache");
 		return json(cache);
 	}
 
-	const apiUrl =
-		"https://huggingface.co/api/models?pipeline_tag=text-generation&filter=conversational&inference_provider=all&limit=100&expand[]=inferenceProviderMapping&expand[]=config&expand[]=library_name&expand[]=pipeline_tag&expand[]=tags&expand[]=mask_token&expand[]=trendingScore";
-
-	const res = await fetch(apiUrl, {
-		credentials: "include",
-		headers: {
-			"Upgrade-Insecure-Requests": "1",
-			"Sec-Fetch-Dest": "document",
-			"Sec-Fetch-Mode": "navigate",
-			"Sec-Fetch-Site": "none",
-			"Sec-Fetch-User": "?1",
-			"Priority": "u=0, i",
-			"Pragma": "no-cache",
-			"Cache-Control": "no-cache",
-		},
-		method: "GET",
-		mode: "cors",
-	});
-
-	if (!res.ok) {
-		console.error(`Error fetching warm models`, res.status, res.statusText);
-		return json({ models: [] });
-	}
+	try {
+		// Fetch both types of models in parallel
+		const textGenPromise = fetch(buildApiUrl({ ...queryParams, pipeline_tag: "text-generation" }), requestInit);
+		const imgText2TextPromise = fetch(buildApiUrl({ ...queryParams, pipeline_tag: "image-text-to-text" }), requestInit);
+		const [textGenResponse, imgText2TextResponse] = await Promise.all([textGenPromise, imgText2TextPromise]);
 
-	const compatibleModels: Model[] = await res.json();
-	compatibleModels.sort((a, b) => a.id.toLowerCase().localeCompare(b.id.toLowerCase()));
-
-	const promises = compatibleModels.map(async model => {
-		const configUrl = `https://huggingface.co/${model.id}/raw/main/tokenizer_config.json`;
-		const res = await fetch(configUrl, {
-			credentials: "include",
-			headers: {
-				"Upgrade-Insecure-Requests": "1",
-				"Sec-Fetch-Dest": "document",
-				"Sec-Fetch-Mode": "navigate",
-				"Sec-Fetch-Site": "none",
-				"Sec-Fetch-User": "?1",
-				"Priority": "u=0, i",
-				"Pragma": "no-cache",
-				"Cache-Control": "no-cache",
-			},
-			method: "GET",
-			mode: "cors",
-		});
+		if (!textGenResponse.ok) {
+			console.error(`Error fetching text-generation models`, textGenResponse.status, textGenResponse.statusText);
+		}
 
-		if (!res.ok) {
-			// console.error(`Error fetching tokenizer file for ${model.id}`, res.status, res.statusText);
-			return null; // Ignore failed requests by returning null
+		if (!imgText2TextResponse.ok) {
+			console.error(
+				`Error fetching image-text-to-text models`,
+				imgText2TextResponse.status,
+				imgText2TextResponse.statusText
+			);
 		}
 
-		const tokenizerConfig = await res.json();
-		return { ...model, tokenizerConfig } satisfies ModelWithTokenizer;
-	});
+		// Parse the responses
+		const textGenModels: Model[] = textGenResponse.ok ? await textGenResponse.json() : [];
+		const imgText2TextModels: Model[] = imgText2TextResponse.ok ? await imgText2TextResponse.json() : [];
+
+		// Combine the models
+		const compatibleModels: Model[] = [...textGenModels, ...imgText2TextModels];
+
+		// Sort the models
+		compatibleModels.sort((a, b) => a.id.toLowerCase().localeCompare(b.id.toLowerCase()));
 
-	const models: ModelWithTokenizer[] = (await Promise.all(promises)).filter(model => model !== null);
-	cache = models;
+		// Fetch tokenizer configs for each model
+		const promises = compatibleModels.map(async model => {
+			const configUrl = `https://huggingface.co/${model.id}/raw/main/tokenizer_config.json`;
+			const res = await fetch(configUrl, {
+				credentials: "include",
+				headers,
+				method: "GET",
+				mode: "cors",
+			});
 
-	return json(cache);
+			if (!res.ok) {
+				// console.error(`Error fetching tokenizer file for ${model.id}`, res.status, res.statusText);
+				return null; // Ignore failed requests by returning null
+			}
+
+			const tokenizerConfig = await res.json();
+			return { ...model, tokenizerConfig } satisfies ModelWithTokenizer;
+		});
+
+		const models: ModelWithTokenizer[] = (await Promise.all(promises)).filter(
+			(model): model is ModelWithTokenizer => model !== null
+		);
+		cache = models;
+
+		return json(cache);
+	} catch (error) {
+		console.error("Error fetching models:", error);
+		return json([]);
+	}
 };