Accurate max tokens (#77)

.env.example

@@ -0,0 +1,9 @@
+HYPERBOLIC_API_KEY=
+COHERE_API_KEY=
+TOGETHER_API_KEY=
+FIREWORKS_API_KEY=
+REPLICATE_API_KEY=
+NEBIUS_API_KEY=
+NOVITA_API_KEY=
+FAL_API_KEY=
+HF_TOKEN=

.prettierignore

@@ -16,3 +16,5 @@ node_modules
 # Ignore files for PNPM, NPM and YARN
 pnpm-lock.yaml
 yarn.lock
+
+context_length.json

eslint.config.mts

@@ -86,6 +86,7 @@ export default ts.config(
 			"**/pnpm-lock.yaml",
 			"**/package-lock.json",
 			"**/yarn.lock",
+			"context_length.json",
 		],
 	},
 	{

package.json

@@ -3,24 +3,25 @@
 	"version": "0.0.1",
 	"private": true,
 	"scripts": {
-		"dev": "vite dev",
-		"build": "vite build",
+		"dev": "pnpm run update-ctx-length && vite dev",
+		"build": "pnpm run update-ctx-length && vite build",
 		"preview": "vite preview",
 		"prepare": "ts-patch install && svelte-kit sync || echo ''",
 		"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
 		"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
 		"lint": "prettier . --check . && eslint src/",
 		"format": "prettier . --write .",
-		"clean": "rm -rf ./node_modules/ && rm -rf ./.svelte-kit/ && ni && echo 'Project cleaned!'"
+		"clean": "rm -rf ./node_modules/ && rm -rf ./.svelte-kit/ && ni && echo 'Project cleaned!'",
+		"update-ctx-length": "jiti scripts/update-ctx-length.ts"
 	},
 	"devDependencies": {
 		"@eslint/eslintrc": "^3.3.0",
 		"@eslint/js": "^9.22.0",
 		"@floating-ui/dom": "^1.6.13",
 		"@huggingface/hub": "^1.0.1",
-		"@huggingface/transformers": "^3.4.2",
 		"@huggingface/inference": "^3.5.1",
 		"@huggingface/tasks": "^0.17.1",
+		"@huggingface/transformers": "^3.4.2",
 		"@iconify-json/carbon": "^1.2.8",
 		"@iconify-json/material-symbols": "^1.2.15",
 		"@ryoppippi/unplugin-typia": "^1.0.0",
@@ -31,7 +32,9 @@
 		"@sveltejs/vite-plugin-svelte": "^4.0.0",
 		"@tailwindcss/container-queries": "^0.1.1",
 		"@tailwindcss/postcss": "^4.0.9",
+		"@types/node": "^22.14.1",
 		"clsx": "^2.1.1",
+		"dotenv": "^16.5.0",
 		"eslint": "^9.22.0",
 		"eslint-config-prettier": "^10.1.1",
 		"eslint-plugin-prettier": "^5.2.3",

pnpm-lock.yaml

@@ -44,31 +44,37 @@ importers:
         version: 1.2.15
       '@ryoppippi/unplugin-typia':
         specifier: ^1.0.0
-        version: 1.2.0(@samchon/[email protected])(@types/node@18.19.84)([email protected])([email protected])([email protected])([email protected])
+        version: 1.2.0(@samchon/[email protected])(@types/node@22.14.1)([email protected])([email protected])([email protected])([email protected])
       '@samchon/openapi':
         specifier: ^3.0.0
         version: 3.0.0
       '@sveltejs/adapter-auto':
         specifier: ^3.2.2
-        version: 3.3.1(@sveltejs/[email protected](@sveltejs/[email protected]([email protected])([email protected](@types/node@18.19.84)([email protected])))([email protected])([email protected](@types/node@18.19.84)([email protected])))
+        version: 3.3.1(@sveltejs/[email protected](@sveltejs/[email protected]([email protected])([email protected](@types/node@22.14.1)([email protected])))([email protected])([email protected](@types/node@22.14.1)([email protected])))
       '@sveltejs/adapter-node':
         specifier: ^5.2.0
-        version: 5.2.12(@sveltejs/[email protected](@sveltejs/[email protected]([email protected])([email protected](@types/node@18.19.84)([email protected])))([email protected])([email protected](@types/node@18.19.84)([email protected])))
+        version: 5.2.12(@sveltejs/[email protected](@sveltejs/[email protected]([email protected])([email protected](@types/node@22.14.1)([email protected])))([email protected])([email protected](@types/node@22.14.1)([email protected])))
       '@sveltejs/kit':
         specifier: ^2.5.27
-        version: 2.18.0(@sveltejs/[email protected]([email protected])([email protected](@types/node@18.19.84)([email protected])))([email protected])([email protected](@types/node@18.19.84)([email protected]))
+        version: 2.18.0(@sveltejs/[email protected]([email protected])([email protected](@types/node@22.14.1)([email protected])))([email protected])([email protected](@types/node@22.14.1)([email protected]))
       '@sveltejs/vite-plugin-svelte':
         specifier: ^4.0.0
-        version: 4.0.4([email protected])([email protected](@types/node@18.19.84)([email protected]))
+        version: 4.0.4([email protected])([email protected](@types/node@22.14.1)([email protected]))
       '@tailwindcss/container-queries':
         specifier: ^0.1.1
         version: 0.1.1([email protected])
       '@tailwindcss/postcss':
         specifier: ^4.0.9
         version: 4.0.9
+      '@types/node':
+        specifier: ^22.14.1
+        version: 22.14.1
       clsx:
         specifier: ^2.1.1
         version: 2.1.1
+      dotenv:
+        specifier: ^16.5.0
+        version: 16.5.0
       eslint:
         specifier: ^9.22.0
         version: 9.22.0([email protected])
@@ -137,7 +143,7 @@ importers:
         version: 22.1.0([email protected])
       vite:
         specifier: ^5.4.4
-        version: 5.4.14(@types/node@18.19.84)([email protected])
+        version: 5.4.14(@types/node@22.14.1)([email protected])
 
 packages:
 
@@ -1001,6 +1007,9 @@ packages:
   '@types/[email protected]':
     resolution: {integrity: sha512-ACYy2HGcZPHxEeWTqowTF7dhXN+JU1o7Gr4b41klnn6pj2LD6rsiGqSZojMdk1Jh2ys3m76ap+ae1vvE4+5+vg==}
 
+  '@types/[email protected]':
+    resolution: {integrity: sha512-u0HuPQwe/dHrItgHHpmw3N2fYCR6x4ivMNbPHRkBVP4CvN+kiRrKHWk3i8tXiO/joPwXLMYvF9TTF0eqgHIuOw==}
+
   '@types/[email protected]':
     resolution: {integrity: sha512-60BCwRFOZCQhDncwQdxxeOEEkbc5dIMccYLwbxsS4TUNeVECQ/pBJ0j09mrHOl/JJvpRPGwO9SvE4nR2Nb/a4Q==}
 
@@ -1284,6 +1293,10 @@ packages:
     resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==}
     engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
 
+  [email protected]:
+    resolution: {integrity: sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==}
+    engines: {node: '>=12'}
+
   [email protected]:
     resolution: {integrity: sha512-pYxfDYpued//QpnLIm4Avk7rsNtAtQkUES2cwAYSvD/wd2pKD71gN2Ebj3e7klzXwjocvE8c5vx/1fxwpqmSxA==}
     engines: {node: '>=4'}
@@ -2404,6 +2417,9 @@ packages:
   [email protected]:
     resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}
 
+  [email protected]:
+    resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}
+
   [email protected]:
     resolution: {integrity: sha512-ect2ZNtk1Zgwb0NVHd0C1IDW/MV+Jk/xaq4t8o6rYdVS3+L660ZdD5kTSQZvsgdwCvquRw+/wYn75hsweRjoIA==}
     peerDependencies:
@@ -3087,7 +3103,7 @@ snapshots:
   '@rollup/[email protected]':
     optional: true
 
-  '@ryoppippi/[email protected](@samchon/[email protected])(@types/node@18.19.84)([email protected])([email protected])([email protected])([email protected])':
+  '@ryoppippi/[email protected](@samchon/[email protected])(@types/node@22.14.1)([email protected])([email protected])([email protected])([email protected])':
     dependencies:
       '@rollup/pluginutils': 5.1.4([email protected])
       consola: 3.4.0
@@ -3101,7 +3117,7 @@ snapshots:
       typescript: 5.6.3
       typia: 7.6.4(@samchon/[email protected])([email protected])
       unplugin: 1.16.1
-      vite: 6.2.1(@types/node@18.19.84)([email protected])([email protected])([email protected])
+      vite: 6.2.1(@types/node@22.14.1)([email protected])([email protected])([email protected])
     transitivePeerDependencies:
       - '@samchon/openapi'
       - '@types/node'
@@ -3125,22 +3141,22 @@ snapshots:
     dependencies:
       acorn: 8.14.0
 
-  '@sveltejs/[email protected](@sveltejs/[email protected](@sveltejs/[email protected]([email protected])([email protected](@types/node@18.19.84)([email protected])))([email protected])([email protected](@types/node@18.19.84)([email protected])))':
+  '@sveltejs/[email protected](@sveltejs/[email protected](@sveltejs/[email protected]([email protected])([email protected](@types/node@22.14.1)([email protected])))([email protected])([email protected](@types/node@22.14.1)([email protected])))':
     dependencies:
-      '@sveltejs/kit': 2.18.0(@sveltejs/[email protected]([email protected])([email protected](@types/node@18.19.84)([email protected])))([email protected])([email protected](@types/node@18.19.84)([email protected]))
+      '@sveltejs/kit': 2.18.0(@sveltejs/[email protected]([email protected])([email protected](@types/node@22.14.1)([email protected])))([email protected])([email protected](@types/node@22.14.1)([email protected]))
       import-meta-resolve: 4.1.0
 
-  '@sveltejs/[email protected](@sveltejs/[email protected](@sveltejs/[email protected]([email protected])([email protected](@types/node@18.19.84)([email protected])))([email protected])([email protected](@types/node@18.19.84)([email protected])))':
+  '@sveltejs/[email protected](@sveltejs/[email protected](@sveltejs/[email protected]([email protected])([email protected](@types/node@22.14.1)([email protected])))([email protected])([email protected](@types/node@22.14.1)([email protected])))':
     dependencies:
       '@rollup/plugin-commonjs': 28.0.2([email protected])
       '@rollup/plugin-json': 6.1.0([email protected])
       '@rollup/plugin-node-resolve': 16.0.0([email protected])
-      '@sveltejs/kit': 2.18.0(@sveltejs/[email protected]([email protected])([email protected](@types/node@18.19.84)([email protected])))([email protected])([email protected](@types/node@18.19.84)([email protected]))
+      '@sveltejs/kit': 2.18.0(@sveltejs/[email protected]([email protected])([email protected](@types/node@22.14.1)([email protected])))([email protected])([email protected](@types/node@22.14.1)([email protected]))
       rollup: 4.34.9
 
-  '@sveltejs/[email protected](@sveltejs/[email protected]([email protected])([email protected](@types/node@18.19.84)([email protected])))([email protected])([email protected](@types/node@18.19.84)([email protected]))':
+  '@sveltejs/[email protected](@sveltejs/[email protected]([email protected])([email protected](@types/node@22.14.1)([email protected])))([email protected])([email protected](@types/node@22.14.1)([email protected]))':
     dependencies:
-      '@sveltejs/vite-plugin-svelte': 4.0.4([email protected])([email protected](@types/node@18.19.84)([email protected]))
+      '@sveltejs/vite-plugin-svelte': 4.0.4([email protected])([email protected](@types/node@22.14.1)([email protected]))
       '@types/cookie': 0.6.0
       cookie: 0.6.0
       devalue: 5.1.1
@@ -3153,27 +3169,27 @@ snapshots:
       set-cookie-parser: 2.7.1
       sirv: 3.0.1
       svelte: 5.28.2
-      vite: 5.4.14(@types/node@18.19.84)([email protected])
+      vite: 5.4.14(@types/node@22.14.1)([email protected])
 
-  '@sveltejs/[email protected](@sveltejs/[email protected]([email protected])([email protected](@types/node@18.19.84)([email protected])))([email protected])([email protected](@types/node@18.19.84)([email protected]))':
+  '@sveltejs/[email protected](@sveltejs/[email protected]([email protected])([email protected](@types/node@22.14.1)([email protected])))([email protected])([email protected](@types/node@22.14.1)([email protected]))':
     dependencies:
-      '@sveltejs/vite-plugin-svelte': 4.0.4([email protected])([email protected](@types/node@18.19.84)([email protected]))
+      '@sveltejs/vite-plugin-svelte': 4.0.4([email protected])([email protected](@types/node@22.14.1)([email protected]))
       debug: 4.4.0
       svelte: 5.28.2
-      vite: 5.4.14(@types/node@18.19.84)([email protected])
+      vite: 5.4.14(@types/node@22.14.1)([email protected])
     transitivePeerDependencies:
       - supports-color
 
-  '@sveltejs/[email protected]([email protected])([email protected](@types/node@18.19.84)([email protected]))':
+  '@sveltejs/[email protected]([email protected])([email protected](@types/node@22.14.1)([email protected]))':
     dependencies:
-      '@sveltejs/vite-plugin-svelte-inspector': 3.0.1(@sveltejs/[email protected]([email protected])([email protected](@types/node@18.19.84)([email protected])))([email protected])([email protected](@types/node@18.19.84)([email protected]))
+      '@sveltejs/vite-plugin-svelte-inspector': 3.0.1(@sveltejs/[email protected]([email protected])([email protected](@types/node@22.14.1)([email protected])))([email protected])([email protected](@types/node@22.14.1)([email protected]))
       debug: 4.4.0
       deepmerge: 4.3.1
       kleur: 4.1.5
       magic-string: 0.30.17
       svelte: 5.28.2
-      vite: 5.4.14(@types/node@18.19.84)([email protected])
-      vitefu: 1.0.6([email protected](@types/node@18.19.84)([email protected]))
+      vite: 5.4.14(@types/node@22.14.1)([email protected])
+      vitefu: 1.0.6([email protected](@types/node@22.14.1)([email protected]))
     transitivePeerDependencies:
       - supports-color
 
@@ -3251,13 +3267,17 @@ snapshots:
 
   '@types/[email protected]':
     dependencies:
-      '@types/node': 18.19.84
+      '@types/node': 22.14.1
       form-data: 4.0.2
 
   '@types/[email protected]':
     dependencies:
       undici-types: 5.26.5
 
+  '@types/[email protected]':
+    dependencies:
+      undici-types: 6.21.0
+
   '@types/[email protected]': {}
 
   '@typescript-eslint/[email protected](@typescript-eslint/[email protected]([email protected]([email protected]))([email protected]))([email protected]([email protected]))([email protected])':
@@ -3524,6 +3544,8 @@ snapshots:
 
   [email protected]: {}
 
+  [email protected]: {}
+
   [email protected]: {}
 
   [email protected]:
@@ -4336,7 +4358,7 @@ snapshots:
       '@protobufjs/path': 1.1.2
       '@protobufjs/pool': 1.1.0
       '@protobufjs/utf8': 1.1.0
-      '@types/node': 18.19.84
+      '@types/node': 22.14.1
       long: 5.3.1
 
   [email protected]: {}
@@ -4641,6 +4663,8 @@ snapshots:
 
   [email protected]: {}
 
+  [email protected]: {}
+
   [email protected]([email protected]):
     dependencies:
       '@antfu/install-pkg': 1.0.0
@@ -4669,31 +4693,31 @@ snapshots:
 
   [email protected]: {}
 
-  [email protected](@types/node@18.19.84)([email protected]):
+  [email protected](@types/node@22.14.1)([email protected]):
     dependencies:
       esbuild: 0.21.5
       postcss: 8.5.3
       rollup: 4.34.9
     optionalDependencies:
-      '@types/node': 18.19.84
+      '@types/node': 22.14.1
       fsevents: 2.3.3
       lightningcss: 1.29.1
 
-  [email protected](@types/node@18.19.84)([email protected])([email protected])([email protected]):
+  [email protected](@types/node@22.14.1)([email protected])([email protected])([email protected]):
     dependencies:
       esbuild: 0.25.1
       postcss: 8.5.3
       rollup: 4.34.9
     optionalDependencies:
-      '@types/node': 18.19.84
+      '@types/node': 22.14.1
       fsevents: 2.3.3
       jiti: 2.4.2
       lightningcss: 1.29.1
       yaml: 2.7.0
 
-  [email protected]([email protected](@types/node@18.19.84)([email protected])):
+  [email protected]([email protected](@types/node@22.14.1)([email protected])):
     optionalDependencies:
-      vite: 5.4.14(@types/node@18.19.84)([email protected])
+      vite: 5.4.14(@types/node@22.14.1)([email protected])
 
   [email protected]:
     dependencies:

scripts/update-ctx-length.ts

@@ -0,0 +1,55 @@
+import dotenv from "dotenv";
+dotenv.config(); // Load .env file into process.env
+
+import { fetchAllProviderData, type ApiKeys } from "../src/lib/server/providers/index.js"; // Import ApiKeys type
+import fs from "fs/promises";
+import path from "path";
+
+const CACHE_FILE_PATH = path.resolve("src/lib/data/context_length.json");
+
+async function runUpdate() {
+	console.log("Starting context length cache update...");
+
+	// Gather API keys from process.env
+	const apiKeys: ApiKeys = {
+		COHERE_API_KEY: process.env.COHERE_API_KEY,
+		TOGETHER_API_KEY: process.env.TOGETHER_API_KEY,
+		FIREWORKS_API_KEY: process.env.FIREWORKS_API_KEY,
+		HYPERBOLIC_API_KEY: process.env.HYPERBOLIC_API_KEY,
+		REPLICATE_API_KEY: process.env.REPLICATE_API_KEY,
+		NEBIUS_API_KEY: process.env.NEBIUS_API_KEY,
+		NOVITA_API_KEY: process.env.NOVITA_API_KEY,
+		SAMBANOVA_API_KEY: process.env.SAMBANOVA_API_KEY,
+	};
+
+	try {
+		// Fetch data from all supported providers concurrently, passing keys
+		const fetchedData = await fetchAllProviderData(apiKeys);
+
+		// Read existing manual/cached data
+		let existingData = {};
+		try {
+			const currentCache = await fs.readFile(CACHE_FILE_PATH, "utf-8");
+			existingData = JSON.parse(currentCache);
+		} catch {
+			// Remove unused variable name
+			console.log("No existing cache file found or error reading, creating new one.");
+		}
+
+		// Merge fetched data with existing data (fetched data takes precedence)
+		const combinedData = { ...existingData, ...fetchedData };
+
+		// Write the combined data back to the file
+		const tempFilePath = CACHE_FILE_PATH + ".tmp";
+		await fs.writeFile(tempFilePath, JSON.stringify(combinedData, null, "\t"), "utf-8");
+		await fs.rename(tempFilePath, CACHE_FILE_PATH);
+
+		console.log("Context length cache update complete.");
+		console.log(`Cache file written to: ${CACHE_FILE_PATH}`);
+	} catch (error) {
+		console.error("Error during context length cache update:", error);
+		process.exit(1); // Exit with error code
+	}
+}
+
+runUpdate();

src/app.css

@@ -67,6 +67,18 @@
 	@apply flex h-[39px] items-center justify-center gap-2 rounded-lg border border-gray-200 bg-white px-3 py-2.5 text-sm font-medium text-gray-900 hover:bg-gray-100 hover:text-blue-700 focus:ring-4 focus:ring-gray-100 focus:outline-hidden dark:border-gray-600 dark:bg-gray-800 dark:text-gray-400 dark:hover:bg-gray-700 dark:hover:text-white dark:focus:ring-gray-700;
 }
 
+@utility btn-sm {
+	@apply flex h-[32px] items-center justify-center gap-1.5 rounded-md border border-gray-200 bg-white px-2.5 py-2 text-xs font-medium text-gray-900 hover:bg-gray-100 hover:text-blue-700 focus:ring-4 focus:ring-gray-100 focus:outline-hidden dark:border-gray-600 dark:bg-gray-800 dark:text-gray-400 dark:hover:bg-gray-700 dark:hover:text-white dark:focus:ring-gray-700;
+}
+
+@utility btn-xs {
+	@apply flex h-[28px] items-center justify-center gap-1 rounded border border-gray-200 bg-white px-2 py-1.5 text-xs font-medium text-gray-900 hover:bg-gray-100 hover:text-blue-700 focus:ring-4 focus:ring-gray-100 focus:outline-hidden dark:border-gray-600 dark:bg-gray-800 dark:text-gray-400 dark:hover:bg-gray-700 dark:hover:text-white dark:focus:ring-gray-700;
+}
+
+@utility btn-mini {
+	@apply flex h-[24px] items-center justify-center gap-0.5 rounded-sm border border-gray-200 bg-white px-1.5 py-1 text-[10px] font-medium text-gray-900 hover:bg-gray-100 hover:text-blue-700 focus:ring-2 focus:ring-gray-100 focus:outline-hidden dark:border-gray-600 dark:bg-gray-800 dark:text-gray-400 dark:hover:bg-gray-700 dark:hover:text-white dark:focus:ring-gray-700;
+}
+
 @utility custom-outline {
 	@apply outline-hidden;
 	@apply border-blue-500 ring ring-blue-500;

src/lib/components/inference-playground/generation-config-settings.ts

@@ -7,7 +7,7 @@ export type GenerationConfigKey = (typeof GENERATION_CONFIG_KEYS)[number];
 export type GenerationConfig = Pick<ChatCompletionInput, GenerationConfigKey>;
 
 interface GenerationKeySettings {
-	default: number;
+	default?: number;
 	step: number;
 	min: number;
 	max: number;
@@ -23,7 +23,6 @@ export const GENERATION_CONFIG_SETTINGS: Record<GenerationConfigKey, GenerationK
 		label: "Temperature",
 	},
 	max_tokens: {
-		default: 2048,
 		step: 256,
 		min: 0,
 		max: 8192, // changed dynamically based on model

src/lib/components/inference-playground/generation-config.svelte

@@ -1,8 +1,10 @@
 <script lang="ts">
-	import type { Conversation } from "$lib/types.js";
-
+	import { type Conversation } from "$lib/types.js";
+	import { watch } from "runed";
 	import { GENERATION_CONFIG_KEYS, GENERATION_CONFIG_SETTINGS } from "./generation-config-settings.js";
-	import { customMaxTokens } from "./utils.js";
+	import { maxAllowedTokens } from "./utils.js";
+	import { isNumber } from "$lib/utils/is.js";
+	import IconX from "~icons/carbon/close";
 
 	interface Props {
 		conversation: Conversation;
@@ -11,37 +13,58 @@
 
 	let { conversation = $bindable(), classNames = "" }: Props = $props();
 
-	let modelMaxLength = $derived(customMaxTokens[conversation.model.id] ?? 100000);
-	let maxTokens = $derived(Math.min(modelMaxLength ?? GENERATION_CONFIG_SETTINGS["max_tokens"].max, 64_000));
+	const maxTokens = $derived(maxAllowedTokens(conversation));
+
+	watch(
+		() => maxTokens,
+		() => {
+			const curr = conversation.config.max_tokens;
+			if (!curr || curr <= maxTokens) return;
+			conversation.config.max_tokens = maxTokens;
+		}
+	);
 </script>
 
 <div class="flex flex-col gap-y-7 {classNames}">
 	{#each GENERATION_CONFIG_KEYS as key}
 		{@const { label, min, step } = GENERATION_CONFIG_SETTINGS[key]}
-		{@const max = key === "max_tokens" ? maxTokens : GENERATION_CONFIG_SETTINGS[key].max}
+		{@const isMaxTokens = key === "max_tokens"}
+		{@const max = isMaxTokens ? maxTokens : GENERATION_CONFIG_SETTINGS[key].max}
+
 		<div>
 			<div class="flex items-center justify-between">
-				<label for="temperature-range" class="mb-2 block text-sm font-medium text-gray-900 dark:text-white"
-					>{label}</label
-				>
+				<label for={key} class="mb-2 block text-sm font-medium text-gray-900 dark:text-white">
+					{label}
+				</label>
+				<div class="flex items-center gap-2">
+					{#if !isMaxTokens || isNumber(conversation.config[key])}
+						<input
+							type="number"
+							class="w-20 rounded-sm border bg-transparent px-1 py-0.5 text-right text-sm dark:border-gray-700"
+							{min}
+							{max}
+							{step}
+							bind:value={conversation.config[key]}
+						/>
+					{/if}
+					{#if isMaxTokens && isNumber(conversation.config[key])}
+						<button class="btn-mini" onclick={() => (conversation.config[key] = undefined)}> <IconX /> </button>
+					{:else if isMaxTokens}
+						<button class="btn-mini" onclick={() => (conversation.config[key] = maxTokens / 2)}> set </button>
+					{/if}
+				</div>
+			</div>
+			{#if !isMaxTokens || isNumber(conversation.config[key])}
 				<input
-					type="number"
-					class="w-18 rounded-sm border bg-transparent px-1 py-0.5 text-right text-sm dark:border-gray-700"
+					id={key}
+					type="range"
 					{min}
 					{max}
 					{step}
 					bind:value={conversation.config[key]}
+					class="h-2 w-full cursor-pointer appearance-none rounded-lg bg-gray-200 accent-black dark:bg-gray-700 dark:accent-blue-500"
 				/>
-			</div>
-			<input
-				id="temperature-range"
-				type="range"
-				{min}
-				{max}
-				{step}
-				bind:value={conversation.config[key]}
-				class="h-2 w-full cursor-pointer appearance-none rounded-lg bg-gray-200 accent-black dark:bg-gray-700 dark:accent-blue-500"
-			/>
+			{/if}
 		</div>
 	{/each}
 

src/lib/components/inference-playground/playground.svelte

@@ -6,7 +6,6 @@
 	import { isConversationWithHFModel, type ConversationMessage, type Model, type Project } from "$lib/types.js";
 	import { cmdOrCtrl, optOrAlt } from "$lib/utils/platform.js";
 	import { Popover } from "melt/components";
-	import { watch } from "runed";
 	import typia from "typia";
 	import { default as IconDelete } from "~icons/carbon/trash-can";
 	import { showShareModal } from "../share-modal.svelte";
@@ -19,7 +18,7 @@
 	import ModelSelectorModal from "./model-selector-modal.svelte";
 	import ModelSelector from "./model-selector.svelte";
 	import ProjectSelect from "./project-select.svelte";
-	import { getTokens, isSystemPromptSupported } from "./utils.js";
+	import { isSystemPromptSupported } from "./utils.js";
 
 	import { iterate } from "$lib/utils/array.js";
 	import IconChatLeft from "~icons/carbon/align-box-bottom-left";
@@ -43,19 +42,6 @@
 
 	let selectCompareModelOpen = $state(false);
 
-	watch(
-		() => $state.snapshot(session.project),
-		() => {
-			session.project.conversations.forEach(async (c, i) => {
-				session.generationStats[i] = {
-					latency: 0,
-					...session.generationStats[i],
-					generatedTokensCount: await getTokens(c),
-				};
-			});
-		}
-	);
-
 	const systemPromptSupported = $derived(
 		session.project.conversations.some(conversation => isSystemPromptSupported(conversation.model))
 	);

src/lib/components/inference-playground/utils.ts

@@ -1,15 +1,18 @@
-import { AutoTokenizer, PreTrainedTokenizer } from "@huggingface/transformers";
+import ctxLengthData from "$lib/data/context_length.json";
+import { token } from "$lib/state/token.svelte";
 import {
 	isCustomModel,
+	isHFModel,
 	type Conversation,
 	type ConversationMessage,
 	type CustomModel,
 	type Model,
 } from "$lib/types.js";
+import { tryGet } from "$lib/utils/object.js";
+import { HfInference, snippets, type InferenceProvider } from "@huggingface/inference";
 import type { ChatCompletionInputMessage, InferenceSnippet } from "@huggingface/tasks";
 import { type ChatCompletionOutputMessage } from "@huggingface/tasks";
-import { token } from "$lib/state/token.svelte";
-import { HfInference, snippets, type InferenceProvider } from "@huggingface/inference";
+import { AutoTokenizer, PreTrainedTokenizer } from "@huggingface/transformers";
 import OpenAI from "openai";
 
 type ChatCompletionInputMessageChunk =
@@ -48,6 +51,24 @@ type OpenAICompletionMetadata = {
 
 type CompletionMetadata = HFCompletionMetadata | OpenAICompletionMetadata;
 
+export function maxAllowedTokens(conversation: Conversation) {
+	const ctxLength = (() => {
+		const { provider, model } = conversation;
+		if (!provider || !isHFModel(model)) return;
+
+		const idOnProvider = model.inferenceProviderMapping.find(data => data.provider === provider)?.providerId;
+		if (!idOnProvider) return;
+
+		const models = tryGet(ctxLengthData, provider);
+		if (!models) return;
+
+		return tryGet(models, idOnProvider) as number | undefined;
+	})();
+
+	if (!ctxLength) return customMaxTokens[conversation.model.id] ?? 100000;
+	return ctxLength;
+}
+
 function getCompletionMetadata(conversation: Conversation, signal?: AbortSignal): CompletionMetadata {
 	const { model, systemMessage } = conversation;
 
@@ -88,6 +109,7 @@ function getCompletionMetadata(conversation: Conversation, signal?: AbortSignal)
 			messages: messages.map(parseMessage),
 			provider: conversation.provider,
 			...conversation.config,
+			// max_tokens: maxAllowedTokens(conversation) - currTokens,
 		},
 	};
 }
@@ -284,11 +306,20 @@ export async function getTokenizer(model: Model) {
 	}
 }
 
+// When you don't have access to a tokenizer, guesstimate
+export function estimateTokens(conversation: Conversation) {
+	const content = conversation.messages.reduce((acc, curr) => {
+		return acc + (curr?.content ?? "");
+	}, "");
+
+	return content.length / 4; // 1 token ~ 4 characters
+}
+
 export async function getTokens(conversation: Conversation): Promise<number> {
 	const model = conversation.model;
-	if (isCustomModel(model)) return 0;
+	if (isCustomModel(model)) return estimateTokens(conversation);
 	const tokenizer = await getTokenizer(model);
-	if (tokenizer === null) return 0;
+	if (tokenizer === null) return estimateTokens(conversation);
 
 	// This is a simplified version - you might need to adjust based on your exact needs
 	let formattedText = "";

src/lib/data/context_length.json

@@ -0,0 +1,299 @@
+{
+	"replicate": {},
+	"sambanova": {
+		"DeepSeek-R1": 16384,
+		"DeepSeek-R1-Distill-Llama-70B": 131072,
+		"DeepSeek-V3-0324": 16384,
+		"E5-Mistral-7B-Instruct": 4096,
+		"Llama-4-Maverick-17B-128E-Instruct": 8192,
+		"Llama-4-Scout-17B-16E-Instruct": 8192,
+		"Meta-Llama-3.1-405B-Instruct": 16384,
+		"Meta-Llama-3.1-8B-Instruct": 16384,
+		"Meta-Llama-3.2-1B-Instruct": 16384,
+		"Meta-Llama-3.2-3B-Instruct": 4096,
+		"Meta-Llama-3.3-70B-Instruct": 131072,
+		"Meta-Llama-Guard-3-8B": 16384,
+		"QwQ-32B": 16384,
+		"Qwen2-Audio-7B-Instruct": 4096,
+		"Qwen3-32B": 8192
+	},
+	"nebius": {
+		"meta-llama/Meta-Llama-3.1-8B-Instruct-fast": 131072,
+		"meta-llama/Meta-Llama-3.1-8B-Instruct": 131072,
+		"meta-llama/Meta-Llama-3.1-70B-Instruct-fast": 131072,
+		"meta-llama/Meta-Llama-3.1-70B-Instruct": 131072,
+		"meta-llama/Meta-Llama-3.1-405B-Instruct": 131072,
+		"meta-llama/Llama-Guard-3-8B": 131072,
+		"nvidia/Llama-3.1-Nemotron-70B-Instruct-HF-fast": 131072,
+		"nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": 131072,
+		"nvidia/Llama-3_1-Nemotron-Ultra-253B-v1": 131072,
+		"mistralai/Mistral-Nemo-Instruct-2407-fast": 128000,
+		"mistralai/Mistral-Nemo-Instruct-2407": 128000,
+		"mistralai/Mixtral-8x7B-Instruct-v0.1-fast": 32768,
+		"mistralai/Mixtral-8x7B-Instruct-v0.1": 32768,
+		"mistralai/Mixtral-8x22B-Instruct-v0.1-fast": 65536,
+		"mistralai/Mixtral-8x22B-Instruct-v0.1": 65536,
+		"allenai/OLMo-7B-Instruct-hf": 2048,
+		"microsoft/Phi-3-mini-4k-instruct-fast": 4096,
+		"microsoft/Phi-3-mini-4k-instruct": 4096,
+		"microsoft/Phi-3-medium-128k-instruct-fast": 131072,
+		"microsoft/Phi-3-medium-128k-instruct": 131072,
+		"google/gemma-2-2b-it-fast": 8192,
+		"google/gemma-2-2b-it": 8192,
+		"google/gemma-2-9b-it-fast": 8192,
+		"google/gemma-2-9b-it": 8192,
+		"google/gemma-2-27b-it-fast": 8192,
+		"google/gemma-2-27b-it": 8192,
+		"deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct-fast": 128000,
+		"deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 128000,
+		"Qwen/Qwen2.5-Coder-7B-fast": 32768,
+		"Qwen/Qwen2.5-Coder-7B": 32768,
+		"Qwen/Qwen2.5-Coder-7B-Instruct-fast": 32768,
+		"Qwen/Qwen2.5-Coder-7B-Instruct": 32768,
+		"Qwen/Qwen2.5-Coder-32B-Instruct-fast": 131072,
+		"Qwen/Qwen2.5-Coder-32B-Instruct": 131072,
+		"Qwen/Qwen2.5-32B-Instruct-fast": 131072,
+		"Qwen/Qwen2.5-32B-Instruct": 131072,
+		"Qwen/Qwen2.5-72B-Instruct-fast": 131072,
+		"Qwen/Qwen2.5-72B-Instruct": 131072,
+		"Qwen/Qwen2-VL-72B-Instruct": 32768,
+		"Qwen/Qwen2-VL-7B-Instruct": 32768,
+		"llava-hf/llava-1.5-7b-hf": 4096,
+		"llava-hf/llava-1.5-13b-hf": 4096,
+		"aaditya/Llama3-OpenBioLLM-8B": 8192,
+		"aaditya/Llama3-OpenBioLLM-70B": 8192,
+		"BAAI/bge-en-icl": 32768,
+		"BAAI/bge-multilingual-gemma2": 4096,
+		"intfloat/e5-mistral-7b-instruct": 32768,
+		"cognitivecomputations/dolphin-2.9.2-mixtral-8x22b": 65536,
+		"microsoft/Phi-3.5-MoE-instruct": 131072,
+		"microsoft/Phi-3.5-mini-instruct": 131072,
+		"Qwen/Qwen2.5-1.5B-Instruct": 32768,
+		"meta-llama/Llama-3.3-70B-Instruct": 131072,
+		"meta-llama/Llama-3.3-70B-Instruct-fast": 131072,
+		"meta-llama/Llama-3.2-1B-Instruct": 131072,
+		"meta-llama/Llama-3.2-3B-Instruct": 131072,
+		"Qwen/QwQ-32B-Preview": 32768,
+		"Qwen/QVQ-72B-preview": 128000,
+		"microsoft/phi-4": 16384,
+		"deepseek-ai/DeepSeek-V3": 163840,
+		"deepseek-ai/DeepSeek-R1": 163840,
+		"NousResearch/Hermes-3-Llama-405B": 131072,
+		"deepseek-ai/DeepSeek-R1-Distill-Llama-70B": 131072,
+		"deepseek-ai/DeepSeek-R1-fast": 163840,
+		"Qwen/QwQ-32B-fast": 131072,
+		"Qwen/QwQ-32B": 131072,
+		"nvidia/Llama-3_3-Nemotron-Super-49B-v1": 131072,
+		"mistralai/Mistral-Small-3.1-24B-Instruct-2503": 131072,
+		"google/gemma-3-27b-it": 131072,
+		"google/gemma-3-27b-it-fast": 131072,
+		"Qwen/Qwen2.5-VL-72B-Instruct": 32000,
+		"deepseek-ai/DeepSeek-V3-0324": 163840,
+		"deepseek-ai/DeepSeek-V3-0324-fast": 163840,
+		"black-forest-labs/flux-dev": 0,
+		"black-forest-labs/flux-schnell": 0,
+		"stability-ai/sdxl": 0
+	},
+	"novita": {
+		"deepseek/deepseek-prover-v2-671b": 160000,
+		"qwen/qwen3-235b-a22b-fp8": 128000,
+		"qwen/qwen3-30b-a3b-fp8": 128000,
+		"qwen/qwen3-32b-fp8": 128000,
+		"deepseek/deepseek-v3-0324": 128000,
+		"qwen/qwen2.5-vl-72b-instruct": 96000,
+		"deepseek/deepseek-v3-turbo": 64000,
+		"deepseek/deepseek-r1-turbo": 64000,
+		"meta-llama/llama-4-maverick-17b-128e-instruct-fp8": 1048576,
+		"google/gemma-3-27b-it": 32000,
+		"qwen/qwq-32b": 32768,
+		"Sao10K/L3-8B-Stheno-v3.2": 8192,
+		"gryphe/mythomax-l2-13b": 4096,
+		"meta-llama/llama-4-scout-17b-16e-instruct": 131072,
+		"deepseek/deepseek-r1-distill-llama-8b": 32000,
+		"deepseek/deepseek_v3": 64000,
+		"meta-llama/llama-3.1-8b-instruct": 16384,
+		"deepseek/deepseek-r1-distill-qwen-14b": 64000,
+		"meta-llama/llama-3.3-70b-instruct": 131072,
+		"qwen/qwen-2.5-72b-instruct": 32000,
+		"mistralai/mistral-nemo": 131072,
+		"deepseek/deepseek-r1-distill-qwen-32b": 64000,
+		"meta-llama/llama-3-8b-instruct": 8192,
+		"microsoft/wizardlm-2-8x22b": 65535,
+		"deepseek/deepseek-r1-distill-llama-70b": 32000,
+		"meta-llama/llama-3.1-70b-instruct": 32768,
+		"google/gemma-2-9b-it": 8192,
+		"mistralai/mistral-7b-instruct": 32768,
+		"meta-llama/llama-3-70b-instruct": 8192,
+		"deepseek/deepseek-r1": 64000,
+		"nousresearch/hermes-2-pro-llama-3-8b": 8192,
+		"sao10k/l3-70b-euryale-v2.1": 8192,
+		"cognitivecomputations/dolphin-mixtral-8x22b": 16000,
+		"jondurbin/airoboros-l2-70b": 4096,
+		"sophosympatheia/midnight-rose-70b": 4096,
+		"sao10k/l3-8b-lunaris": 8192,
+		"qwen/qwen3-0.6b-fp8": 32000,
+		"qwen/qwen3-1.7b-fp8": 32000,
+		"qwen/qwen3-8b-fp8": 128000,
+		"qwen/qwen3-4b-fp8": 128000,
+		"qwen/qwen3-14b-fp8": 128000,
+		"thudm/glm-4-9b-0414": 32000,
+		"thudm/glm-z1-9b-0414": 32000,
+		"thudm/glm-z1-32b-0414": 32000,
+		"thudm/glm-4-32b-0414": 32000,
+		"thudm/glm-z1-rumination-32b-0414": 32000,
+		"qwen/qwen2.5-7b-instruct": 32000,
+		"meta-llama/llama-3.2-1b-instruct": 131000,
+		"meta-llama/llama-3.2-11b-vision-instruct": 32768,
+		"meta-llama/llama-3.2-3b-instruct": 32768,
+		"meta-llama/llama-3.1-8b-instruct-bf16": 8192,
+		"sao10k/l31-70b-euryale-v2.2": 8192
+	},
+	"fal": {
+		"fal/model-name": 4096
+	},
+	"cerebras": {
+		"cerebras/model-name": 8192
+	},
+	"hf-inference": {
+		"google/gemma-2-9b-it": 8192,
+		"meta-llama/Meta-Llama-3-8B-Instruct": 8192
+	},
+	"hyperbolic": {
+		"Qwen/Qwen2.5-72B-Instruct": 131072,
+		"Qwen/Qwen2.5-VL-72B-Instruct": 32768,
+		"meta-llama/Meta-Llama-3-70B-Instruct": 8192,
+		"deepseek-ai/DeepSeek-V3": 131072,
+		"deepseek-ai/DeepSeek-V3-0324": 163840,
+		"meta-llama/Llama-3.3-70B-Instruct": 131072,
+		"Qwen/QwQ-32B-Preview": 32768,
+		"Qwen/Qwen2.5-Coder-32B-Instruct": 32768,
+		"meta-llama/Llama-3.2-3B-Instruct": 131072,
+		"NousResearch/Hermes-3-Llama-3.1-70B": 12288,
+		"meta-llama/Meta-Llama-3.1-405B-Instruct": 131000,
+		"meta-llama/Meta-Llama-3.1-70B-Instruct": 131072,
+		"meta-llama/Meta-Llama-3.1-8B-Instruct": 131072,
+		"mistralai/Pixtral-12B-2409": 32768,
+		"Qwen/Qwen2.5-VL-7B-Instruct": 32768,
+		"meta-llama/Meta-Llama-3.1-405B": 32768,
+		"meta-llama/Meta-Llama-3.1-405B-FP8": 32768,
+		"deepseek-ai/DeepSeek-R1": 163840,
+		"Qwen/QwQ-32B": 131072
+	},
+	"cohere": {
+		"embed-english-light-v3.0": 512,
+		"embed-multilingual-v2.0": 256,
+		"rerank-v3.5": 4096,
+		"embed-v4.0": 8192,
+		"rerank-english-v3.0": 4096,
+		"command-r": 128000,
+		"embed-english-light-v3.0-image": 0,
+		"embed-english-v3.0-image": 0,
+		"command-a-03-2025": 288000,
+		"command-nightly": 288000,
+		"command-r7b-12-2024": 128000,
+		"command-r-plus": 128000,
+		"c4ai-aya-vision-32b": 16384,
+		"command-r7b-arabic-02-2025": 128000,
+		"command-light-nightly": 4096,
+		"embed-english-v3.0": 512,
+		"embed-multilingual-light-v3.0-image": 0,
+		"embed-multilingual-v3.0-image": 0,
+		"c4ai-aya-expanse-32b": 128000,
+		"command": 4096,
+		"c4ai-aya-vision-8b": 16384
+	},
+	"together": {
+		"meta-llama/Llama-3.3-70B-Instruct-Turbo": 131072,
+		"togethercomputer/m2-bert-80M-32k-retrieval": 32768,
+		"google/gemma-2-9b-it": 8192,
+		"cartesia/sonic": 0,
+		"Qwen/Qwen2.5-7B-Instruct-Turbo": 32768,
+		"deepseek-ai/DeepSeek-R1-Distill-Llama-70B-free": 8192,
+		"meta-llama-llama-2-70b-hf": 4096,
+		"BAAI/bge-base-en-v1.5": 512,
+		"Gryphe/MythoMax-L2-13b": 4096,
+		"google/gemma-2-27b-it": 8192,
+		"Qwen/Qwen2-VL-72B-Instruct": 32768,
+		"meta-llama/LlamaGuard-2-8b": 8192,
+		"cartesia/sonic-2": 0,
+		"togethercomputer/m2-bert-80M-8k-retrieval": 8192,
+		"meta-llama/Llama-3.3-70B-Instruct-Turbo-Free": 131072,
+		"arcee-ai/maestro-reasoning": 131072,
+		"Qwen/QwQ-32B": 131072,
+		"togethercomputer/MoA-1": 32768,
+		"mistralai/Mistral-7B-Instruct-v0.2": 32768,
+		"meta-llama/Meta-Llama-3-70B-Instruct-Turbo": 8192,
+		"google/gemma-2b-it": 8192,
+		"mistralai/Mistral-Small-24B-Instruct-2501": 32768,
+		"Gryphe/MythoMax-L2-13b-Lite": 4096,
+		"meta-llama/Meta-Llama-Guard-3-8B": 8192,
+		"scb10x/scb10x-llama3-1-typhoon2-8b-instruct": 8192,
+		"Qwen/Qwen3-235B-A22B-fp8-tput": 40960,
+		"meta-llama/Llama-3-8b-chat-hf": 8192,
+		"arcee-ai/caller": 32768,
+		"togethercomputer/MoA-1-Turbo": 32768,
+		"mistralai/Mistral-7B-Instruct-v0.1": 32768,
+		"NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO": 32768,
+		"scb10x/scb10x-llama3-1-typhoon2-70b-instruct": 8192,
+		"meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": 131072,
+		"meta-llama/Llama-3.2-11B-Vision-Instruct-Turbo": 131072,
+		"deepseek-ai/DeepSeek-R1-Distill-Llama-70B": 131072,
+		"arcee-ai/virtuoso-medium-v2": 131072,
+		"arcee-ai/coder-large": 32768,
+		"arcee-ai/virtuoso-large": 131072,
+		"deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B": 131072,
+		"meta-llama/Meta-Llama-3-8B-Instruct-Lite": 8192,
+		"meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo": 131072,
+		"meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": 1048576,
+		"meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": 131072,
+		"mistralai/Mixtral-8x7B-v0.1": 32768,
+		"meta-llama/Llama-4-Scout-17B-16E-Instruct": 1048576,
+		"meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo": 130815,
+		"deepseek-ai/DeepSeek-R1": 163840,
+		"arcee-ai/arcee-blitz": 32768,
+		"deepseek-ai/DeepSeek-V3-p-dp": 131072,
+		"deepseek-ai/DeepSeek-R1-Distill-Qwen-14B": 131072,
+		"deepseek-ai/DeepSeek-V3": 131072,
+		"Qwen/Qwen2.5-Coder-32B-Instruct": 16384,
+		"Qwen/Qwen2-72B-Instruct": 32768,
+		"mistralai/Mixtral-8x7B-Instruct-v0.1": 32768,
+		"meta-llama/Llama-3-70b-chat-hf": 8192,
+		"mistralai/Mistral-7B-Instruct-v0.3": 32768,
+		"Salesforce/Llama-Rank-V1": 8192,
+		"nvidia/Llama-3.1-Nemotron-70B-Instruct-HF": 32768,
+		"meta-llama/Llama-Vision-Free": 131072,
+		"meta-llama/Llama-Guard-3-11B-Vision-Turbo": 131072,
+		"meta-llama/Llama-3.2-3B-Instruct-Turbo": 131072,
+		"Qwen/Qwen2.5-72B-Instruct-Turbo": 131072,
+		"arcee_ai/arcee-spotlight": 131072,
+		"meta-llama/Llama-2-70b-hf": 4096,
+		"Qwen/Qwen2.5-VL-72B-Instruct": 32768
+	},
+	"fireworks-ai": {
+		"accounts/fireworks/models/qwq-32b": 131072,
+		"accounts/fireworks/models/qwen2-vl-72b-instruct": 32768,
+		"accounts/fireworks/models/deepseek-v3": 131072,
+		"accounts/fireworks/models/llama-v3p1-8b-instruct": 131072,
+		"accounts/fireworks/models/llama-v3p1-70b-instruct": 131072,
+		"accounts/fireworks/models/llama-v3p2-90b-vision-instruct": 131072,
+		"accounts/fireworks/models/llama-v3-70b-instruct": 8192,
+		"accounts/fireworks/models/deepseek-v3-0324": 163840,
+		"accounts/fireworks/models/qwen2p5-vl-32b-instruct": 128000,
+		"accounts/fireworks/models/llama4-maverick-instruct-basic": 1048576,
+		"accounts/fireworks/models/qwen3-30b-a3b": 131072,
+		"accounts/fireworks/models/llama4-scout-instruct-basic": 1048576,
+		"accounts/fireworks/models/deepseek-r1-basic": 163840,
+		"accounts/fireworks/models/qwen-qwq-32b-preview": 32768,
+		"accounts/fireworks/models/phi-3-vision-128k-instruct": 32064,
+		"accounts/fireworks/models/firesearch-ocr-v6": 131072,
+		"accounts/fireworks/models/llama-v3p3-70b-instruct": 131072,
+		"accounts/fireworks/models/deepseek-r1": 163840,
+		"accounts/yi-01-ai/models/yi-large": 32768,
+		"accounts/fireworks/models/llama-v3p1-405b-instruct": 131072,
+		"accounts/fireworks/models/llama-guard-3-8b": 131072,
+		"accounts/sentientfoundation/models/dobby-unhinged-llama-3-3-70b-new": 131072,
+		"accounts/fireworks/models/mixtral-8x22b-instruct": 65536,
+		"accounts/fireworks/models/qwen2p5-72b-instruct": 32768,
+		"accounts/perplexity/models/r1-1776": 163840
+	}
+}

src/lib/server/providers/cohere.ts

@@ -0,0 +1,35 @@
+import type { MaxTokensCache } from "./index.js";
+
+const COHERE_API_URL = "https://api.cohere.ai/v1/models";
+
+// Accept apiKey as an argument
+export async function fetchCohereData(apiKey: string | undefined): Promise<MaxTokensCache["cohere"]> {
+	if (!apiKey) {
+		console.warn("Cohere API key not provided. Skipping Cohere fetch.");
+		return {};
+	}
+	try {
+		const response = await fetch(COHERE_API_URL, {
+			headers: {
+				Authorization: `Bearer ${apiKey}`, // Use passed-in apiKey
+			},
+		});
+		if (!response.ok) {
+			throw new Error(`Cohere API request failed: ${response.status} ${response.statusText}`);
+		}
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		const data: any = await response.json();
+		const modelsData: MaxTokensCache["cohere"] = {};
+		if (data?.models && Array.isArray(data.models)) {
+			for (const model of data.models) {
+				if (model.name && typeof model.context_length === "number") {
+					modelsData[model.name] = model.context_length;
+				}
+			}
+		}
+		return modelsData;
+	} catch (error) {
+		console.error("Error fetching Cohere data:", error);
+		return {};
+	}
+}

src/lib/server/providers/fireworks.ts

@@ -0,0 +1,41 @@
+import type { MaxTokensCache } from "./index.js";
+
+const FIREWORKS_API_URL = "https://api.fireworks.ai/inference/v1/models"; // Assumed
+
+export async function fetchFireworksData(apiKey: string | undefined): Promise<MaxTokensCache["fireworks-ai"]> {
+	if (!apiKey) {
+		console.warn("Fireworks AI API key not provided. Skipping Fireworks AI fetch.");
+		return {};
+	}
+	try {
+		const response = await fetch(FIREWORKS_API_URL, {
+			headers: {
+				Authorization: `Bearer ${apiKey}`,
+			},
+		});
+		if (!response.ok) {
+			throw new Error(`Fireworks AI API request failed: ${response.status} ${response.statusText}`);
+		}
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		const data: any = await response.json(); // Assuming OpenAI structure { data: [ { id: string, ... } ] }
+		const modelsData: MaxTokensCache["fireworks-ai"] = {};
+
+		// Check if data and data.data exist and are an array
+		if (data?.data && Array.isArray(data.data)) {
+			for (const model of data.data) {
+				// Check for common context length fields (OpenAI uses context_window)
+				const contextLength = model.context_length ?? model.context_window ?? model.config?.max_tokens ?? null;
+				// Fireworks uses model.id
+				if (model.id && typeof contextLength === "number") {
+					modelsData[model.id] = contextLength;
+				}
+			}
+		} else {
+			console.warn("Unexpected response structure from Fireworks AI API:", data);
+		}
+		return modelsData;
+	} catch (error) {
+		console.error("Error fetching Fireworks AI data:", error);
+		return {}; // Return empty on error
+	}
+}

src/lib/server/providers/hyperbolic.ts

@@ -0,0 +1,41 @@
+import type { MaxTokensCache } from "./index.js";
+
+const HYPERBOLIC_API_URL = "https://api.hyperbolic.xyz/v1/models"; // Assumed
+
+export async function fetchHyperbolicData(apiKey: string | undefined): Promise<MaxTokensCache["hyperbolic"]> {
+	if (!apiKey) {
+		console.warn("Hyperbolic API key not provided. Skipping Hyperbolic fetch.");
+		return {};
+	}
+	try {
+		const response = await fetch(HYPERBOLIC_API_URL, {
+			headers: {
+				Authorization: `Bearer ${apiKey}`,
+			},
+		});
+		if (!response.ok) {
+			throw new Error(`Hyperbolic API request failed: ${response.status} ${response.statusText}`);
+		}
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		const data: any = await response.json(); // Assuming OpenAI structure { data: [ { id: string, ... } ] }
+		const modelsData: MaxTokensCache["hyperbolic"] = {};
+
+		// Check if data and data.data exist and are an array
+		if (data?.data && Array.isArray(data.data)) {
+			for (const model of data.data) {
+				// Check for common context length fields (OpenAI uses context_window)
+				const contextLength = model.context_length ?? model.context_window ?? model.config?.max_tokens ?? null;
+				// Assuming Hyperbolic uses model.id
+				if (model.id && typeof contextLength === "number") {
+					modelsData[model.id] = contextLength;
+				}
+			}
+		} else {
+			console.warn("Unexpected response structure from Hyperbolic API:", data);
+		}
+		return modelsData;
+	} catch (error) {
+		console.error("Error fetching Hyperbolic data:", error);
+		return {}; // Return empty on error
+	}
+}

src/lib/server/providers/index.ts

@@ -0,0 +1,224 @@
+import fs from "fs/promises";
+import path from "path";
+import { fetchCohereData } from "./cohere.js";
+import { fetchTogetherData } from "./together.js";
+import { fetchFireworksData } from "./fireworks.js";
+import { fetchHyperbolicData } from "./hyperbolic.js";
+import { fetchReplicateData } from "./replicate.js";
+import { fetchNebiusData } from "./nebius.js";
+import { fetchNovitaData } from "./novita.js";
+import { fetchSambanovaData } from "./sambanova.js";
+
+// --- Constants ---
+const CACHE_FILE_PATH = path.resolve("src/lib/server/data/context_length.json");
+
+// --- Types ---
+export interface MaxTokensCache {
+	[provider: string]: {
+		[modelId: string]: number;
+	};
+}
+
+// Type for API keys object passed to fetchAllProviderData
+export interface ApiKeys {
+	COHERE_API_KEY?: string;
+	TOGETHER_API_KEY?: string;
+	FIREWORKS_API_KEY?: string;
+	HYPERBOLIC_API_KEY?: string;
+	REPLICATE_API_KEY?: string;
+	NEBIUS_API_KEY?: string;
+	NOVITA_API_KEY?: string;
+	SAMBANOVA_API_KEY?: string;
+}
+
+// --- Cache Handling ---
+// (readCache and updateCache remain the same)
+let memoryCache: MaxTokensCache | null = null;
+let cacheReadPromise: Promise<MaxTokensCache> | null = null;
+
+async function readCache(): Promise<MaxTokensCache> {
+	if (memoryCache) {
+		return memoryCache;
+	}
+	if (cacheReadPromise) {
+		return cacheReadPromise;
+	}
+	cacheReadPromise = (async () => {
+		try {
+			const data = await fs.readFile(CACHE_FILE_PATH, "utf-8");
+			memoryCache = JSON.parse(data) as MaxTokensCache;
+			return memoryCache!;
+		} catch (error: unknown) {
+			if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") {
+				console.warn(`Cache file not found at ${CACHE_FILE_PATH}, starting with empty cache.`);
+				memoryCache = {};
+				return {};
+			}
+			console.error("Error reading context length cache file:", error);
+			memoryCache = {};
+			return {};
+		} finally {
+			cacheReadPromise = null;
+		}
+	})();
+	return cacheReadPromise;
+}
+
+const isBrowser = typeof window !== "undefined";
+
+function serverLog(...txt: unknown[]) {
+	if (isBrowser) return;
+	console.log(...txt);
+}
+
+function serverError(...txt: unknown[]) {
+	if (isBrowser) return;
+	console.error(...txt);
+}
+
+async function updateCache(provider: string, modelId: string, maxTokens: number): Promise<void> {
+	try {
+		let cache: MaxTokensCache;
+		try {
+			const data = await fs.readFile(CACHE_FILE_PATH, "utf-8");
+			cache = JSON.parse(data) as MaxTokensCache;
+		} catch (readError: unknown) {
+			if (typeof readError === "object" && readError !== null && "code" in readError && readError.code === "ENOENT") {
+				cache = {};
+			} else {
+				throw readError;
+			}
+		}
+		if (!cache[provider]) {
+			cache[provider] = {};
+		}
+		cache[provider][modelId] = maxTokens;
+		const tempFilePath = CACHE_FILE_PATH + ".tmp";
+		await fs.writeFile(tempFilePath, JSON.stringify(cache, null, "\t"), "utf-8");
+		await fs.rename(tempFilePath, CACHE_FILE_PATH);
+		memoryCache = cache;
+		serverLog(`Cache updated for ${provider} - ${modelId}: ${maxTokens}`);
+	} catch (error) {
+		serverError(`Error updating context length cache for ${provider} - ${modelId}:`, error);
+		memoryCache = null;
+	}
+}
+
+// --- Main Exported Function ---
+// Now accepts apiKey as the third argument
+export async function getMaxTokens(
+	provider: string,
+	modelId: string,
+	apiKey: string | undefined
+): Promise<number | null> {
+	const cache = await readCache();
+	const cachedValue = cache[provider]?.[modelId];
+
+	if (cachedValue !== undefined) {
+		return cachedValue;
+	}
+
+	serverLog(`Cache miss for ${provider} - ${modelId}. Attempting live fetch...`);
+
+	let liveData: number | null = null;
+	let fetchedProviderData: MaxTokensCache[string] | null = null;
+
+	try {
+		// Pass the received apiKey to the fetcher functions
+		switch (provider) {
+			case "cohere":
+				fetchedProviderData = await fetchCohereData(apiKey); // Pass apiKey
+				liveData = fetchedProviderData?.[modelId] ?? null;
+				break;
+			case "together":
+				fetchedProviderData = await fetchTogetherData(apiKey); // Pass apiKey
+				liveData = fetchedProviderData?.[modelId] ?? null;
+				break;
+			case "fireworks-ai":
+				fetchedProviderData = await fetchFireworksData(apiKey); // Pass apiKey
+				liveData = fetchedProviderData?.[modelId] ?? null;
+				break;
+			case "hyperbolic":
+				fetchedProviderData = await fetchHyperbolicData(apiKey); // Pass apiKey
+				liveData = fetchedProviderData?.[modelId] ?? null;
+				break;
+			case "replicate":
+				fetchedProviderData = await fetchReplicateData(apiKey);
+				liveData = fetchedProviderData?.[modelId] ?? null;
+				break;
+			case "nebius":
+				fetchedProviderData = await fetchNebiusData(apiKey);
+				liveData = fetchedProviderData?.[modelId] ?? null;
+				break;
+			case "novita":
+				fetchedProviderData = await fetchNovitaData(apiKey);
+				liveData = fetchedProviderData?.[modelId] ?? null;
+				break;
+			case "sambanova":
+				fetchedProviderData = await fetchSambanovaData(apiKey);
+				liveData = fetchedProviderData?.[modelId] ?? null;
+				break;
+			default:
+				serverLog(`Live fetch not supported or implemented for provider: ${provider}`);
+				return null;
+		}
+
+		if (liveData !== null) {
+			serverLog(`Live fetch successful for ${provider} - ${modelId}: ${liveData}`);
+			updateCache(provider, modelId, liveData).catch(err => {
+				serverError(`Async cache update failed for ${provider} - ${modelId}:`, err);
+			});
+			return liveData;
+		} else {
+			serverLog(`Live fetch for ${provider} did not return data for model ${modelId}.`);
+			return null;
+		}
+	} catch (error) {
+		serverError(`Error during live fetch for ${provider} - ${modelId}:`, error);
+		return null;
+	}
+}
+
+// --- Helper for Build Script ---
+// Now accepts an apiKeys object
+export async function fetchAllProviderData(apiKeys: ApiKeys): Promise<MaxTokensCache> {
+	serverLog("Fetching data for all providers...");
+	const results: MaxTokensCache = {};
+
+	// Define fetchers, passing the specific key from the apiKeys object
+	const providerFetchers = [
+		{ name: "cohere", fetcher: () => fetchCohereData(apiKeys.COHERE_API_KEY) },
+		{ name: "together", fetcher: () => fetchTogetherData(apiKeys.TOGETHER_API_KEY) },
+		{ name: "fireworks-ai", fetcher: () => fetchFireworksData(apiKeys.FIREWORKS_API_KEY) },
+		{ name: "hyperbolic", fetcher: () => fetchHyperbolicData(apiKeys.HYPERBOLIC_API_KEY) },
+		{ name: "replicate", fetcher: () => fetchReplicateData(apiKeys.REPLICATE_API_KEY) },
+		{ name: "nebius", fetcher: () => fetchNebiusData(apiKeys.NEBIUS_API_KEY) },
+		{ name: "novita", fetcher: () => fetchNovitaData(apiKeys.NOVITA_API_KEY) },
+		{ name: "sambanova", fetcher: () => fetchSambanovaData(apiKeys.SAMBANOVA_API_KEY) },
+	];
+
+	const settledResults = await Promise.allSettled(providerFetchers.map(p => p.fetcher()));
+
+	settledResults.forEach((result, index) => {
+		const providerInfo = providerFetchers[index];
+		if (!providerInfo) {
+			serverError(`Error: No provider info found for index ${index}`);
+			return;
+		}
+		const providerName = providerInfo.name;
+
+		if (result.status === "fulfilled" && result.value) {
+			if (Object.keys(result.value).length > 0) {
+				results[providerName] = result.value;
+				serverLog(`Successfully fetched data for ${providerName}`);
+			} else {
+				serverLog(`No data returned for ${providerName}.`);
+			}
+		} else if (result.status === "rejected") {
+			serverError(`Error fetching ${providerName} data:`, result.reason);
+		}
+	});
+
+	serverLog("Finished fetching provider data.");
+	return results;
+}

src/lib/server/providers/nebius.ts

@@ -0,0 +1,49 @@
+import type { MaxTokensCache } from "./index.js";
+
+interface NebiusModel {
+	id: string;
+	config?: {
+		max_tokens?: number;
+	};
+	context_length?: number;
+}
+
+interface NebiusResponse {
+	data?: NebiusModel[];
+}
+
+const NEBIUS_API_URL = "https://api.studio.nebius.com/v1/models?verbose=true";
+
+export async function fetchNebiusData(apiKey: string | undefined): Promise<MaxTokensCache["nebius"]> {
+	if (!apiKey) {
+		console.warn("Nebius API key not provided. Skipping Nebius fetch.");
+		return {};
+	}
+	try {
+		const response = await fetch(NEBIUS_API_URL, {
+			headers: {
+				Authorization: `Bearer ${apiKey}`,
+			},
+		});
+		if (!response.ok) {
+			throw new Error(`Nebius API request failed: ${response.status} ${response.statusText}`);
+		}
+		const data: NebiusResponse = await response.json();
+		const modelsData: MaxTokensCache["nebius"] = {};
+
+		if (data?.data && Array.isArray(data.data)) {
+			for (const model of data.data) {
+				const contextLength = model.context_length ?? model.config?.max_tokens ?? null;
+				if (model.id && typeof contextLength === "number") {
+					modelsData[model.id] = contextLength;
+				}
+			}
+		} else {
+			console.warn("Unexpected response structure from Nebius API:", data);
+		}
+		return modelsData;
+	} catch (error) {
+		console.error("Error fetching Nebius data:", error);
+		return {};
+	}
+}

src/lib/server/providers/novita.ts

@@ -0,0 +1,46 @@
+import type { MaxTokensCache } from "./index.js";
+
+const NOVITA_API_URL = "https://api.novita.ai/v3/openai/models";
+
+interface NovitaModel {
+	id: string;
+	object: string;
+	context_size: number;
+}
+
+interface NovitaResponse {
+	data: NovitaModel[];
+}
+
+export async function fetchNovitaData(apiKey: string | undefined): Promise<MaxTokensCache["novita"]> {
+	if (!apiKey) {
+		console.warn("Novita API key not provided. Skipping Novita fetch.");
+		return {};
+	}
+	try {
+		const response = await fetch(NOVITA_API_URL, {
+			headers: {
+				Authorization: `Bearer ${apiKey}`,
+			},
+		});
+		if (!response.ok) {
+			throw new Error(`Novita API request failed: ${response.status} ${response.statusText}`);
+		}
+		const data: NovitaResponse = await response.json();
+		const modelsData: MaxTokensCache["novita"] = {};
+
+		if (data?.data && Array.isArray(data.data)) {
+			for (const model of data.data) {
+				if (model.id && typeof model.context_size === "number") {
+					modelsData[model.id] = model.context_size;
+				}
+			}
+		} else {
+			console.warn("Unexpected response structure from Novita API:", data);
+		}
+		return modelsData;
+	} catch (error) {
+		console.error("Error fetching Novita data:", error);
+		return {};
+	}
+}

src/lib/server/providers/replicate.ts

@@ -0,0 +1,37 @@
+import type { MaxTokensCache } from "./index.js";
+
+const REPLICATE_API_URL = "https://api.replicate.com/v1/models";
+
+export async function fetchReplicateData(apiKey: string | undefined): Promise<MaxTokensCache["replicate"]> {
+	if (!apiKey) {
+		console.warn("Replicate API key not provided. Skipping Replicate fetch.");
+		return {};
+	}
+	try {
+		const response = await fetch(REPLICATE_API_URL, {
+			headers: {
+				Authorization: `Token ${apiKey}`,
+			},
+		});
+		if (!response.ok) {
+			throw new Error(`Replicate API request failed: ${response.status} ${response.statusText}`);
+		}
+		const data = await response.json();
+		const modelsData: MaxTokensCache["replicate"] = {};
+
+		if (data?.results && Array.isArray(data.results)) {
+			for (const model of data.results) {
+				const contextLength = model.context_length ?? model.config?.max_tokens ?? null;
+				if (model.id && typeof contextLength === "number") {
+					modelsData[model.id] = contextLength;
+				}
+			}
+		} else {
+			console.warn("Unexpected response structure from Replicate API:", data);
+		}
+		return modelsData;
+	} catch (error) {
+		console.error("Error fetching Replicate data:", error);
+		return {};
+	}
+}

src/lib/server/providers/sambanova.ts

@@ -0,0 +1,52 @@
+import type { MaxTokensCache } from "./index.js";
+
+const SAMBANOVA_API_URL = "https://api.sambanova.ai/v1/models";
+
+interface SambanovaModel {
+	id: string;
+	object: string;
+	context_length: number;
+	max_completion_tokens?: number;
+	pricing?: {
+		prompt: string;
+		completion: string;
+	};
+}
+
+interface SambanovaResponse {
+	data: SambanovaModel[];
+	object: string;
+}
+
+export async function fetchSambanovaData(apiKey: string | undefined): Promise<MaxTokensCache["sambanova"]> {
+	if (!apiKey) {
+		console.warn("SambaNova API key not provided. Skipping SambaNova fetch.");
+		return {};
+	}
+	try {
+		const response = await fetch(SAMBANOVA_API_URL, {
+			headers: {
+				Authorization: `Bearer ${apiKey}`,
+			},
+		});
+		if (!response.ok) {
+			throw new Error(`SambaNova API request failed: ${response.status} ${response.statusText}`);
+		}
+		const data: SambanovaResponse = await response.json();
+		const modelsData: MaxTokensCache["sambanova"] = {};
+
+		if (data?.data && Array.isArray(data.data)) {
+			for (const model of data.data) {
+				if (model.id && typeof model.context_length === "number") {
+					modelsData[model.id] = model.context_length;
+				}
+			}
+		} else {
+			console.warn("Unexpected response structure from SambaNova API:", data);
+		}
+		return modelsData;
+	} catch (error) {
+		console.error("Error fetching SambaNova data:", error);
+		return {};
+	}
+}

src/lib/server/providers/together.ts

@@ -0,0 +1,37 @@
+import type { MaxTokensCache } from "./index.js";
+
+const TOGETHER_API_URL = "https://api.together.xyz/v1/models";
+
+// Accept apiKey as an argument
+export async function fetchTogetherData(apiKey: string | undefined): Promise<MaxTokensCache["together"]> {
+	if (!apiKey) {
+		console.warn("Together AI API key not provided. Skipping Together AI fetch.");
+		return {};
+	}
+	try {
+		const response = await fetch(TOGETHER_API_URL, {
+			headers: {
+				Authorization: `Bearer ${apiKey}`, // Use passed-in apiKey
+			},
+		});
+		if (!response.ok) {
+			throw new Error(`Together AI API request failed: ${response.status} ${response.statusText}`);
+		}
+		// eslint-disable-next-line @typescript-eslint/no-explicit-any
+		const data: any[] = await response.json();
+		const modelsData: MaxTokensCache["together"] = {};
+
+		if (Array.isArray(data)) {
+			for (const model of data) {
+				const contextLength = model.context_length ?? model.config?.max_tokens ?? null;
+				if (model.id && typeof contextLength === "number") {
+					modelsData[model.id] = contextLength;
+				}
+			}
+		}
+		return modelsData;
+	} catch (error) {
+		console.error("Error fetching Together AI data:", error);
+		return {};
+	}
+}

src/lib/state/generation-stats.svelte.ts

@@ -0,0 +1,31 @@
+import { getTokens } from "$lib/components/inference-playground/utils.js";
+import { watch } from "runed";
+import { session } from "./session.svelte";
+
+export interface GenerationStats {
+	latency: number;
+	generatedTokensCount: number;
+}
+
+function createGenerationStats() {
+	let stats = $state([] as Array<GenerationStats>);
+
+	const init = () => {
+		watch(
+			() => $state.snapshot(session.project),
+			() => {
+				session.project.conversations.forEach(async (c, i) => {
+					generationStats[i] = { latency: 0, ...generationStats[i], generatedTokensCount: await getTokens(c) };
+				});
+			}
+		);
+	};
+
+	const set = (s: Array<GenerationStats>) => {
+		stats = s;
+	};
+
+	return Object.assign(stats, { set, init });
+}
+
+export const generationStats = createGenerationStats();

src/lib/types.ts

@@ -27,6 +27,7 @@ export type ConversationWithHFModel = Conversation & {
 export const isConversationWithHFModel = typia.createIs<ConversationWithHFModel>();
 export const isConversationWithCustomModel = typia.createIs<ConversationWithCustomModel>();
 
+export const isHFModel = typia.createIs<Model>();
 export const isCustomModel = typia.createIs<CustomModel>();
 
 export type Project = {

src/lib/utils/is.ts

@@ -1,4 +1,5 @@
 import { SvelteSet } from "svelte/reactivity";
+import typia from "typia";
 
 export function isHtmlElement(element: unknown): element is HTMLElement {
 	return element instanceof HTMLElement;
@@ -35,3 +36,5 @@ export function isTouch(event: PointerEvent): boolean {
 export function isPromise(value: unknown): value is Promise<unknown> {
 	return value instanceof Promise;
 }
+
+export const isNumber = typia.createIs<number>();

src/lib/utils/object.ts

@@ -32,3 +32,12 @@ export function pick<T extends Record<string, unknown>, K extends keyof T>(obj:
 	}
 	return result;
 }
+
+/**
+ * Try and get a value from an object, or return undefined.
+ * The key does not need to match the type of the object, so the
+ * returned type is an union of all values, and undefined
+ */
+export function tryGet<T extends Record<string, unknown>>(obj: T, key: string): T[keyof T] | undefined {
+	return obj[key as keyof T];
+}

src/routes/+layout.svelte

@@ -6,6 +6,7 @@
 	import ShareModal from "$lib/components/share-modal.svelte";
 	import "../app.css";
 	import { session } from "$lib/state/session.svelte";
+	import { generationStats } from "$lib/state/generation-stats.svelte";
 
 	interface Props {
 		children?: import("svelte").Snippet;
@@ -13,6 +14,7 @@
 
 	let { children }: Props = $props();
 	session.init();
+	generationStats.init();
 </script>
 
 {@render children?.()}