Anthropic Endpoint Support (#923)

.env

@@ -9,6 +9,7 @@ COOKIE_NAME=hf-chat
 HF_TOKEN=#hf_<token> from from https://huggingface.co/settings/token
 HF_API_ROOT=https://api-inference.huggingface.co/models
 OPENAI_API_KEY=#your openai api key here
+ANTHROPIC_API_KEY=#your anthropic api key here
 
 HF_ACCESS_TOKEN=#LEGACY! Use HF_TOKEN instead
 

README.md

@@ -459,6 +459,51 @@ MODELS=`[
 ]`
 ```
 
+#### Anthropic
+
+We also support Anthropic models through the official SDK. You may provide your API key via the `ANTHROPIC_API_KEY` env variable, or alternatively, through the `endpoints.apiKey` as per the following example.
+
+```
+MODELS=`[
+  {
+      "name": "claude-3-sonnet-20240229",
+      "displayName": "Claude 3 Sonnet",
+      "description": "Ideal balance of intelligence and speed",
+      "parameters": {
+        "max_new_tokens": 4096,
+      },
+      "endpoints": [
+        {
+          "type": "anthropic",
+          // optionals
+          "apiKey": "sk-ant-...",
+          "baseURL": "https://api.anthropic.com",
+          defaultHeaders: {},
+          defaultQuery: {}
+        }
+      ]
+  },
+  {
+      "name": "claude-3-opus-20240229",
+      "displayName": "Claude 3 Opus",
+      "description": "Most powerful model for highly complex tasks",
+      "parameters": {
+         "max_new_tokens": 4096
+      },
+      "endpoints": [
+        {
+          "type": "anthropic",
+          // optionals
+          "apiKey": "sk-ant-...",
+          "baseURL": "https://api.anthropic.com",
+          defaultHeaders: {},
+          defaultQuery: {}
+        }
+      ]
+  }
+]`
+```
+
 #### Amazon
 
 You can also specify your Amazon SageMaker instance as an endpoint for chat-ui. The config goes like this:

package-lock.json

@@ -8,6 +8,7 @@
 			"name": "chat-ui",
 			"version": "0.7.0",
 			"dependencies": {
+				"@anthropic-ai/sdk": "^0.17.1",
 				"@huggingface/hub": "^0.5.1",
 				"@huggingface/inference": "^2.6.3",
 				"@iconify-json/bi": "^1.1.21",
@@ -122,6 +123,30 @@
 				"url": "https://github.com/sponsors/antfu"
 			}
 		},
+		"node_modules/@anthropic-ai/sdk": {
+			"version": "0.17.1",
+			"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.17.1.tgz",
+			"integrity": "sha512-ke/JGfaa4sc1PB58L4B9hXI/BlJphXc696+cVX8Z8gQt51l++a9umZTN/7UymV8Dcat6KKYNQE8P8yeeyAldHg==",
+			"dependencies": {
+				"@types/node": "^18.11.18",
+				"@types/node-fetch": "^2.6.4",
+				"abort-controller": "^3.0.0",
+				"agentkeepalive": "^4.2.1",
+				"digest-fetch": "^1.3.0",
+				"form-data-encoder": "1.7.2",
+				"formdata-node": "^4.3.2",
+				"node-fetch": "^2.6.7",
+				"web-streams-polyfill": "^3.2.1"
+			}
+		},
+		"node_modules/@anthropic-ai/sdk/node_modules/web-streams-polyfill": {
+			"version": "3.3.3",
+			"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz",
+			"integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==",
+			"engines": {
+				"node": ">= 8"
+			}
+		},
 		"node_modules/@cspotcode/source-map-support": {
 			"version": "0.8.1",
 			"resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
@@ -2013,7 +2038,6 @@
 			"version": "2.6.5",
 			"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.5.tgz",
 			"integrity": "sha512-OZsUlr2nxvkqUFLSaY2ZbA+P1q22q+KrlxWOn/38RX+u5kTkYL2mTujEpzUhGkS+K/QCYp9oagfXG39XOzyySg==",
-			"optional": true,
 			"dependencies": {
 				"@types/node": "*",
 				"form-data": "^4.0.0"
@@ -2409,7 +2433,6 @@
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
 			"integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
-			"optional": true,
 			"dependencies": {
 				"event-target-shim": "^5.0.0"
 			},
@@ -2462,7 +2485,6 @@
 			"version": "4.5.0",
 			"resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz",
 			"integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==",
-			"optional": true,
 			"dependencies": {
 				"humanize-ms": "^1.2.1"
 			},
@@ -2630,8 +2652,7 @@
 		"node_modules/base-64": {
 			"version": "0.1.0",
 			"resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz",
-			"integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==",
-			"optional": true
+			"integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA=="
 		},
 		"node_modules/base64-js": {
 			"version": "1.5.1",
@@ -2899,7 +2920,6 @@
 			"version": "0.0.2",
 			"resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz",
 			"integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==",
-			"optional": true,
 			"engines": {
 				"node": "*"
 			}
@@ -3096,7 +3116,6 @@
 			"version": "0.0.2",
 			"resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz",
 			"integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==",
-			"optional": true,
 			"engines": {
 				"node": "*"
 			}
@@ -3352,7 +3371,6 @@
 			"version": "1.3.0",
 			"resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz",
 			"integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==",
-			"optional": true,
 			"dependencies": {
 				"base-64": "^0.1.0",
 				"md5": "^2.3.0"
@@ -3724,7 +3742,6 @@
 			"version": "5.0.1",
 			"resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
 			"integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
-			"optional": true,
 			"engines": {
 				"node": ">=6"
 			}
@@ -3907,14 +3924,12 @@
 		"node_modules/form-data-encoder": {
 			"version": "1.7.2",
 			"resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
-			"integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==",
-			"optional": true
+			"integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="
 		},
 		"node_modules/formdata-node": {
 			"version": "4.4.1",
 			"resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
 			"integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
-			"optional": true,
 			"dependencies": {
 				"node-domexception": "1.0.0",
 				"web-streams-polyfill": "4.0.0-beta.3"
@@ -4206,7 +4221,6 @@
 			"version": "1.2.1",
 			"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
 			"integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
-			"optional": true,
 			"dependencies": {
 				"ms": "^2.0.0"
 			}
@@ -4344,8 +4358,7 @@
 		"node_modules/is-buffer": {
 			"version": "1.1.6",
 			"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
-			"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
-			"optional": true
+			"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
 		},
 		"node_modules/is-builtin-module": {
 			"version": "3.2.1",
@@ -4819,7 +4832,6 @@
 			"version": "2.3.0",
 			"resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz",
 			"integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==",
-			"optional": true,
 			"dependencies": {
 				"charenc": "0.0.2",
 				"crypt": "0.0.2",
@@ -5116,7 +5128,6 @@
 					"url": "https://paypal.me/jimmywarting"
 				}
 			],
-			"optional": true,
 			"engines": {
 				"node": ">=10.5.0"
 			}
@@ -5125,7 +5136,6 @@
 			"version": "2.7.0",
 			"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
 			"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
-			"optional": true,
 			"dependencies": {
 				"whatwg-url": "^5.0.0"
 			},
@@ -5144,20 +5154,17 @@
 		"node_modules/node-fetch/node_modules/tr46": {
 			"version": "0.0.3",
 			"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
-			"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
-			"optional": true
+			"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
 		},
 		"node_modules/node-fetch/node_modules/webidl-conversions": {
 			"version": "3.0.1",
 			"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
-			"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
-			"optional": true
+			"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
 		},
 		"node_modules/node-fetch/node_modules/whatwg-url": {
 			"version": "5.0.0",
 			"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
 			"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
-			"optional": true,
 			"dependencies": {
 				"tr46": "~0.0.3",
 				"webidl-conversions": "^3.0.0"
@@ -8176,7 +8183,6 @@
 			"version": "4.0.0-beta.3",
 			"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
 			"integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
-			"optional": true,
 			"engines": {
 				"node": ">= 14"
 			}

package.json

@@ -82,6 +82,7 @@
 		"zod": "^3.22.3"
 	},
 	"optionalDependencies": {
+		"@anthropic-ai/sdk": "^0.17.1",
 		"aws4fetch": "^1.0.17",
 		"openai": "^4.14.2"
 	}

src/lib/server/endpoints/anthropic/endpointAnthropic.ts

@@ -0,0 +1,95 @@
+import { z } from "zod";
+import { ANTHROPIC_API_KEY } from "$env/static/private";
+import type { Endpoint } from "../endpoints";
+import type { TextGenerationStreamOutput } from "@huggingface/inference";
+
+export const endpointAnthropicParametersSchema = z.object({
+	weight: z.number().int().positive().default(1),
+	model: z.any(),
+	type: z.literal("anthropic"),
+	baseURL: z.string().url().default("https://api.anthropic.com"),
+	apiKey: z.string().default(ANTHROPIC_API_KEY ?? "sk-"),
+	defaultHeaders: z.record(z.string()).optional(),
+	defaultQuery: z.record(z.string()).optional(),
+});
+
+export async function endpointAnthropic(
+	input: z.input<typeof endpointAnthropicParametersSchema>
+): Promise<Endpoint> {
+	const { baseURL, apiKey, model, defaultHeaders, defaultQuery } =
+		endpointAnthropicParametersSchema.parse(input);
+	let Anthropic;
+	try {
+		Anthropic = (await import("@anthropic-ai/sdk")).default;
+	} catch (e) {
+		throw new Error("Failed to import @anthropic-ai/sdk", { cause: e });
+	}
+
+	const anthropic = new Anthropic({
+		apiKey,
+		baseURL,
+		defaultHeaders,
+		defaultQuery,
+	});
+
+	return async ({ messages, preprompt }) => {
+		let system = preprompt;
+		if (messages?.[0]?.from === "system") {
+			system = messages[0].content;
+		}
+
+		const messagesFormatted = messages
+			.filter((message) => message.from !== "system")
+			.map((message) => ({
+				role: message.from,
+				content: message.content,
+			})) as unknown as {
+			role: "user" | "assistant";
+			content: string;
+		}[];
+
+		let tokenId = 0;
+		return (async function* () {
+			const stream = anthropic.messages.stream({
+				model: model.id ?? model.name,
+				messages: messagesFormatted,
+				max_tokens: model.parameters?.max_new_tokens,
+				temperature: model.parameters?.temperature,
+				top_p: model.parameters?.top_p,
+				top_k: model.parameters?.top_k,
+				stop_sequences: model.parameters?.stop,
+				system,
+			});
+			while (true) {
+				const result = await Promise.race([stream.emitted("text"), stream.emitted("end")]);
+
+				// Stream end
+				if (result === undefined) {
+					yield {
+						token: {
+							id: tokenId++,
+							text: "",
+							logprob: 0,
+							special: true,
+						},
+						generated_text: await stream.finalText(),
+						details: null,
+					} satisfies TextGenerationStreamOutput;
+					return;
+				}
+
+				// Text delta
+				yield {
+					token: {
+						id: tokenId++,
+						text: result as unknown as string,
+						special: false,
+						logprob: 0,
+					},
+					generated_text: null,
+					details: null,
+				} satisfies TextGenerationStreamOutput;
+			}
+		})();
+	};
+}

src/lib/server/endpoints/endpoints.ts

@@ -6,6 +6,10 @@ import endpointAws, { endpointAwsParametersSchema } from "./aws/endpointAws";
 import { endpointOAIParametersSchema, endpointOai } from "./openai/endpointOai";
 import endpointLlamacpp, { endpointLlamacppParametersSchema } from "./llamacpp/endpointLlamacpp";
 import endpointOllama, { endpointOllamaParametersSchema } from "./ollama/endpointOllama";
+import {
+	endpointAnthropic,
+	endpointAnthropicParametersSchema,
+} from "./anthropic/endpointAnthropic";
 
 // parameters passed when generating text
 export interface EndpointParameters {
@@ -28,6 +32,7 @@ export type EndpointGenerator<T extends CommonEndpoint> = (parameters: T) => End
 // list of all endpoint generators
 export const endpoints = {
 	tgi: endpointTgi,
+	anthropic: endpointAnthropic,
 	aws: endpointAws,
 	openai: endpointOai,
 	llamacpp: endpointLlamacpp,
@@ -35,6 +40,7 @@ export const endpoints = {
 };
 
 export const endpointSchema = z.discriminatedUnion("type", [
+	endpointAnthropicParametersSchema,
 	endpointAwsParametersSchema,
 	endpointOAIParametersSchema,
 	endpointTgiParametersSchema,

src/lib/server/models.ts

@@ -109,6 +109,8 @@ const addEndpoint = (m: Awaited<ReturnType<typeof processModel>>) => ({
 				switch (args.type) {
 					case "tgi":
 						return endpoints.tgi(args);
+					case "anthropic":
+						return endpoints.anthropic(args);
 					case "aws":
 						return await endpoints.aws(args);
 					case "openai":