Migrate to new model + proxy from server (#5)

* use fork of hf.js to support fully custom endpoints for testing purpose

* proxy textGenerationStream call to the backend to hide token from client

* migrate to patch-package instead of pnpm

* fix issue after merge conflict

* use env var instead of hardcoded value for endpoint

* fix messages not being split between assistant/user

* fix stream response sometimes not split by token

* remove PUBLIC_ from private env variables + rename ENDPOINT to MODEL_ENDPOINT

* only set hf token as private, model can stay public

* move HF_TOKEN to a dynamic env

* fix env var import typo

* remove @microsoft/fetch-event-source

* update parameters to be identical to Python demo

* small refactor to avoid typing issue

* cleanup while loop

Co-authored-by: Julien Chaumond <[email protected]>

* make comment clearer on what is happening on stream chunks split

* fix chunk spliting not being handled properly

* cleanup model tokens sometimes containing "<|endoftext|>" text

* refactor how we proxy from the server to simplify logic

* use latest version of hf.js

* use .env + .env.local instead of .env.example

* rewrite logic to trim "<|endoftext|>" artifact properly

* update to latest hf.js

* expose env var to Docker during build time for deployment

* remove patch-package

---------

Co-authored-by: Julien Chaumond <[email protected]>

.env

@@ -0,0 +1,7 @@
+PUBLIC_MODEL_ENDPOINT=https://api-inference.huggingface.co/models/OpenAssistant/oasst-sft-1-pythia-12b
+PUBLIC_MODEL_NAME=OpenAssistant/oasst-sft-1-pythia-12b
+PUBLIC_MODEL_TAGLINE=This is the first iteration English supervised-fine-tuning (SFT) model of the <a class="underline" href="https://github.com/LAION-AI/Open-Assistant">Open-Assistant</a> project. It is based on a Pythia 12B that was fine-tuned on ~22k human demonstrations of assistant conversations collected through the <a class="underline" href="https://open-assistant.io/">https://open-assistant.io/</a> human feedback web app before March 7, 2023.
+PUBLIC_DISABLE_INTRO_TILES=true
+PUBLIC_USER_MESSAGE_TOKEN=<|prompter|>
+PUBLIC_ASSISTANT_MESSAGE_TOKEN=<|assistant|>
+PUBLIC_SEP_TOKEN=<|endoftext|>

.env.example

@@ -1,7 +0,0 @@
-PUBLIC_HF_TOKEN=
-PUBLIC_ENDPOINT=https://api-inference.huggingface.co/models/OpenAssistant/oasst-sft-1-pythia-12b
-PUBLIC_MODEL_NAME=OpenAssistant/llama_30b_oasst_latcyr_1000
-PUBLIC_DISABLE_INTRO_TILES=true
-PUBLIC_USER_MESSAGE_TOKEN=<|prompter|>
-PUBLIC_ASSISTANT_MESSAGE_TOKEN=<|assistant|>
-PUBLIC_SEP_TOKEN=<|endoftext|>

Dockerfile

@@ -11,7 +11,8 @@ RUN npm i
 
 RUN chown -R 1000:1000 /app
 
-RUN npm run build
+RUN --mount=type=secret,id=PUBLIC_MODEL_ENDPOINT,mode=0444,required=true \
+   PUBLIC_MODEL_ENDPOINT=$(cat /run/secrets/PUBLIC_MODEL_ENDPOINT) npm run build
 
 ENV PORT 7860
 

package-lock.json

@@ -8,7 +8,7 @@
 			"name": "chat-ui",
 			"version": "0.0.1",
 			"dependencies": {
-				"@microsoft/fetch-event-source": "^2.0.1",
+				"@huggingface/inference": "^2.0.0-rc2",
 				"autoprefixer": "^10.4.14",
 				"postcss": "^8.4.21",
 				"tailwind-scrollbar": "^3.0.0",
@@ -33,9 +33,9 @@
 			}
 		},
 		"node_modules/@esbuild/android-arm": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.16.17.tgz",
-			"integrity": "sha512-N9x1CMXVhtWEAMS7pNNONyA14f71VPQN9Cnavj1XQh6T7bskqiLLrSca4O0Vr8Wdcga943eThxnVp3JLnBMYtw==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.17.16.tgz",
+			"integrity": "sha512-baLqRpLe4JnKrUXLJChoTN0iXZH7El/mu58GE3WIA6/H834k0XWvLRmGLG8y8arTRS9hJJibPnF0tiGhmWeZgw==",
 			"cpu": [
 				"arm"
 			],
@@ -49,9 +49,9 @@
 			}
 		},
 		"node_modules/@esbuild/android-arm64": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.16.17.tgz",
-			"integrity": "sha512-MIGl6p5sc3RDTLLkYL1MyL8BMRN4tLMRCn+yRJJmEDvYZ2M7tmAf80hx1kbNEUX2KJ50RRtxZ4JHLvCfuB6kBg==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.17.16.tgz",
+			"integrity": "sha512-QX48qmsEZW+gcHgTmAj+x21mwTz8MlYQBnzF6861cNdQGvj2jzzFjqH0EBabrIa/WVZ2CHolwMoqxVryqKt8+Q==",
 			"cpu": [
 				"arm64"
 			],
@@ -65,9 +65,9 @@
 			}
 		},
 		"node_modules/@esbuild/android-x64": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.16.17.tgz",
-			"integrity": "sha512-a3kTv3m0Ghh4z1DaFEuEDfz3OLONKuFvI4Xqczqx4BqLyuFaFkuaG4j2MtA6fuWEFeC5x9IvqnX7drmRq/fyAQ==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.17.16.tgz",
+			"integrity": "sha512-G4wfHhrrz99XJgHnzFvB4UwwPxAWZaZBOFXh+JH1Duf1I4vIVfuYY9uVLpx4eiV2D/Jix8LJY+TAdZ3i40tDow==",
 			"cpu": [
 				"x64"
 			],
@@ -81,9 +81,9 @@
 			}
 		},
 		"node_modules/@esbuild/darwin-arm64": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.16.17.tgz",
-			"integrity": "sha512-/2agbUEfmxWHi9ARTX6OQ/KgXnOWfsNlTeLcoV7HSuSTv63E4DqtAc+2XqGw1KHxKMHGZgbVCZge7HXWX9Vn+w==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.17.16.tgz",
+			"integrity": "sha512-/Ofw8UXZxuzTLsNFmz1+lmarQI6ztMZ9XktvXedTbt3SNWDn0+ODTwxExLYQ/Hod91EZB4vZPQJLoqLF0jvEzA==",
 			"cpu": [
 				"arm64"
 			],
@@ -97,9 +97,9 @@
 			}
 		},
 		"node_modules/@esbuild/darwin-x64": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.16.17.tgz",
-			"integrity": "sha512-2By45OBHulkd9Svy5IOCZt376Aa2oOkiE9QWUK9fe6Tb+WDr8hXL3dpqi+DeLiMed8tVXspzsTAvd0jUl96wmg==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.17.16.tgz",
+			"integrity": "sha512-SzBQtCV3Pdc9kyizh36Ol+dNVhkDyIrGb/JXZqFq8WL37LIyrXU0gUpADcNV311sCOhvY+f2ivMhb5Tuv8nMOQ==",
 			"cpu": [
 				"x64"
 			],
@@ -113,9 +113,9 @@
 			}
 		},
 		"node_modules/@esbuild/freebsd-arm64": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.16.17.tgz",
-			"integrity": "sha512-mt+cxZe1tVx489VTb4mBAOo2aKSnJ33L9fr25JXpqQqzbUIw/yzIzi+NHwAXK2qYV1lEFp4OoVeThGjUbmWmdw==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.17.16.tgz",
+			"integrity": "sha512-ZqftdfS1UlLiH1DnS2u3It7l4Bc3AskKeu+paJSfk7RNOMrOxmeFDhLTMQqMxycP1C3oj8vgkAT6xfAuq7ZPRA==",
 			"cpu": [
 				"arm64"
 			],
@@ -129,9 +129,9 @@
 			}
 		},
 		"node_modules/@esbuild/freebsd-x64": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.16.17.tgz",
-			"integrity": "sha512-8ScTdNJl5idAKjH8zGAsN7RuWcyHG3BAvMNpKOBaqqR7EbUhhVHOqXRdL7oZvz8WNHL2pr5+eIT5c65kA6NHug==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.17.16.tgz",
+			"integrity": "sha512-rHV6zNWW1tjgsu0dKQTX9L0ByiJHHLvQKrWtnz8r0YYJI27FU3Xu48gpK2IBj1uCSYhJ+pEk6Y0Um7U3rIvV8g==",
 			"cpu": [
 				"x64"
 			],
@@ -145,9 +145,9 @@
 			}
 		},
 		"node_modules/@esbuild/linux-arm": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.16.17.tgz",
-			"integrity": "sha512-iihzrWbD4gIT7j3caMzKb/RsFFHCwqqbrbH9SqUSRrdXkXaygSZCZg1FybsZz57Ju7N/SHEgPyaR0LZ8Zbe9gQ==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.17.16.tgz",
+			"integrity": "sha512-n4O8oVxbn7nl4+m+ISb0a68/lcJClIbaGAoXwqeubj/D1/oMMuaAXmJVfFlRjJLu/ZvHkxoiFJnmbfp4n8cdSw==",
 			"cpu": [
 				"arm"
 			],
@@ -161,9 +161,9 @@
 			}
 		},
 		"node_modules/@esbuild/linux-arm64": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.16.17.tgz",
-			"integrity": "sha512-7S8gJnSlqKGVJunnMCrXHU9Q8Q/tQIxk/xL8BqAP64wchPCTzuM6W3Ra8cIa1HIflAvDnNOt2jaL17vaW+1V0g==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.17.16.tgz",
+			"integrity": "sha512-8yoZhGkU6aHu38WpaM4HrRLTFc7/VVD9Q2SvPcmIQIipQt2I/GMTZNdEHXoypbbGao5kggLcxg0iBKjo0SQYKA==",
 			"cpu": [
 				"arm64"
 			],
@@ -177,9 +177,9 @@
 			}
 		},
 		"node_modules/@esbuild/linux-ia32": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.16.17.tgz",
-			"integrity": "sha512-kiX69+wcPAdgl3Lonh1VI7MBr16nktEvOfViszBSxygRQqSpzv7BffMKRPMFwzeJGPxcio0pdD3kYQGpqQ2SSg==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.17.16.tgz",
+			"integrity": "sha512-9ZBjlkdaVYxPNO8a7OmzDbOH9FMQ1a58j7Xb21UfRU29KcEEU3VTHk+Cvrft/BNv0gpWJMiiZ/f4w0TqSP0gLA==",
 			"cpu": [
 				"ia32"
 			],
@@ -193,9 +193,9 @@
 			}
 		},
 		"node_modules/@esbuild/linux-loong64": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.16.17.tgz",
-			"integrity": "sha512-dTzNnQwembNDhd654cA4QhbS9uDdXC3TKqMJjgOWsC0yNCbpzfWoXdZvp0mY7HU6nzk5E0zpRGGx3qoQg8T2DQ==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.17.16.tgz",
+			"integrity": "sha512-TIZTRojVBBzdgChY3UOG7BlPhqJz08AL7jdgeeu+kiObWMFzGnQD7BgBBkWRwOtKR1i2TNlO7YK6m4zxVjjPRQ==",
 			"cpu": [
 				"loong64"
 			],
@@ -209,9 +209,9 @@
 			}
 		},
 		"node_modules/@esbuild/linux-mips64el": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.16.17.tgz",
-			"integrity": "sha512-ezbDkp2nDl0PfIUn0CsQ30kxfcLTlcx4Foz2kYv8qdC6ia2oX5Q3E/8m6lq84Dj/6b0FrkgD582fJMIfHhJfSw==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.17.16.tgz",
+			"integrity": "sha512-UPeRuFKCCJYpBbIdczKyHLAIU31GEm0dZl1eMrdYeXDH+SJZh/i+2cAmD3A1Wip9pIc5Sc6Kc5cFUrPXtR0XHA==",
 			"cpu": [
 				"mips64el"
 			],
@@ -225,9 +225,9 @@
 			}
 		},
 		"node_modules/@esbuild/linux-ppc64": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.16.17.tgz",
-			"integrity": "sha512-dzS678gYD1lJsW73zrFhDApLVdM3cUF2MvAa1D8K8KtcSKdLBPP4zZSLy6LFZ0jYqQdQ29bjAHJDgz0rVbLB3g==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.17.16.tgz",
+			"integrity": "sha512-io6yShgIEgVUhExJejJ21xvO5QtrbiSeI7vYUnr7l+v/O9t6IowyhdiYnyivX2X5ysOVHAuyHW+Wyi7DNhdw6Q==",
 			"cpu": [
 				"ppc64"
 			],
@@ -241,9 +241,9 @@
 			}
 		},
 		"node_modules/@esbuild/linux-riscv64": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.16.17.tgz",
-			"integrity": "sha512-ylNlVsxuFjZK8DQtNUwiMskh6nT0vI7kYl/4fZgV1llP5d6+HIeL/vmmm3jpuoo8+NuXjQVZxmKuhDApK0/cKw==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.17.16.tgz",
+			"integrity": "sha512-WhlGeAHNbSdG/I2gqX2RK2gfgSNwyJuCiFHMc8s3GNEMMHUI109+VMBfhVqRb0ZGzEeRiibi8dItR3ws3Lk+cA==",
 			"cpu": [
 				"riscv64"
 			],
@@ -257,9 +257,9 @@
 			}
 		},
 		"node_modules/@esbuild/linux-s390x": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.16.17.tgz",
-			"integrity": "sha512-gzy7nUTO4UA4oZ2wAMXPNBGTzZFP7mss3aKR2hH+/4UUkCOyqmjXiKpzGrY2TlEUhbbejzXVKKGazYcQTZWA/w==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.17.16.tgz",
+			"integrity": "sha512-gHRReYsJtViir63bXKoFaQ4pgTyah4ruiMRQ6im9YZuv+gp3UFJkNTY4sFA73YDynmXZA6hi45en4BGhNOJUsw==",
 			"cpu": [
 				"s390x"
 			],
@@ -273,9 +273,9 @@
 			}
 		},
 		"node_modules/@esbuild/linux-x64": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.16.17.tgz",
-			"integrity": "sha512-mdPjPxfnmoqhgpiEArqi4egmBAMYvaObgn4poorpUaqmvzzbvqbowRllQ+ZgzGVMGKaPkqUmPDOOFQRUFDmeUw==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.17.16.tgz",
+			"integrity": "sha512-mfiiBkxEbUHvi+v0P+TS7UnA9TeGXR48aK4XHkTj0ZwOijxexgMF01UDFaBX7Q6CQsB0d+MFNv9IiXbIHTNd4g==",
 			"cpu": [
 				"x64"
 			],
@@ -289,9 +289,9 @@
 			}
 		},
 		"node_modules/@esbuild/netbsd-x64": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.16.17.tgz",
-			"integrity": "sha512-/PzmzD/zyAeTUsduZa32bn0ORug+Jd1EGGAUJvqfeixoEISYpGnAezN6lnJoskauoai0Jrs+XSyvDhppCPoKOA==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.17.16.tgz",
+			"integrity": "sha512-n8zK1YRDGLRZfVcswcDMDM0j2xKYLNXqei217a4GyBxHIuPMGrrVuJ+Ijfpr0Kufcm7C1k/qaIrGy6eG7wvgmA==",
 			"cpu": [
 				"x64"
 			],
@@ -305,9 +305,9 @@
 			}
 		},
 		"node_modules/@esbuild/openbsd-x64": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.16.17.tgz",
-			"integrity": "sha512-2yaWJhvxGEz2RiftSk0UObqJa/b+rIAjnODJgv2GbGGpRwAfpgzyrg1WLK8rqA24mfZa9GvpjLcBBg8JHkoodg==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.17.16.tgz",
+			"integrity": "sha512-lEEfkfsUbo0xC47eSTBqsItXDSzwzwhKUSsVaVjVji07t8+6KA5INp2rN890dHZeueXJAI8q0tEIfbwVRYf6Ew==",
 			"cpu": [
 				"x64"
 			],
@@ -321,9 +321,9 @@
 			}
 		},
 		"node_modules/@esbuild/sunos-x64": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.16.17.tgz",
-			"integrity": "sha512-xtVUiev38tN0R3g8VhRfN7Zl42YCJvyBhRKw1RJjwE1d2emWTVToPLNEQj/5Qxc6lVFATDiy6LjVHYhIPrLxzw==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.17.16.tgz",
+			"integrity": "sha512-jlRjsuvG1fgGwnE8Afs7xYDnGz0dBgTNZfgCK6TlvPH3Z13/P5pi6I57vyLE8qZYLrGVtwcm9UbUx1/mZ8Ukag==",
 			"cpu": [
 				"x64"
 			],
@@ -337,9 +337,9 @@
 			}
 		},
 		"node_modules/@esbuild/win32-arm64": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.16.17.tgz",
-			"integrity": "sha512-ga8+JqBDHY4b6fQAmOgtJJue36scANy4l/rL97W+0wYmijhxKetzZdKOJI7olaBaMhWt8Pac2McJdZLxXWUEQw==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.17.16.tgz",
+			"integrity": "sha512-TzoU2qwVe2boOHl/3KNBUv2PNUc38U0TNnzqOAcgPiD/EZxT2s736xfC2dYQbszAwo4MKzzwBV0iHjhfjxMimg==",
 			"cpu": [
 				"arm64"
 			],
@@ -353,9 +353,9 @@
 			}
 		},
 		"node_modules/@esbuild/win32-ia32": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.16.17.tgz",
-			"integrity": "sha512-WnsKaf46uSSF/sZhwnqE4L/F89AYNMiD4YtEcYekBt9Q7nj0DiId2XH2Ng2PHM54qi5oPrQ8luuzGszqi/veig==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.17.16.tgz",
+			"integrity": "sha512-B8b7W+oo2yb/3xmwk9Vc99hC9bNolvqjaTZYEfMQhzdpBsjTvZBlXQ/teUE55Ww6sg//wlcDjOaqldOKyigWdA==",
 			"cpu": [
 				"ia32"
 			],
@@ -369,9 +369,9 @@
 			}
 		},
 		"node_modules/@esbuild/win32-x64": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.16.17.tgz",
-			"integrity": "sha512-y+EHuSchhL7FjHgvQL/0fnnFmO4T1bhvWANX6gcnqTjtnKWbTvUMCpGnv2+t+31d7RzyEAYAd4u2fnIhHL6N/Q==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.17.16.tgz",
+			"integrity": "sha512-xJ7OH/nanouJO9pf03YsL9NAFQBHd8AqfrQd7Pf5laGyyTt/gToul6QYOA/i5i/q8y9iaM5DQFNTgpi995VkOg==",
 			"cpu": [
 				"x64"
 			],
@@ -384,15 +384,39 @@
 				"node": ">=12"
 			}
 		},
+		"node_modules/@eslint-community/eslint-utils": {
+			"version": "4.4.0",
+			"resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz",
+			"integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==",
+			"dev": true,
+			"dependencies": {
+				"eslint-visitor-keys": "^3.3.0"
+			},
+			"engines": {
+				"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+			},
+			"peerDependencies": {
+				"eslint": "^6.0.0 || ^7.0.0 || >=8.0.0"
+			}
+		},
+		"node_modules/@eslint-community/regexpp": {
+			"version": "4.5.0",
+			"resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.5.0.tgz",
+			"integrity": "sha512-vITaYzIcNmjn5tF5uxcZ/ft7/RXGrMUIS9HalWckEOF6ESiwXKoMzAQf2UW0aVd6rnOeExTJVd5hmWXucBKGXQ==",
+			"dev": true,
+			"engines": {
+				"node": "^12.0.0 || ^14.0.0 || >=16.0.0"
+			}
+		},
 		"node_modules/@eslint/eslintrc": {
-			"version": "1.4.1",
-			"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.4.1.tgz",
-			"integrity": "sha512-XXrH9Uarn0stsyldqDYq8r++mROmWRI1xKMXa640Bb//SY1+ECYX6VzT6Lcx5frD0V30XieqJ0oX9I2Xj5aoMA==",
+			"version": "2.0.2",
+			"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.0.2.tgz",
+			"integrity": "sha512-3W4f5tDUra+pA+FzgugqL2pRimUTDJWKr7BINqOpkZrC0uYI0NIc0/JFgBROCU07HR6GieA5m3/rsPIhDmCXTQ==",
 			"dev": true,
 			"dependencies": {
 				"ajv": "^6.12.4",
 				"debug": "^4.3.2",
-				"espree": "^9.4.0",
+				"espree": "^9.5.1",
 				"globals": "^13.19.0",
 				"ignore": "^5.2.0",
 				"import-fresh": "^3.2.1",
@@ -407,6 +431,23 @@
 				"url": "https://opencollective.com/eslint"
 			}
 		},
+		"node_modules/@eslint/js": {
+			"version": "8.38.0",
+			"resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.38.0.tgz",
+			"integrity": "sha512-IoD2MfUnOV58ghIHCiil01PcohxjbYR/qCxsoC+xNgUwh1EY8jOOrYmu3d3a71+tJJ23uscEV4X2HJWMsPJu4g==",
+			"dev": true,
+			"engines": {
+				"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+			}
+		},
+		"node_modules/@huggingface/inference": {
+			"version": "2.0.0-rc2",
+			"resolved": "https://registry.npmjs.org/@huggingface/inference/-/inference-2.0.0-rc2.tgz",
+			"integrity": "sha512-jCU+zl1fmbmaWbTa3P2KC1GQMwjMc9ZLcR8Nq5UeFmDbmdz0GiBHsK7F1hTHRAPeF1K3X5L3V3OQUNZ6nzEu/w==",
+			"engines": {
+				"node": ">=18"
+			}
+		},
 		"node_modules/@humanwhocodes/config-array": {
 			"version": "0.11.8",
 			"resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz",
@@ -483,11 +524,6 @@
 				"@jridgewell/sourcemap-codec": "1.4.14"
 			}
 		},
-		"node_modules/@microsoft/fetch-event-source": {
-			"version": "2.0.1",
-			"resolved": "https://registry.npmjs.org/@microsoft/fetch-event-source/-/fetch-event-source-2.0.1.tgz",
-			"integrity": "sha512-W6CLUJ2eBMw3Rec70qrsEW0jOm/3twwJv21mrmj2yORiaVmVYGS4sSS5yUwvQc1ZlDLYGPnClVWmUUMagKNsfA=="
-		},
 		"node_modules/@nodelib/fs.scandir": {
 			"version": "2.1.5",
 			"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
@@ -1476,9 +1512,9 @@
 			"dev": true
 		},
 		"node_modules/esbuild": {
-			"version": "0.16.17",
-			"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.16.17.tgz",
-			"integrity": "sha512-G8LEkV0XzDMNwXKgM0Jwu3nY3lSTwSGY6XbxM9cr9+s0T/qSV1q1JVPBGzm3dcjhCic9+emZDmMffkwgPeOeLg==",
+			"version": "0.17.16",
+			"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.17.16.tgz",
+			"integrity": "sha512-aeSuUKr9aFVY9Dc8ETVELGgkj4urg5isYx8pLf4wlGgB0vTFjxJQdHnNH6Shmx4vYYrOTLCHtRI5i1XZ9l2Zcg==",
 			"dev": true,
 			"hasInstallScript": true,
 			"bin": {
@@ -1488,28 +1524,28 @@
 				"node": ">=12"
 			},
 			"optionalDependencies": {
-				"@esbuild/android-arm": "0.16.17",
-				"@esbuild/android-arm64": "0.16.17",
-				"@esbuild/android-x64": "0.16.17",
-				"@esbuild/darwin-arm64": "0.16.17",
-				"@esbuild/darwin-x64": "0.16.17",
-				"@esbuild/freebsd-arm64": "0.16.17",
-				"@esbuild/freebsd-x64": "0.16.17",
-				"@esbuild/linux-arm": "0.16.17",
-				"@esbuild/linux-arm64": "0.16.17",
-				"@esbuild/linux-ia32": "0.16.17",
-				"@esbuild/linux-loong64": "0.16.17",
-				"@esbuild/linux-mips64el": "0.16.17",
-				"@esbuild/linux-ppc64": "0.16.17",
-				"@esbuild/linux-riscv64": "0.16.17",
-				"@esbuild/linux-s390x": "0.16.17",
-				"@esbuild/linux-x64": "0.16.17",
-				"@esbuild/netbsd-x64": "0.16.17",
-				"@esbuild/openbsd-x64": "0.16.17",
-				"@esbuild/sunos-x64": "0.16.17",
-				"@esbuild/win32-arm64": "0.16.17",
-				"@esbuild/win32-ia32": "0.16.17",
-				"@esbuild/win32-x64": "0.16.17"
+				"@esbuild/android-arm": "0.17.16",
+				"@esbuild/android-arm64": "0.17.16",
+				"@esbuild/android-x64": "0.17.16",
+				"@esbuild/darwin-arm64": "0.17.16",
+				"@esbuild/darwin-x64": "0.17.16",
+				"@esbuild/freebsd-arm64": "0.17.16",
+				"@esbuild/freebsd-x64": "0.17.16",
+				"@esbuild/linux-arm": "0.17.16",
+				"@esbuild/linux-arm64": "0.17.16",
+				"@esbuild/linux-ia32": "0.17.16",
+				"@esbuild/linux-loong64": "0.17.16",
+				"@esbuild/linux-mips64el": "0.17.16",
+				"@esbuild/linux-ppc64": "0.17.16",
+				"@esbuild/linux-riscv64": "0.17.16",
+				"@esbuild/linux-s390x": "0.17.16",
+				"@esbuild/linux-x64": "0.17.16",
+				"@esbuild/netbsd-x64": "0.17.16",
+				"@esbuild/openbsd-x64": "0.17.16",
+				"@esbuild/sunos-x64": "0.17.16",
+				"@esbuild/win32-arm64": "0.17.16",
+				"@esbuild/win32-ia32": "0.17.16",
+				"@esbuild/win32-x64": "0.17.16"
 			}
 		},
 		"node_modules/escalade": {
@@ -1533,12 +1569,15 @@
 			}
 		},
 		"node_modules/eslint": {
-			"version": "8.34.0",
-			"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.34.0.tgz",
-			"integrity": "sha512-1Z8iFsucw+7kSqXNZVslXS8Ioa4u2KM7GPwuKtkTFAqZ/cHMcEaR+1+Br0wLlot49cNxIiZk5wp8EAbPcYZxTg==",
+			"version": "8.38.0",
+			"resolved": "https://registry.npmjs.org/eslint/-/eslint-8.38.0.tgz",
+			"integrity": "sha512-pIdsD2jwlUGf/U38Jv97t8lq6HpaU/G9NKbYmpWpZGw3LdTNhZLbJePqxOXGB5+JEKfOPU/XLxYxFh03nr1KTg==",
 			"dev": true,
 			"dependencies": {
-				"@eslint/eslintrc": "^1.4.1",
+				"@eslint-community/eslint-utils": "^4.2.0",
+				"@eslint-community/regexpp": "^4.4.0",
+				"@eslint/eslintrc": "^2.0.2",
+				"@eslint/js": "8.38.0",
 				"@humanwhocodes/config-array": "^0.11.8",
 				"@humanwhocodes/module-importer": "^1.0.1",
 				"@nodelib/fs.walk": "^1.2.8",
@@ -1549,10 +1588,9 @@
 				"doctrine": "^3.0.0",
 				"escape-string-regexp": "^4.0.0",
 				"eslint-scope": "^7.1.1",
-				"eslint-utils": "^3.0.0",
-				"eslint-visitor-keys": "^3.3.0",
-				"espree": "^9.4.0",
-				"esquery": "^1.4.0",
+				"eslint-visitor-keys": "^3.4.0",
+				"espree": "^9.5.1",
+				"esquery": "^1.4.2",
 				"esutils": "^2.0.2",
 				"fast-deep-equal": "^3.1.3",
 				"file-entry-cache": "^6.0.1",
@@ -1573,7 +1611,6 @@
 				"minimatch": "^3.1.2",
 				"natural-compare": "^1.4.0",
 				"optionator": "^0.9.1",
-				"regexpp": "^3.2.0",
 				"strip-ansi": "^6.0.1",
 				"strip-json-comments": "^3.1.0",
 				"text-table": "^0.2.0"
@@ -1651,12 +1688,15 @@
 			}
 		},
 		"node_modules/eslint-visitor-keys": {
-			"version": "3.3.0",
-			"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz",
-			"integrity": "sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA==",
+			"version": "3.4.0",
+			"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.0.tgz",
+			"integrity": "sha512-HPpKPUBQcAsZOsHAFwTtIKcYlCje62XB7SEAcxjtmW6TD1WVpkS6i6/hOVtTZIl4zGj/mBqpFVGvaDneik+VoQ==",
 			"dev": true,
 			"engines": {
 				"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
+			},
+			"funding": {
+				"url": "https://opencollective.com/eslint"
 			}
 		},
 		"node_modules/eslint/node_modules/eslint-scope": {
@@ -1688,14 +1728,14 @@
 			"dev": true
 		},
 		"node_modules/espree": {
-			"version": "9.4.1",
-			"resolved": "https://registry.npmjs.org/espree/-/espree-9.4.1.tgz",
-			"integrity": "sha512-XwctdmTO6SIvCzd9810yyNzIrOrqNYV9Koizx4C/mRhf9uq0o4yHoCEU/670pOxOL/MSraektvSAji79kX90Vg==",
+			"version": "9.5.1",
+			"resolved": "https://registry.npmjs.org/espree/-/espree-9.5.1.tgz",
+			"integrity": "sha512-5yxtHSZXRSW5pvv3hAlXM5+/Oswi1AUFqBmbibKb5s6bp3rGIDkyXU6xCoyuuLhijr4SFwPrXRoZjz0AZDN9tg==",
 			"dev": true,
 			"dependencies": {
 				"acorn": "^8.8.0",
 				"acorn-jsx": "^5.3.2",
-				"eslint-visitor-keys": "^3.3.0"
+				"eslint-visitor-keys": "^3.4.0"
 			},
 			"engines": {
 				"node": "^12.22.0 || ^14.17.0 || >=16.0.0"
@@ -2879,9 +2919,9 @@
 			}
 		},
 		"node_modules/rollup": {
-			"version": "3.16.0",
-			"resolved": "https://registry.npmjs.org/rollup/-/rollup-3.16.0.tgz",
-			"integrity": "sha512-9wE1H5N1SJqnROpQanBGJ7lrIitwlUYGj4Va4eyf3+vNhoIHLPLag2/CUGIiq3V9lHOBJB6zTsGsPvc50oeihg==",
+			"version": "3.20.2",
+			"resolved": "https://registry.npmjs.org/rollup/-/rollup-3.20.2.tgz",
+			"integrity": "sha512-3zwkBQl7Ai7MFYQE0y1MeQ15+9jsi7XxfrqwTb/9EK8D9C9+//EBR4M+CuA1KODRaNbFez/lWxA5vhEGZp4MUg==",
 			"dev": true,
 			"bin": {
 				"rollup": "dist/bin/rollup"
@@ -3494,15 +3534,15 @@
 			"integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="
 		},
 		"node_modules/vite": {
-			"version": "4.1.2",
-			"resolved": "https://registry.npmjs.org/vite/-/vite-4.1.2.tgz",
-			"integrity": "sha512-MWDb9Rfy3DI8omDQySbMK93nQqStwbsQWejXRY2EBzEWKmLAXWb1mkI9Yw2IJrc+oCvPCI1Os5xSSIBYY6DEAw==",
+			"version": "4.2.1",
+			"resolved": "https://registry.npmjs.org/vite/-/vite-4.2.1.tgz",
+			"integrity": "sha512-7MKhqdy0ISo4wnvwtqZkjke6XN4taqQ2TBaTccLIpOKv7Vp2h4Y+NpmWCnGDeSvvn45KxvWgGyb0MkHvY1vgbg==",
 			"dev": true,
 			"dependencies": {
-				"esbuild": "^0.16.14",
+				"esbuild": "^0.17.5",
 				"postcss": "^8.4.21",
 				"resolve": "^1.22.1",
-				"rollup": "^3.10.0"
+				"rollup": "^3.18.0"
 			},
 			"bin": {
 				"vite": "bin/vite.js"

package.json

@@ -30,7 +30,7 @@
 	},
 	"type": "module",
 	"dependencies": {
-		"@microsoft/fetch-event-source": "^2.0.1",
+		"@huggingface/inference": "^2.0.0-rc2",
 		"autoprefixer": "^10.4.14",
 		"postcss": "^8.4.21",
 		"tailwind-scrollbar": "^3.0.0",

src/routes/+page.svelte

@@ -1,9 +1,8 @@
 <script lang="ts">
-	import type { Message, StreamResponse } from '$lib/Types';
+	import type { Message } from '$lib/Types';
 
 	import { afterUpdate } from 'svelte';
-
-	import { fetchEventSource } from '@microsoft/fetch-event-source';
+	import { HfInference } from '@huggingface/inference';
 
 	import ChatMessage from '$lib/components/chat/ChatMessage.svelte';
 	import ChatIntroduction from '$lib/components/chat/ChatIntroduction.svelte';
@@ -11,16 +10,18 @@
 
 	import {
 		PUBLIC_ASSISTANT_MESSAGE_TOKEN,
-		PUBLIC_ENDPOINT,
-		PUBLIC_HF_TOKEN,
 		PUBLIC_SEP_TOKEN,
 		PUBLIC_USER_MESSAGE_TOKEN
 	} from '$env/static/public';
+	import { page } from '$app/stores';
 
 	const userToken = PUBLIC_USER_MESSAGE_TOKEN || '<|prompter|>';
 	const assistantToken = PUBLIC_ASSISTANT_MESSAGE_TOKEN || '<|assistant|>';
 	const sepToken = PUBLIC_SEP_TOKEN || '<|endoftext|>';
 
+	const hf = new HfInference();
+	const model = hf.endpoint(`${$page.url.origin}/api/conversation`);
+
 	let messages: Message[] = [];
 	let message = '';
 
@@ -41,6 +42,53 @@
 		}
 	}
 
+	async function getTextGenerationStream(inputs: string) {
+		const response = model.textGenerationStream(
+			{
+				inputs,
+				parameters: {
+					// Taken from https://huggingface.co/spaces/huggingface/open-assistant-private-testing/blob/main/app.py#L54
+					// @ts-ignore
+					stop: ['<|endoftext|>'],
+					max_new_tokens: 1024,
+					truncate: 1024,
+					typical_p: 0.2
+				}
+			},
+			{
+				use_cache: false
+			}
+		);
+
+		// Regex to check if the text finishes by "<" but is not a piece of code like "`<img>`"
+		const endOfTextRegex = /(?<!`)<(?!`)/;
+
+		for await (const data of response) {
+			if (!data) break;
+
+			try {
+				if (!data.token.special) {
+					if (messages.at(-1)?.from !== 'bot') {
+						// First token has a space at the beginning, trim it
+						messages = [...messages, { from: 'bot', content: data.token.text.trimStart() }];
+					} else {
+						const isEndOfText = endOfTextRegex.test(data.token.text);
+
+						messages.at(-1)!.content += isEndOfText
+							? data.token.text.replace('<', '')
+							: data.token.text;
+						messages = messages;
+
+						if (isEndOfText) break;
+					}
+				}
+			} catch (error) {
+				console.error(error);
+				break;
+			}
+		}
+	}
+
 	function onWrite() {
 		if (!message) return;
 
@@ -55,49 +103,7 @@
 				)
 				.join('') + assistantToken;
 
-		console.log(inputs);
-		fetchEventSource(PUBLIC_ENDPOINT, {
-			method: 'POST',
-			headers: {
-				Accept: 'text/event-stream',
-				'Content-Type': 'application/json',
-				'user-agent': 'chat-ui/0.0.1',
-				...(PUBLIC_HF_TOKEN
-					? {
-							authorization: `Bearer ${PUBLIC_HF_TOKEN}`
-					  }
-					: {})
-			},
-			body: JSON.stringify({
-				inputs: inputs,
-				stream: true,
-				parameters: {
-					do_sample: false,
-					max_new_tokens: 500,
-					return_full_text: false,
-					stop: [],
-					truncate: 1000,
-					typical_p: 0.2,
-					watermark: false,
-					details: true
-				}
-			}),
-			async onopen(response) {
-				if (response.ok && response.headers.get('content-type') === 'text/event-stream') {
-					messages = [...messages, { from: 'bot', content: '' }];
-				} else {
-					console.error('error opening the SSE endpoint');
-				}
-			},
-			onmessage(msg) {
-				const data = JSON.parse(msg.data) as StreamResponse;
-				// console.log(data);
-				if (!data.token.special) {
-					messages.at(-1)!.content += data.token.text;
-					messages = messages;
-				}
-			}
-		});
+		getTextGenerationStream(inputs);
 	}
 </script>
 

src/routes/api/conversation/+server.ts

@@ -0,0 +1,14 @@
+import { env } from '$env/dynamic/private';
+import { PUBLIC_MODEL_ENDPOINT } from '$env/static/public';
+
+export async function POST({ request }) {
+	return await fetch(PUBLIC_MODEL_ENDPOINT, {
+		headers: {
+			...request.headers,
+			'Content-Type': 'application/json',
+			Authorization: `Basic ${env.HF_TOKEN}`
+		},
+		method: 'POST',
+		body: await request.text()
+	});
+}