adding a small hack/workaround for Pythonia async timeouts

Dockerfile

@@ -29,4 +29,8 @@ RUN pip install --upgrade pip && \
 
 COPY --chown=user . .
 
+# a limitation of Pythonia is that async calls can timeout
+# so we perform the slow operation of downloading the model to cache beforehand
+RUN python download-model.py
+
 CMD [ "npm", "run", "start" ]

download-model.py

@@ -0,0 +1,4 @@
+# a small hack for Pythonia, this allows us to download WizardCoder to it's download directory
+# that way it will become readily available to our Node program
+from ctransformers import AutoModelForCausalLM
+AutoModelForCausalLM.from_pretrained("TheBloke/WizardCoder-15B-1.0-GGML", model_file="WizardCoder-15B-1.0.ggmlv3.q4_0.bin", model_type="starcoder")

package.json

@@ -8,7 +8,6 @@
     "docker": "npm run docker:build && npm run docker:run",
     "docker:build": "docker build -t template-node-wizardcoder-express .",
     "docker:run": "docker run -it -p 7860:7860 template-node-wizardcoder-express"
-
   },
   "author": "Julian Bilcke <[email protected]>",
   "license": "Apache License",

src/index.mts

@@ -2,7 +2,7 @@ import express from 'express'
 import { python } from 'pythonia'
 
 const { AutoModelForCausalLM } = await python('ctransformers')
-const llm = AutoModelForCausalLM.from_pretrained$(
+const llm = await AutoModelForCausalLM.from_pretrained$(
   'TheBloke/WizardCoder-15B-1.0-GGML', {
     model_file: 'WizardCoder-15B-1.0.ggmlv3.q4_0.bin',
     model_type: 'starcoder'