add support for real-time

.nvmrc

@@ -1 +1 @@
-v20.10.0
+v20.10.0

package.json

@@ -4,11 +4,11 @@
   "description": "A service which wraps and chains video and audio spaces together",
   "main": "src/index.mts",
   "scripts": {
-    "start": "node --loader ts-node/esm src/index.mts",
-    "test:submitVideo": "node --loader ts-node/esm src/tests/submitVideo.mts",
-    "test:checkStatus": "node --loader ts-node/esm src/tests/checkStatus.mts",
-    "test:downloadFileToTmp": "node --loader ts-node/esm src/tests/downloadFileToTmp.mts",
-    "test:stuff": "node --loader ts-node/esm src/utils/segmentImage.mts",
+    "start": "tsx  src/index.mts",
+    "test:submitVideo": "tsx  src/tests/submitVideo.mts",
+    "test:checkStatus": "tsx  src/tests/checkStatus.mts",
+    "test:downloadFileToTmp": "tsx src/tests/downloadFileToTmp.mts",
+    "test:stuff": "tsx src/utils/segmentImage.mts",
     "docker": "npm run docker:build && npm run docker:run",
     "docker:build": "docker build -t videochain-api .",
     "docker:run": "docker run -it -p 7860:7860 videochain-api"
@@ -21,24 +21,24 @@
     "@gradio/client": "^0.1.4",
     "@huggingface/inference": "^2.6.1",
     "@types/express": "^4.17.17",
-    "@types/ffmpeg-concat": "^1.1.2",
+    "@types/node": "^20.12.7",
     "@types/uuid": "^9.0.2",
     "dotenv": "^16.3.1",
     "eventsource-parser": "^1.0.0",
     "express": "^4.18.2",
-    "ffmpeg-concat": "^1.3.0",
     "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^11.1.1",
     "gpt-tokens": "^1.1.1",
     "node-fetch": "^3.3.1",
     "nodejs-whisper": "^0.1.4",
-    "openai": "^3.3.0",
-    "puppeteer": "^21.5.1",
-    "replicate": "^0.21.1",
+    "openai": "^4.38.2",
+    "puppeteer": "^22.6.5",
+    "replicate": "^0.29.1",
     "resize-base64": "^1.0.12",
     "sharp": "^0.32.4",
     "temp-dir": "^3.0.0",
-    "ts-node": "^10.9.1",
+    "ts-node": "^10.9.2",
+    "tsx": "^4.7.0",
     "tts-react": "^3.0.1",
     "uuid": "^9.0.0",
     "yaml": "^2.3.1"

src/production/renderVideo.mts

@@ -2,7 +2,8 @@ import { RenderedScene, RenderRequest, VideoGenerationParams } from "../types.mt
 
 // import { generateVideo } from "../providers/video-generation/generateVideoWithZeroscope.mts"
 // import { generateVideo } from "../providers/video-generation/generateVideoWithHotshotGradioAPI.mts"
-import { generateVideoWithAnimateLCM } from "../providers/video-generation/generateVideoWithAnimateLCM.mts"
+// import { generateVideoWithAnimateLCM } from "../providers/video-generation/generateVideoWithAnimateLCM.mts"
+import { generateVideoWithAnimateDiffLightning } from "../providers/video-generation/generateVideoWithAnimateDiffLightning.mts"
 import { generateSeed } from "../utils/misc/generateSeed.mts"
 
 export async function renderVideo(
@@ -10,25 +11,7 @@ export async function renderVideo(
   response: RenderedScene
 ): Promise<RenderedScene> {
 
-  const params: VideoGenerationParams = {
-    prompt: request.prompt,
-    // image?: undefined, // can be empty (and thus, is empty)
-    // lora?: string // hardcoded on "3D render"
-    // style?: string // hardcoded on "3D render" for now
-    orientation: "landscape",
-    projection: "cartesian",
-    width: 512,
-    height: 256,
-    
-    // ok, now what about those? they are in the gradio, are not exposed yet in the API
-    // nbFrames: request.nbFrames,
-    // nbSteps: request.nbSteps,
-
-    seed: request.seed || generateSeed(),
-    debug: true,
-  }
-
-  response.assetUrl = await generateVideoWithAnimateLCM(params)
+  response.assetUrl = await generateVideoWithAnimateDiffLightning(request, response)
 
   return response
 }

src/providers/language-model/openai/openai.mts

@@ -1,7 +1,5 @@
-import { Configuration, OpenAIApi } from "openai"
+import OpenAI from "openai";
 
-export const openai = new OpenAIApi(
-  new Configuration({
-    apiKey: process.env.VC_OPENAI_API_KEY
-  })
-)
+export const openai = new OpenAI({
+  apiKey: `${process.env.VC_OPENAI_API_KEY || ""}`
+})

src/providers/video-generation/defaultPrompts.mts

@@ -14,9 +14,13 @@ export function addWordsIfNotPartOfThePrompt(prompt: string = "", words: string[
  export function getPositivePrompt(prompt: string, triggerWord = "") {
   return addWordsIfNotPartOfThePrompt(prompt, [
     triggerWord,
-    "crisp",
+    "sublime",
+    "pro quality",
     "sharp",
+    "crisp",
     "beautiful",
+    "impressive",
+    "amazing",
     "4K",
     "hd"
   ])
@@ -28,9 +32,13 @@ export function getNegativePrompt(prompt: string) {
     // "underexposed", // <-- can be a desired style
     // "overexposed", // <-- can be a desired style
     "logo",
+    "hud",
+    "ui",
     "censored",
+    "blurry",
     "watermark",
     "watermarked",
+    "copyright",
     "extra digit",
     "fewer digits",
     "bad fingers",
@@ -39,11 +47,11 @@ export function getNegativePrompt(prompt: string) {
     "low quality",
     "low resolution",
     "glitch", // <-- keep or not? could be a desired style?
-    "deformed",
-    "mutated",
-    "ugly",
-    "disfigured",
-    "3D render", // <-- keep or not? could be a desired style?
+    // "deformed",
+    // "mutated",
+    // "ugly",
+    // "disfigured",
+    // "3D render", // <-- keep or not? could be a desired style?
     "signature"
   ])
 }

src/providers/video-generation/generateVideoWithAnimateDiffLightning.mts

@@ -0,0 +1,123 @@
+import { RenderedScene, RenderRequest } from "../../types.mts"
+import { generateSeed } from "../../utils/misc/generateSeed.mts"
+import { tryApiCalls } from "../../utils/misc/tryApiCall.mts"
+import { getValidNumber } from "../../utils/validators/getValidNumber.mts"
+
+// const gradioApi = `${process.env.AI_TUBE_MODEL_ANIMATELCM_GRADIO_URL || ""}`
+const gradioApi = "https://jbilcke-hf-ai-tube-model-animatediff-lightning.hf.space"
+const accessToken = `${process.env.VC_MICROSERVICE_SECRET_TOKEN || ""}`
+
+export const generateVideoWithAnimateDiffLightning = async (
+  request: RenderRequest,
+  response: RenderedScene,
+): Promise<RenderedScene> => {
+  
+  const debug = false
+
+  const actualFunction = async () => {
+
+    const prompt = request.prompt || ""
+    if (!prompt) {
+      response.error = "prompt is empty"
+      return response
+    }
+    
+    // seed = seed || generateSeed()
+    request.seed = request.seed || generateSeed()
+
+    // see https://huggingface.co/spaces/jbilcke-hf/ai-tube-model-animatediff-lightning/blob/main/app.py#L15-L18
+    const baseModel = "epiCRealism"
+
+    // the motion LoRA - could be useful one day
+    const motion = ""
+
+    // can be 1, 2, 4 or 8
+    // but values below 4 look bad
+    const nbSteps = getValidNumber(request.nbSteps, 1, 8, 4)
+    const width = getValidNumber(request.width, 256, 1024, 512)
+    const height = getValidNumber(request.height, 256, 1024, 256)
+
+    const nbFrames = getValidNumber(request.nbFrames, 10, 60, 20)
+    const nbFPS = getValidNumber(request.nbFPS, 10, 60, 10)
+
+    // by default AnimateDiff generates about 2 seconds of video at 10 fps
+    // the Gradio API now has some code to optional fix that using FFmpeg,
+    // but this will add some delay overhead, so use with care!
+    const durationInSec = Math.round(nbFrames / nbFPS)
+    const framesPerSec = nbFPS
+
+    try {
+      if (debug) {
+        console.log(`calling AnimateDiff Lightning API with params (some are hidden):`, {
+          baseModel,
+          motion,
+          nbSteps,
+          width,
+          height,
+          nbFrames,
+          nbFPS,
+          durationInSec,
+          framesPerSec,
+        })
+      }
+
+      const res = await fetch(gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict", {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          // Authorization: `Bearer ${token}`,
+        },
+        body: JSON.stringify({
+          fn_index: 0, // <- important! it is currently 4, not 1!
+          data: [
+            accessToken,
+            prompt,
+            baseModel,
+            width,
+            height,
+            motion,
+            nbSteps,
+            durationInSec,
+            framesPerSec,
+          ],
+        }),
+        cache: "no-store",
+        // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
+        // next: { revalidate: 1 }
+      })
+
+      // console.log("res:", res)
+
+      const { data } = await res.json()
+
+      // console.log("data:", data)
+      // Recommendation: handle errors
+      if (res.status !== 200 || !Array.isArray(data)) {
+        // This will activate the closest `error.js` Error Boundary
+        throw new Error(`Failed to fetch data (status: ${res.status})`)
+      }
+      // console.log("data:", data.slice(0, 50))
+    
+      const base64Content = (data?.[0] || "") as string
+
+      if (!base64Content) {
+        throw new Error(`invalid response (no content)`)
+      }
+
+      // this API already emits a data-uri with a content type
+      return base64Content // addBase64HeaderToMp4(base64Content)
+    } catch (err) {
+      if (debug) {
+        console.error(`failed to call the AnimateDiff Lightning API:`)
+        console.error(err)
+      }
+      throw err
+    }
+  }
+
+  return tryApiCalls({
+    func: actualFunction,
+    debug,
+    failureMessage: "failed to call the AnimateDiff Lightning API"
+  })
+}

src/providers/video-generation/generateVideoWithAnimateLCM.mts

@@ -16,14 +16,14 @@ export const generateVideoWithAnimateLCM = async ({
   width,
   height,
   style = "",
+  nbSteps = 4,
+  nbFrames = 20,
   seed,
   debug,
 }: VideoGenerationParams): Promise<string> => {
   
 
   const actualFunction = async () => {
-    const negPrompt = ""
-    prompt = prompt || ""
     
     // seed = seed || generateSeed()
     seed = generateSeed()
@@ -42,14 +42,12 @@ export const generateVideoWithAnimateLCM = async ({
     const lcmLoraAlpha = 0.8 // spatial_lora_slider,
 
     // label="Width", value=512, minimum=256, maximum=1024, step=64)
-    const width = 512
 
     // label="Animation length", value=16,  minimum=12,   maximum=20,   step=1)
     const nbFrames = 16
 
     // label="Height", value=512, minimum=256, maximum=1024, step=64)
-    const height = 256 
-
+   
     //  label="CFG Scale", value=1.5, minimum=1,   maximum=2)
     const cfgScale = 1.5
 

src/types.mts

@@ -289,12 +289,9 @@ export type RenderRequest = {
   // actionnables are names of things like "chest", "key", "tree", "chair" etc
   actionnables: string[]
 
-  // note: this is the number of frames for Zeroscope,
-  // which is currently configured to only output 3 seconds, so:
-  // nbFrames=8 -> 1 sec
-  // nbFrames=16 -> 2 sec
-  // nbFrames=24 -> 3 sec
-  nbFrames: number // min: 8, max: 24
+  nbFrames: number
+
+  nbFPS: number
 
   nbSteps: number // min: 1, max: 50
 
@@ -426,6 +423,7 @@ export type VideoGenerationParams = {
   projection: VideoProjection
   width: number
   height: number
+  nbSteps: number
   seed?: number
   debug?: boolean
 }

src/utils/requests/parseRenderRequest.mts

@@ -7,7 +7,11 @@ export function parseRenderRequest(request: RenderRequest) {
 
   // console.log("parseRenderRequest: "+JSON.stringify(request, null, 2))
   try {
-    request.nbFrames = getValidNumber(request.nbFrames, 1, 24, 16)
+    // we are large on the values here, since each model will have their own limits
+    // we just want pseudo-valid numbers
+
+    request.nbFrames = getValidNumber(request.nbFrames, 1, 2147483647, 1)
+    request.nbFPS = getValidNumber(request.nbFPS, 1, 2147483647, 1)
 
     request.negativePrompt = request.negativePrompt || ""