ready for release (sort of)

src/index.mts

@@ -1,6 +1,7 @@
 import { createReadStream, existsSync } from "node:fs"
 import path from "node:path"
 
+import { validate as uuidValidate } from "uuid"
 import express from "express"
 
 import { VideoTask, VideoTaskRequest } from "./types.mts"
@@ -12,7 +13,12 @@ import { completedFilesDirFilePath } from "./config.mts"
 import { deleteTask } from "./scheduler/deleteTask.mts"
 import { getPendingTasks } from "./scheduler/getPendingTasks.mts"
 import { hasValidAuthorization } from "./utils/hasValidAuthorization.mts"
+import { getAllTasksForOwner } from "./scheduler/getAllTasksForOwner.mts"
+import { initFolders } from "./initFolders.mts"
 
+initFolders()
+// to disable all processing (eg. to debug)
+// then comment the following line:
 main()
 
 const app = express()
@@ -59,9 +65,8 @@ app.post("/", async (req, res) => {
   }
 })
 
-// get all pending tasks
-app.get("/", async (req, res) => {
-    
+// only get the tasks for a specific owner
+app.get("/owner/:ownerId", async (req, res) => {
   if (!hasValidAuthorization(req.headers)) {
     console.log("Invalid authorization")
     res.status(401)
@@ -70,21 +75,35 @@ app.get("/", async (req, res) => {
     return
   }
 
+  const ownerId = req.params.ownerId
+
+  if (!uuidValidate(ownerId)) {
+    console.error("invalid owner id")
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid owner id` }))
+    res.end()
+    return
+  }
+
   try {
-    const tasks = await getPendingTasks()
+    const tasks = await getAllTasksForOwner(ownerId)
     res.status(200)
     res.write(JSON.stringify(tasks, null, 2))
     res.end()
   } catch (err) {
     console.error(err)
     res.status(500)
-    res.write(JSON.stringify({ error: "couldn't get the tasks" }))
+    res.write(JSON.stringify({ error: `couldn't get the tasks for owner ${ownerId}` }))
     res.end()
   }
 })
 
-app.get("/:id", async (req, res) => {
+app.get("/download/:id\.mp4", async (req, res) => {
     
+  /*
+  for simplicity, let's skip auth when fetching videos
+  the UUIDs cannot easily be guessed anyway
+
   if (!hasValidAuthorization(req.headers)) {
     console.log("Invalid authorization")
     res.status(401)
@@ -92,55 +111,96 @@ app.get("/:id", async (req, res) => {
     res.end()
     return
   }
+  */
 
-  try {
-    const task = await getTask(req.params.id)
-    res.status(200)
-    res.write(JSON.stringify(task))
+  const [ownerId, videoId] = `${req.params.id}`.split("_")
+
+  if (!uuidValidate(ownerId)) {
+    console.error("invalid owner id")
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid owner id` }))
+    res.end()
+    return
+  }
+
+
+  if (!uuidValidate(videoId)) {
+    console.error("invalid video id")
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid video id` }))
     res.end()
+    return
+  }
+
+  let task: VideoTask = null
+  try {
+    task = await getTask(ownerId, videoId)
+    console.log(`returning video ${videoId} to owner ${ownerId}`)
   } catch (err) {
-    console.error(err)
     res.status(404)
-    res.write(JSON.stringify({ error: "couldn't find this task" }))
+    res.write(JSON.stringify({ error: "this video doesn't exist" }))
     res.end()
+    return
   }
-})
 
-app.delete("/:id", async (req, res) => {
-    
-  if (!hasValidAuthorization(req.headers)) {
-    console.log("Invalid authorization")
-    res.status(401)
-    res.write(JSON.stringify({ error: "invalid token" }))
+  const completedFilePath = path.join(completedFilesDirFilePath, task.fileName)
+
+  // note: we DON'T want to use the pending file path, as there may be operations on it
+  // (ie. a process might be busy writing stuff to it)
+  const filePath = existsSync(completedFilePath) ? completedFilePath : ""
+  if (!filePath) {
+    res.status(400)
+    res.write(JSON.stringify({ error: "video exists, but cannot be previewed yet" }))
     res.end()
     return
   }
 
-
-  let task: VideoTask = null
+  // file path exists, let's try to read it
   try {
-    task = await getTask(req.params.id)
+    // do we need this?
+    // res.status(200)
+    // res.setHeader("Content-Type", "media/mp4")
+    console.log(`creating a video read stream from ${filePath}`)
+    const stream = createReadStream(filePath)
+  
+    stream.on('close', () => {
+      console.log(`finished streaming the video`)
+      res.end()
+    })
+    
+    stream.pipe(res)
   } catch (err) {
-    console.error(err)
-    res.status(404)
-    res.write(JSON.stringify({ error: "couldn't find this task" }))
+    console.error(`failed to read the video file at ${filePath}: ${err}`)
+    res.status(500)
+    res.write(JSON.stringify({ error: "failed to read the video file" }))
     res.end()
   }
+})
+
+// get all pending tasks
+app.get("/", async (req, res) => {
+  if (!hasValidAuthorization(req.headers)) {
+    console.log("Invalid authorization")
+    res.status(401)
+    res.write(JSON.stringify({ error: "invalid token" }))
+    res.end()
+    return
+  }
 
   try {
-    await deleteTask(task)
+    const tasks = await getPendingTasks()
     res.status(200)
-    res.write(JSON.stringify({ success: true }))
+    res.write(JSON.stringify(tasks, null, 2))
     res.end()
   } catch (err) {
     console.error(err)
     res.status(500)
-    res.write(JSON.stringify({ success: false, error: "failed to delete the task" }))
+    res.write(JSON.stringify({ error: "couldn't get the tasks" }))
     res.end()
   }
 })
 
-app.get("/video/:id\.mp4", async (req, res) => {
+app.get("/:id", async (req, res) => {
     
   if (!hasValidAuthorization(req.headers)) {
     console.log("Invalid authorization")
@@ -150,55 +210,85 @@ app.get("/video/:id\.mp4", async (req, res) => {
     return
   }
 
+  const [ownerId, videoId] = `${req.params.id}`.split("_")
 
-  if (!req.params.id) {
+  if (!uuidValidate(ownerId)) {
+    console.error("invalid owner id")
     res.status(400)
-    res.write(JSON.stringify({ error: "please provide a valid video id" }))
+    res.write(JSON.stringify({ error: `invalid owner id` }))
+    res.end()
+    return
+  }
+
+
+  if (!uuidValidate(videoId)) {
+    console.error("invalid video id")
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid video id` }))
     res.end()
     return
   }
 
-  let task: VideoTask = null
   try {
-    task = await getTask(req.params.id)
-    console.log("returning result to user..")
+    const task = await getTask(ownerId, videoId)
+    res.status(200)
+    res.write(JSON.stringify(task))
+    res.end()
   } catch (err) {
+    console.error(err)
     res.status(404)
-    res.write(JSON.stringify({ error: "this video doesn't exist" }))
+    res.write(JSON.stringify({ error: "couldn't find this task" }))
+    res.end()
+  }
+})
+
+app.delete("/:id", async (req, res) => {
+    
+  if (!hasValidAuthorization(req.headers)) {
+    console.log("Invalid authorization")
+    res.status(401)
+    res.write(JSON.stringify({ error: "invalid token" }))
     res.end()
     return
   }
 
-  const completedFilePath = path.join(completedFilesDirFilePath, task.fileName)
+  const [ownerId, videoId] = `${req.params.id}`.split("_")
 
-  // note: we DON'T want to use the pending file path, as there may be operations on it
-  // (ie. a process might be busy writing stuff to it)
-  const filePath = existsSync(completedFilePath) ? completedFilePath : ""
-  if (!filePath) {
+  if (!uuidValidate(ownerId)) {
+    console.error("invalid owner id")
     res.status(400)
-    res.write(JSON.stringify({ error: "video exists, but cannot be previewed yet" }))
+    res.write(JSON.stringify({ error: `invalid owner id` }))
     res.end()
     return
   }
 
-  // file path exists, let's try to read it
+  if (!uuidValidate(videoId)) {
+    console.error("invalid video id")
+    res.status(400)
+    res.write(JSON.stringify({ error: `invalid video id` }))
+    res.end()
+    return
+  }
+
+  let task: VideoTask = null
   try {
-    // do we need this?
-    // res.status(200)
-    // res.setHeader("Content-Type", "media/mp4")
-    console.log(`creating a video read stream from ${filePath}`)
-    const stream = createReadStream(filePath)
-  
-    stream.on('close', () => {
-      console.log(`finished streaming the video`)
-      res.end()
-    })
-    
-    stream.pipe(res)
+    task = await getTask(ownerId, videoId)
   } catch (err) {
-    console.error(`failed to read the video file at ${filePath}: ${err}`)
+    console.error(err)
+    res.status(404)
+    res.write(JSON.stringify({ error: "couldn't find this task" }))
+    res.end()
+  }
+
+  try {
+    await deleteTask(task)
+    res.status(200)
+    res.write(JSON.stringify({ success: true }))
+    res.end()
+  } catch (err) {
+    console.error(err)
     res.status(500)
-    res.write(JSON.stringify({ error: "failed to read the video file" }))
+    res.write(JSON.stringify({ success: false, error: "failed to delete the task" }))
     res.end()
   }
 })

src/main.mts

@@ -2,8 +2,6 @@ import { initFolders } from "./initFolders.mts"
 import { getPendingTasks } from "./scheduler/getPendingTasks.mts"
 import { processTask } from "./scheduler/processTask.mts"
 
-initFolders()
-
 export const main = async () => {
 
   const tasks = await getPendingTasks()

src/scheduler/deleteTask.mts

@@ -10,7 +10,7 @@ import { deleteFileIfExists } from "../utils/deleteFileIfExists.mts"
 
 
 export const deleteTask = async (task: VideoTask) => {
-  const taskFileName = `${task.id}.json`
+  const taskFileName = `${task.ownerId}_${task.id}.json`
   const videoFileName = task.fileName
   
   // .mp4 files

src/scheduler/getAllTasksForOwner.mts

@@ -0,0 +1,9 @@
+import { VideoTask } from "../types.mts"
+import { getCompletedTasks } from "./getCompletedTasks.mts"
+import { getPendingTasks } from "./getPendingTasks.mts"
+
+export const getAllTasksForOwner = async (ownerId: string): Promise<VideoTask[]> => {
+  const pendingTasks = await getPendingTasks(ownerId)
+  const completedTasks = await getCompletedTasks(ownerId)
+  return [...pendingTasks, ...completedTasks]
+}

src/scheduler/getCompletedTasks.mts

@@ -2,8 +2,8 @@ import { VideoTask } from "../types.mts"
 import { completedTasksDirFilePath } from "../config.mts"
 import { readTasks } from "./readTasks.mts"
 
-export const getCompletedTasks = async (): Promise<VideoTask[]> => {
-  const completedTasks = await readTasks(completedTasksDirFilePath)
+export const getCompletedTasks = async (ownerId?: string): Promise<VideoTask[]> => {
+  const completedTasks = await readTasks(completedTasksDirFilePath, ownerId)
 
   return completedTasks
 }

src/scheduler/getPendingTasks.mts

@@ -2,8 +2,8 @@ import { VideoTask } from "../types.mts"
 import { pendingTasksDirFilePath } from "../config.mts"
 import { readTasks } from "./readTasks.mts"
 
-export const getPendingTasks = async (): Promise<VideoTask[]> => {
-  const pendingTasks = await readTasks(pendingTasksDirFilePath)
+export const getPendingTasks = async (ownerId?: string): Promise<VideoTask[]> => {
+  const pendingTasks = await readTasks(pendingTasksDirFilePath, ownerId)
 
   return pendingTasks
 }

src/scheduler/getTask.mts

@@ -3,8 +3,8 @@ import path from "node:path"
 import { completedTasksDirFilePath, pendingTasksDirFilePath } from "../config.mts"
 import { readTask } from "./readTask.mts"
 
-export const getTask = async (id: string) => {
-  const taskFileName = `${id}.json`
+export const getTask = async (ownerId: string, videoId: string) => {
+  const taskFileName = `${ownerId}_${videoId}.json`
 
   const completedTaskFilePath = path.join(completedTasksDirFilePath, taskFileName)
   const pendingTaskFilePath = path.join(pendingTasksDirFilePath, taskFileName)
@@ -17,7 +17,7 @@ export const getTask = async (id: string) => {
       const pendingTask = await readTask(pendingTaskFilePath)
       return pendingTask
     } catch (err) {
-      throw new Error(`couldn't find task ${id}`)
+      throw new Error(`couldn't find video task ${videoId} for owner ${ownerId}`)
     }
   }
 }

src/scheduler/processTask.mts

@@ -11,6 +11,7 @@ import { interpolateVideo } from "../production/interpolateVideo.mts"
 import { postInterpolation } from "../production/postInterpolation.mts"
 import { moveVideoFromPendingToCompleted } from "../utils/moveVideoFromPendingToCompleted.mts"
 import { assembleShots } from "../production/assembleShots.mts"
+import { copyVideoFromPendingToCompleted } from "../utils/copyVideoFromPendingToCompleted.mts"
 
 export const processTask = async (task: VideoTask) => {
   console.log(`processing video task ${task.id}`)
@@ -118,6 +119,8 @@ export const processTask = async (task: VideoTask) => {
         shot.nbCompletedSteps++
 
         await updatePendingTask(task)
+
+        await copyVideoFromPendingToCompleted(shot.fileName, task.fileName)
       } catch (err) {
         console.error(`failed to upscale shot ${shot.id} (${err})`)
         // something is wrong, let's put the whole thing back into the queue
@@ -155,6 +158,8 @@ export const processTask = async (task: VideoTask) => {
 
         await updatePendingTask(task)
 
+        await copyVideoFromPendingToCompleted(shot.fileName, task.fileName)
+
       } catch (err) {
         console.error(`failed to interpolate shot ${shot.id} (${err})`)
         // something is wrong, let's put the whole thing back into the queue
@@ -183,6 +188,8 @@ export const processTask = async (task: VideoTask) => {
 
         await updatePendingTask(task)
 
+        await copyVideoFromPendingToCompleted(shot.fileName, task.fileName)
+
       } catch (err) {
         console.error(`failed to post-process shot ${shot.id} (${err})`)
         // something is wrong, let's put the whole thing back into the queue

src/scheduler/readTask.mts

@@ -1,5 +1,4 @@
 import { promises as fs } from "node:fs"
-import path from "node:path"
 
 import { VideoTask } from "../types.mts"
 

src/scheduler/readTasks.mts

@@ -5,7 +5,7 @@ import { VideoTask } from "../types.mts"
 import { readTask } from "./readTask.mts"
 
 
-export const readTasks = async (taskDirFilePath: string): Promise<VideoTask[]> => {
+export const readTasks = async (taskDirFilePath: string, ownerId?: string): Promise<VideoTask[]> => {
 
   let tasksFiles: string[] = []
   try {
@@ -13,7 +13,9 @@ export const readTasks = async (taskDirFilePath: string): Promise<VideoTask[]> =
     // console.log("filesInDir:", filesInDir)
 
     // we only keep valid files (in UUID.json format)
-    tasksFiles = filesInDir.filter(fileName => fileName.match(/[a-z0-9\-]\.json/i))
+    tasksFiles = filesInDir.filter(fileName =>
+      fileName.match(/[a-z0-9\-_]\.json/i) && (ownerId ? fileName.includes(ownerId): true)
+    )
 
     // console.log("tasksfiles:", tasksFiles)
   } catch (err) {

src/scheduler/saveCompletedTask.mts

@@ -1,4 +1,3 @@
-import { promises as fs } from "node:fs"
 import path from "path"
 
 import { VideoTask } from "../types.mts"
@@ -6,7 +5,7 @@ import { completedTasksDirFilePath, pendingTasksDirFilePath } from "../config.mt
 import { moveFile } from "../utils/moveFile.mts"
 
 export const saveCompletedTask = async (task: VideoTask) => {
-  const fileName = `${task.id}.json`
+  const fileName = `${task.ownerId}_${task.id}.json`
   const pendingFilePath = path.join(pendingTasksDirFilePath, fileName)
   const completedFilePath = path.join(completedTasksDirFilePath, fileName)
   await moveFile(pendingFilePath, completedFilePath)

src/scheduler/savePendingTask.mts

@@ -5,9 +5,7 @@ import { VideoTask } from "../types.mts"
 import { pendingTasksDirFilePath } from "../config.mts"
 
 export const savePendingTask = async (task: VideoTask) => {
-  const fileName = `${task.id}.json`
+  const fileName = `${task.ownerId}_${task.id}.json`
   const filePath = path.join(pendingTasksDirFilePath, fileName)
-  console.log("fileName:", fileName)
-  console.log("filePath:", filePath)
   await fs.writeFile(filePath, JSON.stringify(task, null, 2), "utf8")
 }

src/scheduler/updatePendingTask.mts

@@ -6,7 +6,7 @@ import { pendingTasksDirFilePath } from "../config.mts"
 
 export const updatePendingTask = async (task: VideoTask) => {
   try {
-    const fileName = `${task.id}.json`
+    const fileName = `${task.ownerId}_${task.id}.json`
     const filePath = path.join(pendingTasksDirFilePath, fileName)
     await fs.writeFile(filePath, JSON.stringify(task, null, 2), "utf8")
   } catch (err) {

src/types.mts

@@ -202,6 +202,8 @@ export interface VideoSequenceMeta {
 export interface VideoSequenceData {
   // must be unique
   id: string
+
+  ownerId: string
   
   fileName: string
 
@@ -218,11 +220,12 @@ export interface VideoSequenceData {
 
 export type VideoSequence = VideoSequenceMeta & VideoSequenceData
 
-export type VideoTaskRequest = {
+export type VideoTaskRequest = Partial<{
   prompt: string
+  ownerId: string // to uniquely identify where the video come from
   sequence: Partial<VideoSequenceMeta>
   shots: Array<Partial<VideoShotMeta>>
-}
+}>
 
 export type VideoTask = VideoSequence & {
   shots: VideoShot[]

src/utils/copyVideoFromPendingToCompleted.mts

@@ -1,14 +1,14 @@
 import path from "node:path"
 import { promises as fs } from "node:fs"
 
-import { completedTasksDirFilePath, pendingFilesDirFilePath } from "../config.mts"
+import { completedFilesDirFilePath, pendingFilesDirFilePath } from "../config.mts"
 
 export const copyVideoFromPendingToCompleted = async (pendingFileName: string, completedFileName?: string) => {
   if (!completedFileName) {
     completedFileName = pendingFileName
   }
   const pendingFilePath = path.join(pendingFilesDirFilePath, pendingFileName)
-  const completedFilePath = path.join(completedTasksDirFilePath, completedFileName)
+  const completedFilePath = path.join(completedFilesDirFilePath, completedFileName)
 
   await fs.copyFile(pendingFilePath, completedFilePath)
   console.log(`copied file from ${pendingFilePath} to ${completedFilePath}`)

src/utils/parseShotRequest.mts

@@ -57,7 +57,7 @@ export const parseShotRequest = async (sequence: VideoSequence, maybeShotMeta: P
     // for internal use
 
     version: shotFormatVersion,
-    fileName: `${id}.mp4`,
+    fileName: `${sequence.ownerId}_${sequence.id}_${id}.mp4`,
     hasGeneratedPreview: false,
     hasGeneratedVideo: false,
     hasUpscaledVideo: false,

src/utils/parseVideoRequest.mts

@@ -1,4 +1,4 @@
-import { v4 as uuidv4 } from "uuid"
+import { v4 as uuidv4, validate as uuidValidate } from "uuid"
 import { HfInference } from "@huggingface/inference"
 
 // convert a request (which might be invalid)
@@ -29,11 +29,23 @@ export const parseVideoRequest = async (request: VideoTaskRequest): Promise<Vide
     }]
   }
 
-  console.log("continuing..")
+  // more or less check that we have a UUID
+  // (I think we can also have an exact match over length === 34)
+  if (uuidValidate(request.ownerId)) {
+    console.log("we have a valid owner:", request.ownerId)
+      // TODO: use llama2 to populate this!
+    request.ownerId
+  } else {
+    request.ownerId = uuidv4()
+  }
+
+  // console.log("continuing..")
   const task: VideoTask = {
     // ------------ VideoSequenceMeta -------------
     id,
 
+    ownerId: request.ownerId,
+
     // describe the whole movie
     videoPrompt: `${request.sequence.videoPrompt || ''}`,
 
@@ -67,7 +79,7 @@ export const parseVideoRequest = async (request: VideoTaskRequest): Promise<Vide
 
     // ---------- VideoSequenceData ---------
     version: sequenceFormatVersion,
-    fileName: `${id}.mp4`,
+    fileName: `${request.ownerId}_${id}.mp4`,
     hasAssembledVideo: false,
     nbCompletedShots: 0,
     progressPercent: 0,
@@ -81,12 +93,12 @@ export const parseVideoRequest = async (request: VideoTaskRequest): Promise<Vide
     shots: [],
   }
 
-  console.log("we are still good..")
+  // console.log("we are still good..")
   const maybeShots = Array.isArray(request.shots) ? request.shots : []
 
-  console.log("let's try..")
+  // console.log("let's try..")
   for (const maybeShot of maybeShots) {
-    console.log("trying shot", maybeShot)
+    // console.log("trying shot", maybeShot)
     try {
       const shot = await parseShotRequest(task, maybeShot)
       task.shots.push(shot)