Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
Commit
·
6f310ff
1
Parent(s):
75aad1e
trying some optimizations
Browse files
src/production/renderStaticScene.mts
CHANGED
@@ -4,7 +4,6 @@ import { v4 as uuidv4 } from "uuid"
|
|
4 |
import tmpDir from "temp-dir"
|
5 |
|
6 |
import { ImageSegment, RenderedScene, RenderRequest } from "../types.mts"
|
7 |
-
import { downloadFileToTmp } from "../utils/downloadFileToTmp.mts"
|
8 |
import { segmentImage } from "../utils/segmentImage.mts"
|
9 |
import { generateImageSDXLAsBase64 } from "../utils/generateImageSDXL.mts"
|
10 |
import { writeBase64ToFile } from "../utils/writeBase64ToFile.mts"
|
@@ -58,7 +57,7 @@ export async function renderStaticScene(scene: RenderRequest): Promise<RenderedS
|
|
58 |
|
59 |
const tmpImageFilePath = path.join(tmpDir, `${uuidv4()}.png`)
|
60 |
|
61 |
-
console.log("beginning:", imageBase64.slice(0, 100))
|
62 |
await writeBase64ToFile(imageBase64, tmpImageFilePath)
|
63 |
console.log("wrote the image to ", tmpImageFilePath)
|
64 |
|
@@ -67,10 +66,30 @@ export async function renderStaticScene(scene: RenderRequest): Promise<RenderedS
|
|
67 |
error = "failed to segment the image"
|
68 |
} else {
|
69 |
console.log("got the first frame! segmenting..")
|
70 |
-
|
71 |
-
|
72 |
-
|
73 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
74 |
}
|
75 |
} else {
|
76 |
console.log("no actionnables: just returning the image, then")
|
|
|
4 |
import tmpDir from "temp-dir"
|
5 |
|
6 |
import { ImageSegment, RenderedScene, RenderRequest } from "../types.mts"
|
|
|
7 |
import { segmentImage } from "../utils/segmentImage.mts"
|
8 |
import { generateImageSDXLAsBase64 } from "../utils/generateImageSDXL.mts"
|
9 |
import { writeBase64ToFile } from "../utils/writeBase64ToFile.mts"
|
|
|
57 |
|
58 |
const tmpImageFilePath = path.join(tmpDir, `${uuidv4()}.png`)
|
59 |
|
60 |
+
// console.log("beginning:", imageBase64.slice(0, 100))
|
61 |
await writeBase64ToFile(imageBase64, tmpImageFilePath)
|
62 |
console.log("wrote the image to ", tmpImageFilePath)
|
63 |
|
|
|
66 |
error = "failed to segment the image"
|
67 |
} else {
|
68 |
console.log("got the first frame! segmenting..")
|
69 |
+
try {
|
70 |
+
const result = await segmentImage(tmpImageFilePath, actionnables, width, height)
|
71 |
+
mask = result.pngInBase64
|
72 |
+
segments = result.segments
|
73 |
+
console.log(`it worked the first time! got ${segments.length} segments`)
|
74 |
+
} catch (err) {
|
75 |
+
console.log("this takes too long :/ trying another server..")
|
76 |
+
try {
|
77 |
+
const result = await segmentImage(tmpImageFilePath, actionnables, width, height)
|
78 |
+
mask = result.pngInBase64
|
79 |
+
segments = result.segments
|
80 |
+
console.log(`it worked the second time! got ${segments.length} segments`)
|
81 |
+
} catch (err) {
|
82 |
+
console.log("trying one last time, on a 3rd server..")
|
83 |
+
try {
|
84 |
+
const result = await segmentImage(tmpImageFilePath, actionnables, width, height)
|
85 |
+
mask = result.pngInBase64
|
86 |
+
segments = result.segments
|
87 |
+
console.log(`it worked the third time! got ${segments.length} segments`)
|
88 |
+
} catch (err) {
|
89 |
+
console.log("yeah, all servers are busy it seems.. aborting")
|
90 |
+
}
|
91 |
+
}
|
92 |
+
}
|
93 |
}
|
94 |
} else {
|
95 |
console.log("no actionnables: just returning the image, then")
|
src/production/renderVideoScene.mts
CHANGED
@@ -16,21 +16,36 @@ export async function renderVideoScene(scene: RenderRequest): Promise<RenderedSc
|
|
16 |
const width = 576
|
17 |
const height = 320
|
18 |
|
|
|
|
|
|
|
|
|
|
|
|
|
19 |
try {
|
20 |
-
url = await generateVideo(scene.prompt,
|
21 |
-
seed: getValidNumber(scene.seed, 0, 2147483647, generateSeed()),
|
22 |
-
nbFrames: getValidNumber(scene.nbFrames, 8, 24, 16), // 2 seconds by default
|
23 |
-
nbSteps: getValidNumber(scene.nbSteps, 1, 50, 10), // use 10 by default to go fast, but not too sloppy
|
24 |
-
})
|
25 |
// console.log("successfull generation")
|
26 |
error = ""
|
27 |
if (!url?.length) {
|
28 |
throw new Error(`url for the generated image is empty`)
|
29 |
}
|
30 |
} catch (err) {
|
31 |
-
error
|
32 |
-
|
|
|
|
|
|
|
|
|
33 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
34 |
|
35 |
|
36 |
// TODO add segmentation here
|
|
|
16 |
const width = 576
|
17 |
const height = 320
|
18 |
|
19 |
+
const params = {
|
20 |
+
seed: getValidNumber(scene.seed, 0, 2147483647, generateSeed()),
|
21 |
+
nbFrames: getValidNumber(scene.nbFrames, 8, 24, 16), // 2 seconds by default
|
22 |
+
nbSteps: getValidNumber(scene.nbSteps, 1, 50, 10), // use 10 by default to go fast, but not too sloppy
|
23 |
+
}
|
24 |
+
|
25 |
try {
|
26 |
+
url = await generateVideo(scene.prompt, params)
|
|
|
|
|
|
|
|
|
27 |
// console.log("successfull generation")
|
28 |
error = ""
|
29 |
if (!url?.length) {
|
30 |
throw new Error(`url for the generated image is empty`)
|
31 |
}
|
32 |
} catch (err) {
|
33 |
+
console.error(`failed to render the scene.. but let's try again!`)
|
34 |
+
|
35 |
+
try {
|
36 |
+
url = await generateVideo(scene.prompt, params)
|
37 |
+
// console.log("successfull generation")
|
38 |
+
error = ""
|
39 |
|
40 |
+
if (!url?.length) {
|
41 |
+
throw new Error(`url for the generated image is empty`)
|
42 |
+
}
|
43 |
+
|
44 |
+
} catch (err) {
|
45 |
+
console.error(`it failed the second time ${err}`)
|
46 |
+
error = `failed to render scene: ${err}`
|
47 |
+
}
|
48 |
+
}
|
49 |
|
50 |
|
51 |
// TODO add segmentation here
|
src/utils/segmentImage.mts
CHANGED
@@ -9,6 +9,7 @@ import { resizeBase64Image } from "./resizeBase64Image.mts"
|
|
9 |
const instances: string[] = [
|
10 |
`${process.env.VC_SEGMENTATION_MODULE_SPACE_API_URL_1 || ""}`,
|
11 |
`${process.env.VC_SEGMENTATION_MODULE_SPACE_API_URL_2 || ""}`,
|
|
|
12 |
]
|
13 |
|
14 |
// TODO we should use an inference endpoint instead
|
@@ -32,7 +33,7 @@ export async function segmentImage(
|
|
32 |
|
33 |
const browser = await puppeteer.launch({
|
34 |
headless: true,
|
35 |
-
protocolTimeout:
|
36 |
})
|
37 |
|
38 |
const page = await browser.newPage()
|
@@ -53,13 +54,13 @@ export async function segmentImage(
|
|
53 |
// console.log('looking for the button to submit')
|
54 |
const submitButton = await page.$('button.lg')
|
55 |
|
56 |
-
await sleep(
|
57 |
|
58 |
// console.log('clicking on the button')
|
59 |
await submitButton.click()
|
60 |
|
61 |
await page.waitForSelector('img[data-testid="detailed-image"]', {
|
62 |
-
timeout:
|
63 |
})
|
64 |
|
65 |
const maskUrl = await page.$$eval('img[data-testid="detailed-image"]', el => el.map(x => x.getAttribute("src"))[0])
|
|
|
9 |
const instances: string[] = [
|
10 |
`${process.env.VC_SEGMENTATION_MODULE_SPACE_API_URL_1 || ""}`,
|
11 |
`${process.env.VC_SEGMENTATION_MODULE_SPACE_API_URL_2 || ""}`,
|
12 |
+
`${process.env.VC_SEGMENTATION_MODULE_SPACE_API_URL_3 || ""}`,
|
13 |
]
|
14 |
|
15 |
// TODO we should use an inference endpoint instead
|
|
|
33 |
|
34 |
const browser = await puppeteer.launch({
|
35 |
headless: true,
|
36 |
+
protocolTimeout: 50000,
|
37 |
})
|
38 |
|
39 |
const page = await browser.newPage()
|
|
|
54 |
// console.log('looking for the button to submit')
|
55 |
const submitButton = await page.$('button.lg')
|
56 |
|
57 |
+
await sleep(300)
|
58 |
|
59 |
// console.log('clicking on the button')
|
60 |
await submitButton.click()
|
61 |
|
62 |
await page.waitForSelector('img[data-testid="detailed-image"]', {
|
63 |
+
timeout: 50000, // need to be large enough in case someone else attemps to use our space
|
64 |
})
|
65 |
|
66 |
const maskUrl = await page.$$eval('img[data-testid="detailed-image"]', el => el.map(x => x.getAttribute("src"))[0])
|