jbilcke-hf HF staff commited on
Commit
6f310ff
·
1 Parent(s): 75aad1e

trying some optimizations

Browse files
src/production/renderStaticScene.mts CHANGED
@@ -4,7 +4,6 @@ import { v4 as uuidv4 } from "uuid"
4
  import tmpDir from "temp-dir"
5
 
6
  import { ImageSegment, RenderedScene, RenderRequest } from "../types.mts"
7
- import { downloadFileToTmp } from "../utils/downloadFileToTmp.mts"
8
  import { segmentImage } from "../utils/segmentImage.mts"
9
  import { generateImageSDXLAsBase64 } from "../utils/generateImageSDXL.mts"
10
  import { writeBase64ToFile } from "../utils/writeBase64ToFile.mts"
@@ -58,7 +57,7 @@ export async function renderStaticScene(scene: RenderRequest): Promise<RenderedS
58
 
59
  const tmpImageFilePath = path.join(tmpDir, `${uuidv4()}.png`)
60
 
61
- console.log("beginning:", imageBase64.slice(0, 100))
62
  await writeBase64ToFile(imageBase64, tmpImageFilePath)
63
  console.log("wrote the image to ", tmpImageFilePath)
64
 
@@ -67,10 +66,30 @@ export async function renderStaticScene(scene: RenderRequest): Promise<RenderedS
67
  error = "failed to segment the image"
68
  } else {
69
  console.log("got the first frame! segmenting..")
70
- const result = await segmentImage(tmpImageFilePath, actionnables, width, height)
71
- mask = result.pngInBase64
72
- segments = result.segments
73
- console.log("success!", { segments })
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
74
  }
75
  } else {
76
  console.log("no actionnables: just returning the image, then")
 
4
  import tmpDir from "temp-dir"
5
 
6
  import { ImageSegment, RenderedScene, RenderRequest } from "../types.mts"
 
7
  import { segmentImage } from "../utils/segmentImage.mts"
8
  import { generateImageSDXLAsBase64 } from "../utils/generateImageSDXL.mts"
9
  import { writeBase64ToFile } from "../utils/writeBase64ToFile.mts"
 
57
 
58
  const tmpImageFilePath = path.join(tmpDir, `${uuidv4()}.png`)
59
 
60
+ // console.log("beginning:", imageBase64.slice(0, 100))
61
  await writeBase64ToFile(imageBase64, tmpImageFilePath)
62
  console.log("wrote the image to ", tmpImageFilePath)
63
 
 
66
  error = "failed to segment the image"
67
  } else {
68
  console.log("got the first frame! segmenting..")
69
+ try {
70
+ const result = await segmentImage(tmpImageFilePath, actionnables, width, height)
71
+ mask = result.pngInBase64
72
+ segments = result.segments
73
+ console.log(`it worked the first time! got ${segments.length} segments`)
74
+ } catch (err) {
75
+ console.log("this takes too long :/ trying another server..")
76
+ try {
77
+ const result = await segmentImage(tmpImageFilePath, actionnables, width, height)
78
+ mask = result.pngInBase64
79
+ segments = result.segments
80
+ console.log(`it worked the second time! got ${segments.length} segments`)
81
+ } catch (err) {
82
+ console.log("trying one last time, on a 3rd server..")
83
+ try {
84
+ const result = await segmentImage(tmpImageFilePath, actionnables, width, height)
85
+ mask = result.pngInBase64
86
+ segments = result.segments
87
+ console.log(`it worked the third time! got ${segments.length} segments`)
88
+ } catch (err) {
89
+ console.log("yeah, all servers are busy it seems.. aborting")
90
+ }
91
+ }
92
+ }
93
  }
94
  } else {
95
  console.log("no actionnables: just returning the image, then")
src/production/renderVideoScene.mts CHANGED
@@ -16,21 +16,36 @@ export async function renderVideoScene(scene: RenderRequest): Promise<RenderedSc
16
  const width = 576
17
  const height = 320
18
 
 
 
 
 
 
 
19
  try {
20
- url = await generateVideo(scene.prompt, {
21
- seed: getValidNumber(scene.seed, 0, 2147483647, generateSeed()),
22
- nbFrames: getValidNumber(scene.nbFrames, 8, 24, 16), // 2 seconds by default
23
- nbSteps: getValidNumber(scene.nbSteps, 1, 50, 10), // use 10 by default to go fast, but not too sloppy
24
- })
25
  // console.log("successfull generation")
26
  error = ""
27
  if (!url?.length) {
28
  throw new Error(`url for the generated image is empty`)
29
  }
30
  } catch (err) {
31
- error = `failed to render scene: ${err}`
32
- }
 
 
 
 
33
 
 
 
 
 
 
 
 
 
 
34
 
35
 
36
  // TODO add segmentation here
 
16
  const width = 576
17
  const height = 320
18
 
19
+ const params = {
20
+ seed: getValidNumber(scene.seed, 0, 2147483647, generateSeed()),
21
+ nbFrames: getValidNumber(scene.nbFrames, 8, 24, 16), // 2 seconds by default
22
+ nbSteps: getValidNumber(scene.nbSteps, 1, 50, 10), // use 10 by default to go fast, but not too sloppy
23
+ }
24
+
25
  try {
26
+ url = await generateVideo(scene.prompt, params)
 
 
 
 
27
  // console.log("successfull generation")
28
  error = ""
29
  if (!url?.length) {
30
  throw new Error(`url for the generated image is empty`)
31
  }
32
  } catch (err) {
33
+ console.error(`failed to render the scene.. but let's try again!`)
34
+
35
+ try {
36
+ url = await generateVideo(scene.prompt, params)
37
+ // console.log("successfull generation")
38
+ error = ""
39
 
40
+ if (!url?.length) {
41
+ throw new Error(`url for the generated image is empty`)
42
+ }
43
+
44
+ } catch (err) {
45
+ console.error(`it failed the second time ${err}`)
46
+ error = `failed to render scene: ${err}`
47
+ }
48
+ }
49
 
50
 
51
  // TODO add segmentation here
src/utils/segmentImage.mts CHANGED
@@ -9,6 +9,7 @@ import { resizeBase64Image } from "./resizeBase64Image.mts"
9
  const instances: string[] = [
10
  `${process.env.VC_SEGMENTATION_MODULE_SPACE_API_URL_1 || ""}`,
11
  `${process.env.VC_SEGMENTATION_MODULE_SPACE_API_URL_2 || ""}`,
 
12
  ]
13
 
14
  // TODO we should use an inference endpoint instead
@@ -32,7 +33,7 @@ export async function segmentImage(
32
 
33
  const browser = await puppeteer.launch({
34
  headless: true,
35
- protocolTimeout: 120000,
36
  })
37
 
38
  const page = await browser.newPage()
@@ -53,13 +54,13 @@ export async function segmentImage(
53
  // console.log('looking for the button to submit')
54
  const submitButton = await page.$('button.lg')
55
 
56
- await sleep(200)
57
 
58
  // console.log('clicking on the button')
59
  await submitButton.click()
60
 
61
  await page.waitForSelector('img[data-testid="detailed-image"]', {
62
- timeout: 120000, // need to be large enough in case someone else attemps to use our space
63
  })
64
 
65
  const maskUrl = await page.$$eval('img[data-testid="detailed-image"]', el => el.map(x => x.getAttribute("src"))[0])
 
9
  const instances: string[] = [
10
  `${process.env.VC_SEGMENTATION_MODULE_SPACE_API_URL_1 || ""}`,
11
  `${process.env.VC_SEGMENTATION_MODULE_SPACE_API_URL_2 || ""}`,
12
+ `${process.env.VC_SEGMENTATION_MODULE_SPACE_API_URL_3 || ""}`,
13
  ]
14
 
15
  // TODO we should use an inference endpoint instead
 
33
 
34
  const browser = await puppeteer.launch({
35
  headless: true,
36
+ protocolTimeout: 50000,
37
  })
38
 
39
  const page = await browser.newPage()
 
54
  // console.log('looking for the button to submit')
55
  const submitButton = await page.$('button.lg')
56
 
57
+ await sleep(300)
58
 
59
  // console.log('clicking on the button')
60
  await submitButton.click()
61
 
62
  await page.waitForSelector('img[data-testid="detailed-image"]', {
63
+ timeout: 50000, // need to be large enough in case someone else attemps to use our space
64
  })
65
 
66
  const maskUrl = await page.$$eval('img[data-testid="detailed-image"]', el => el.map(x => x.getAttribute("src"))[0])