import React , { useEffect , useState } from "react" ; import { makeStyles } from "@material-ui/core" ; import { DropDown , FileChooser , Toggle , VideoInputSelect } from "./components/components" ; import { VideoInputType } from "./const" ; import { useVideoInputList } from "./hooks/useVideoInputList" ; import { generateDefaultMODNetParams , generateMODNetDefaultConfig , MODNetWorkerManager , MODNetConfig , MODNetOperationParams , MODEL_INPUT_SIZES } from "@dannadori/modnet-worker-js" ; let GlobalLoopID = 0 ; const models = { webcam_192_16 : `/webcam_192_16/model.json` , webcam_256_16 : `/webcam_192_16/model.json` , webcam_512_16 : `/webcam_192_16/model.json` , } ; const processSize = { webcam_192_16 : [ 192 , 192 ] , webcam_256_16 : [ 256 , 256 ] , webcam_512_16 : [ 512 , 512 ] , } ; const useStyles = makeStyles ( ( ) => ( { inputView : { maxWidth : 512 , } , } ) ) ; interface WorkerProps { manager : MODNetWorkerManager ; config : MODNetConfig ; params : MODNetOperationParams ; count : number ; } interface InputMedia { mediaType : VideoInputType ; media : MediaStream | string ; } const App = ( ) => { const classes = useStyles ( ) ; const { videoInputList } = useVideoInputList ( ) ; const [ workerProps , setWorkerProps ] = useState < WorkerProps > ( ) ; const [ modelKey , setModelKey ] = useState ( Object . keys ( models ) [ 0 ] ) ; const [ onLocal , setOnLocal ] = useState ( false ) ; const [ strict , setStrict ] = useState ( false ) ; const [ inputMedia , setInputMedia ] = useState < InputMedia > ( { mediaType : "IMAGE" , media : "yuka_kawamura.png" , } ) ; const inputChange = ( mediaType , input ) => { setInputMedia ( { mediaType : mediaType , media : input } ) ; } ; const backgroundChange = ( mediaType , input ) => { console . log ( "background:" , mediaType , input ) ; if ( mediaType === "IMAGE" ) { const img = document . getElementById ( "background" ) as HTMLImageElement ; img . src = input ; } } ; useEffect ( ( ) => { const init = async ( ) => { const m = workerProps ? workerProps . manager : new MODNetWorkerManager ( ) ; const count = workerProps ? workerProps . count + 1 : 0 ; const c = generateMODNetDefaultConfig ( ) ; c . processOnLocal = onLocal ; c . modelInputSize = processSize [ modelKey ] [ 0 ] as MODEL_INPUT_SIZES ; console . log ( "NEW MODE LOAD1" ) ; await m . init ( c ) ; console . log ( "NEW MODE LOAD2" ) ; const p = generateDefaultMODNetParams ( ) ; p . processWidth = processSize [ modelKey ] [ 0 ] ; p . processHeight = processSize [ modelKey ] [ 1 ] ; const newProps = { manager : m , config : c , params : p , count : count } ; setWorkerProps ( newProps ) ; } ; init ( ) ; } , [ modelKey , onLocal ] ) ; useEffect ( ( ) => { } , [ ] ) ; useEffect ( ( ) => { const video = document . getElementById ( "input_video" ) as HTMLVideoElement ; if ( inputMedia . mediaType === "IMAGE" ) { const img = document . getElementById ( "input_img" ) as HTMLImageElement ; img . onloadeddata = ( ) => { resizeDst ( img ) ; } ; img . src = inputMedia . media as string ; } else if ( inputMedia . mediaType === "MOVIE" ) { const vid = document . getElementById ( "input_video" ) as HTMLVideoElement ; vid . pause ( ) ; vid . srcObject = null ; vid . src = inputMedia . media as string ; vid . loop = true ; vid . onloadeddata = ( ) => { video . play ( ) ; resizeDst ( vid ) ; } ; } else { const vid = document . getElementById ( "input_video" ) as HTMLVideoElement ; vid . pause ( ) ; vid . srcObject = inputMedia . media as MediaStream ; vid . onloadeddata = ( ) => { video . play ( ) ; resizeDst ( vid ) ; } ; } } , [ inputMedia ] ) ; useEffect ( ( ) => { const input = document . getElementById ( "input_img" ) || document . getElementById ( "input_video" ) ; resizeDst ( input ! ) ; } ) ; const resizeDst = ( input ) => { const cs = getComputedStyle ( input ) ; const width = parseInt ( cs . getPropertyValue ( "width" ) ) ; const height = parseInt ( cs . getPropertyValue ( "height" ) ) ; const dst = document . getElementById ( "output" ) as HTMLCanvasElement ; const tmp = document . getElementById ( "tmp" ) as HTMLCanvasElement ; const front = document . getElementById ( "front" ) as HTMLCanvasElement ; const srcCache = document . getElementById ( "src-cache" ) as HTMLCanvasElement ; [ dst , tmp , front , srcCache ] . forEach ( ( c ) => { c . width = width ; c . height = height ; } ) ; } ; useEffect ( ( ) => { console . log ( "[Pipeline] Start" , workerProps ) ; let renderRequestId ; const LOOP_ID = performance . now ( ) ; GlobalLoopID = LOOP_ID ; const render = async ( ) => { const start = performance . now ( ) ; if ( workerProps ) { const src = ( document . getElementById ( "input_img" ) as HTMLImageElement ) || ( document . getElementById ( "input_video" ) as HTMLVideoElement ) ; const background = document . getElementById ( "background" ) as HTMLImageElement ; const dst = document . getElementById ( "output" ) as HTMLCanvasElement ; const tmp = document . getElementById ( "tmp" ) as HTMLCanvasElement ; const front = document . getElementById ( "front" ) as HTMLCanvasElement ; const srcCache = document . getElementById ( "src-cache" ) as HTMLCanvasElement ; const inference_start = performance . now ( ) ; srcCache . getContext ( "2d" ) ! . drawImage ( src , 0 , 0 , srcCache . width , srcCache . height ) ; const prediction = await workerProps . manager . predict ( srcCache ! , workerProps . params ) ; const inference_end = performance . now ( ) ; const info1 = document . getElementById ( "info" ) as HTMLCanvasElement ; info1 . innerText = `processing time: ${ inference_end - inference_start }` ; const res = new ImageData ( workerProps . params . processWidth , workerProps . params . processHeight ) ; try { for ( let i = 0 ; i < workerProps . params . processHeight ; i ++ ) { for ( let j = 0 ; j < workerProps . params . processWidth ; j ++ ) { const offset = i * workerProps . params . processWidth + j ; res . data [ offset * 4 + 0 ] = 0 ; res . data [ offset * 4 + 1 ] = 0 ; res . data [ offset * 4 + 2 ] = 0 ; res . data [ offset * 4 + 3 ] = prediction ! [ i ] [ j ] * 255 ; } } } catch ( exception ) { console . log ( "exp:" , exception ) ; } tmp . width = workerProps . params . processWidth ; tmp . height = workerProps . params . processHeight ; tmp . getContext ( "2d" ) ! . clearRect ( 0 , 0 , tmp . width , tmp . height ) ; tmp . getContext ( "2d" ) ! . putImageData ( res , 0 , 0 ) ; dst . getContext ( "2d" ) ! . clearRect ( 0 , 0 , dst . width , dst . height ) ; dst . getContext ( "2d" ) ! . drawImage ( background , 0 , 0 , dst . width , dst . height ) ; front . getContext ( "2d" ) ! . clearRect ( 0 , 0 , front . width , front . height ) ; front . getContext ( "2d" ) ! . drawImage ( tmp , 0 , 0 , front . width , front . height ) ; front . getContext ( "2d" ) ! . globalCompositeOperation = "source-atop" ; front . getContext ( "2d" ) ! . drawImage ( srcCache , 0 , 0 , front . width , front . height ) ; front . getContext ( "2d" ) ! . globalCompositeOperation = "source-over" ; dst . getContext ( "2d" ) ! . drawImage ( front , 0 , 0 , dst . width , dst . height ) ; if ( GlobalLoopID === LOOP_ID ) { renderRequestId = requestAnimationFrame ( render ) ; } } const end = performance . now ( ) ; const info2 = document . getElementById ( "info2" ) as HTMLCanvasElement ; info2 . innerText = `processing time: ${ end - start }` ; } ; render ( ) ; return ( ) => { cancelAnimationFrame ( renderRequestId ) ; } ; } , [ workerProps , strict ] ) ; return ( < div > < div style = { { display : "flex" } } > < div style = { { display : "flex" } } > { inputMedia . mediaType === "IMAGE" ? < img className = { classes . inputView } alt = "input_img" id = "input_img" > < /img> : <video className={classes.inputView} id="input_video"></video > } < canvas className = { classes . inputView } id = "output" > < /canvas> < /div> < div > < VideoInputSelect title = "input" current = { "" } onchange = { inputChange } options = { videoInputList } / > < DropDown title = "model" current = { modelKey } onchange = { setModelKey } options = { models } / > < Toggle title = "onLocal" current = { onLocal } onchange = { setOnLocal } / > < Toggle title = "Strict" current = { strict } onchange = { setStrict } / > < FileChooser title = "background" onchange = { backgroundChange } / > < div > < a href = "https://github.com/w-okada/image-analyze-workers" > github repository < /a> < /div> < /div> < /div> < div style = { { display : "flex" } } > < canvas className = { classes . inputView } id = "tmp" hidden > < /canvas> < canvas className = { classes . inputView } id = "front" hidden > < /canvas> < canvas className = { classes . inputView } id = "src-cache" hidden > < /canvas> < img className = { classes . inputView } id = "background" alt = "background" src = "north-star-2869817_640.jpg" hidden > < /img> < /div> < div > < div id = "info" > < /div> < div id = "info2" > < /div> < /div> < /div>); } ; export default App ;