/** * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import { FunctionDeclaration, SchemaType } from "@google/generative-ai"; import { useEffect, useRef, useState, memo } from "react"; import vegaEmbed from "vega-embed"; import { useLiveAPIContext } from "../../contexts/LiveAPIContext"; import { ToolCall } from "../../multimodal-live-types"; function P5SketchComponent() { const containerRef = useRef(null); const [jsonString, setJSONString] = useState(""); const { client, setConfig } = useLiveAPIContext(); useEffect(() => { const w = window as any; if (containerRef.current && w.initSketch) { w.initSketch(containerRef.current); } }, [containerRef]); useEffect(() => { setConfig({ model: "models/gemini-2.0-flash-exp", generationConfig: { responseModalities: "text", //responseModalities: "audio", /* speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: "Kobe" } }, }, */ }, systemInstruction: { parts: [ { text: `We have circles on a canvas that we can move and resize. Every time that I ask you something, I want you to use the "get_circles" function to understand where the circles are. Once you receive the response for "get_circles" I want you to use "change_circle" to modify one or multiple circle properties to accomplish our mutual goal. Every time I ask you ask you to do something you should call "get_circles" then "change_circle" after evaluating the response. The circles blend additively using screen mode, so if I ask you mix colors, you can make the color by moving circles to the same position to blend their colors. ` }, ], }, tools: [ // there is a free-tier quota for search { googleSearch: {} }, { functionDeclarations: [ { name: "get_circles", description: "Get a list of circles in my application", }, { name: "change_circle", description: "Change one of the circles", parameters: { type: SchemaType.OBJECT, properties: { color: { type: SchemaType.STRING, }, x: { type: SchemaType.NUMBER, }, y: { type: SchemaType.NUMBER, }, radius: { type: SchemaType.NUMBER, }, }, required: ["color", "x", "y", "radius"], }, }, ], }, ], }); }, [setConfig]); useEffect(() => { const onToolCall = (toolCall: ToolCall) => { console.log(`got toolcall`, toolCall); toolCall.functionCalls.forEach((fc) => { const w = window as any; const func: Function = w[fc.name]; if (fc.name === "get_circles") { setTimeout( () => client.sendToolResponse({ functionResponses: [ { response: w.get_circles(), id: fc.id, }, ], }), 500, ); } else if (func && typeof func === "function") { func(fc.args); client.sendToolResponse({ functionResponses: [ { response: { content: { ...w.get_circles() }, }, id: fc.id, }, ], }); } else { console.error(`Unhandled function call for ${fc.name}`); } }); }; client.on("toolcall", onToolCall); return () => { client.off("toolcall", onToolCall); }; }, [client]); const embedRef = useRef(null); useEffect(() => { if (embedRef.current && jsonString) { vegaEmbed(embedRef.current, JSON.parse(jsonString)); } }, [embedRef, jsonString]); return (
); } export const P5Sketch = memo(P5SketchComponent);