3 dots game

.dockerignore

@@ -0,0 +1,11 @@
+node_modules
+npm-debug.log
+build
+.git
+.gitignore
+README.md
+.env
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local 

.gcloudignore

@@ -0,0 +1,5 @@
+# Ignore everything except app.yaml and the build directory
+*
+!app.yaml
+!build
+!build/**

.gitattributes

@@ -0,0 +1 @@
+*.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,30 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+
+# testing
+/coverage
+
+# production
+/build
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Environment variables
+.env
+.env.*
+
+# Server
+/server/dist

CONTRIBUTING.md

@@ -0,0 +1,33 @@
+# How to contribute
+
+We'd love to accept your patches and contributions to this project.
+
+## Before you begin
+
+### Sign our Contributor License Agreement
+
+Contributions to this project must be accompanied by a
+[Contributor License Agreement](https://cla.developers.google.com/about) (CLA).
+You (or your employer) retain the copyright to your contribution; this simply
+gives us permission to use and redistribute your contributions as part of the
+project.
+
+If you or your current employer have already signed the Google CLA (even if it
+was for a different project), you probably don't need to do it again.
+
+Visit <https://cla.developers.google.com/> to see your current agreements or to
+sign a new one.
+
+### Review our community guidelines
+
+This project follows
+[Google's Open Source Community Guidelines](https://opensource.google/conduct/).
+
+## Contribution process
+
+### Code reviews
+
+All submissions, including submissions by project members, require review. We
+use GitHub pull requests for this purpose. Consult
+[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
+information on using pull requests.

Dockerfile

@@ -0,0 +1,26 @@
+# Use the official Node.js image with a more recent version
+FROM node:20
+
+# Create and change to the app directory.
+WORKDIR /usr/src/app
+
+# Copy application dependency manifests to the container image.
+COPY package*.json ./
+
+# Install all dependencies (including devDependencies for build)
+RUN npm install
+
+# Copy application code and environment variables
+COPY . .
+
+# Build the app
+RUN npm run build
+
+# Expose the port the app runs on
+EXPOSE 3001
+
+# Set environment variable for production
+ENV NODE_ENV=production
+
+# Run the server
+CMD [ "npm", "run", "start-server" ] 

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,146 @@
+---
+title: Gemini Live API Demo - 3 Dots
+emoji: 🟢
+colorFrom: blue
+colorTo: red
+sdk: docker
+pinned: false
+app_port: 3001
+---
+
+# Multimodal Live API - Web console
+
+This repository contains a react-based starter app for using the [Multimodal Live API](<[https://ai.google.dev/gemini-api](https://ai.google.dev/api/multimodal-live)>) over a websocket. It provides modules for streaming audio playback, recording user media such as from a microphone, webcam or screen capture as well as a unified log view to aid in development of your application.
+
+[![Multimodal Live API Demo](readme/thumbnail.png)](https://www.youtube.com/watch?v=J_q7JY1XxFE)
+
+Watch the demo of the Multimodal Live API [here](https://www.youtube.com/watch?v=J_q7JY1XxFE).
+
+## Usage
+
+To get started, [create a free Gemini API key](https://aistudio.google.com/apikey) and add it to the `.env` file. Then:
+
+```
+$ npm install
+```
+
+This demo is also modified to run on HuggingFace in a Docker container. So you must run the server and the client at the same time. In 2 separate terminals, run:
+
+```
+$ npm run start
+```
+
+```
+$ npm run start-server
+```
+
+We have provided several example applications on other branches of this repository:
+
+- [demos/GenExplainer](https://github.com/google-gemini/multimodal-live-api-web-console/tree/demos/genexplainer)
+- [demos/GenWeather](https://github.com/google-gemini/multimodal-live-api-web-console/tree/demos/genweather)
+- [demos/GenList](https://github.com/google-gemini/multimodal-live-api-web-console/tree/demos/genlist)
+
+## Example
+
+Below is an example of an entire application that will use Google Search grounding and then render graphs using [vega-embed](https://github.com/vega/vega-embed):
+
+```typescript
+import { type FunctionDeclaration, SchemaType } from "@google/generative-ai";
+import { useEffect, useRef, useState, memo } from "react";
+import vegaEmbed from "vega-embed";
+import { useLiveAPIContext } from "../../contexts/LiveAPIContext";
+
+export const declaration: FunctionDeclaration = {
+  name: "render_altair",
+  description: "Displays an altair graph in json format.",
+  parameters: {
+    type: SchemaType.OBJECT,
+    properties: {
+      json_graph: {
+        type: SchemaType.STRING,
+        description:
+          "JSON STRING representation of the graph to render. Must be a string, not a json object",
+      },
+    },
+    required: ["json_graph"],
+  },
+};
+
+export function Altair() {
+  const [jsonString, setJSONString] = useState<string>("");
+  const { client, setConfig } = useLiveAPIContext();
+
+  useEffect(() => {
+    setConfig({
+      model: "models/gemini-2.0-flash-exp",
+      systemInstruction: {
+        parts: [
+          {
+            text: 'You are my helpful assistant. Any time I ask you for a graph call the "render_altair" function I have provided you. Dont ask for additional information just make your best judgement.',
+          },
+        ],
+      },
+      tools: [{ googleSearch: {} }, { functionDeclarations: [declaration] }],
+    });
+  }, [setConfig]);
+
+  useEffect(() => {
+    const onToolCall = (toolCall: ToolCall) => {
+      console.log(`got toolcall`, toolCall);
+      const fc = toolCall.functionCalls.find(
+        (fc) => fc.name === declaration.name
+      );
+      if (fc) {
+        const str = (fc.args as any).json_graph;
+        setJSONString(str);
+      }
+    };
+    client.on("toolcall", onToolCall);
+    return () => {
+      client.off("toolcall", onToolCall);
+    };
+  }, [client]);
+
+  const embedRef = useRef<HTMLDivElement>(null);
+
+  useEffect(() => {
+    if (embedRef.current && jsonString) {
+      vegaEmbed(embedRef.current, JSON.parse(jsonString));
+    }
+  }, [embedRef, jsonString]);
+  return <div className="vega-embed" ref={embedRef} />;
+}
+```
+
+## development
+
+This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).
+Project consists of:
+
+- an Event-emitting websocket-client to ease communication between the websocket and the front-end
+- communication layer for processing audio in and out
+- a boilerplate view for starting to build your apps and view logs
+
+## Available Scripts
+
+In the project directory, you can run:
+
+### `npm start`
+
+Runs the app in the development mode.\
+Open [http://localhost:3000](http://localhost:3000) to view it in the browser.
+
+The page will reload if you make edits.\
+You will also see any lint errors in the console.
+
+### `npm run build`
+
+Builds the app for production to the `build` folder.\
+It correctly bundles React in production mode and optimizes the build for the best performance.
+
+The build is minified and the filenames include the hashes.\
+Your app is ready to be deployed!
+
+See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information.
+
+_This is an experiment showcasing the Multimodal Live API, not an official Google product. We'll do our best to support and maintain this experiment but your mileage may vary. We encourage open sourcing projects as a way of learning from each other. Please respect our and other creators' rights, including copyright and trademark rights when present, when sharing these works and creating derivative work. If you want more info on Google's policy, you can find that [here](https://developers.google.com/terms/site-policies)._

app.yaml

@@ -0,0 +1,29 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+runtime: nodejs20
+env: standard
+
+handlers:
+  # serve static files
+  - url: /(.*\..+)$
+    static_files: build/\1
+    upload: build/(.*\..+)$
+
+  # Catch all handler to index.html
+  - url: /.*
+    static_files: build/index.html
+    secure: always
+    redirect_http_response_code: 301
+    upload: buid/index.html

docker-compose.yml

@@ -0,0 +1,9 @@
+version: '3.8'
+
+services:
+  app:
+    build: .
+    ports:
+      - "3001:3001"
+    environment:
+      - GEMINI_API_KEY=${GEMINI_API_KEY} 

package.json

@@ -0,0 +1,74 @@
+{
+  "name": "multimodal-live-api-web-console",
+  "version": "0.1.0",
+  "dependencies": {
+    "classnames": "^2.5.1",
+    "dotenv": "^16.4.1",
+    "dotenv-flow": "^4.1.0",
+    "eventemitter3": "^5.0.1",
+    "express": "^4.18.2",
+    "http-proxy-middleware": "^3.0.3",
+    "lodash": "^4.17.21",
+    "react": "^18.3.1",
+    "react-dom": "^18.3.1",
+    "react-icons": "^5.3.0",
+    "react-scripts": "5.0.1",
+    "react-select": "^5.8.3",
+    "react-syntax-highlighter": "^15.6.1",
+    "sass": "^1.80.6",
+    "vega": "^5.30.0",
+    "vega-embed": "^6.29.0",
+    "vega-lite": "^5.22.0",
+    "web-vitals": "^2.1.4",
+    "websocket": "^1.0.34",
+    "ws": "^8.16.0",
+    "zustand": "^5.0.1"
+  },
+  "scripts": {
+    "start-https": "HTTPS=true react-scripts start",
+    "start": "react-scripts start",
+    "build": "react-scripts build",
+    "test": "react-scripts test",
+    "eject": "react-scripts eject",
+    "startone": "PORT=3001 react-scripts start",
+    "start-server": "node server/index.js",
+    "start-network": "./scripts/start-network.sh"
+  },
+  "eslintConfig": {
+    "extends": [
+      "react-app",
+      "react-app/jest"
+    ]
+  },
+  "browserslist": {
+    "production": [
+      ">0.2%",
+      "not dead",
+      "not op_mini all"
+    ],
+    "development": [
+      "last 1 chrome version",
+      "last 1 firefox version",
+      "last 1 safari version"
+    ]
+  },
+  "devDependencies": {
+    "@google/generative-ai": "^0.21.0",
+    "@testing-library/jest-dom": "^5.17.0",
+    "@testing-library/react": "^13.4.0",
+    "@testing-library/user-event": "^13.5.0",
+    "@types/express": "^4.17.21",
+    "@types/jest": "^27.5.2",
+    "@types/lodash": "^4.17.13",
+    "@types/node": "^16.18.119",
+    "@types/react": "^18.3.12",
+    "@types/react-dom": "^18.3.1",
+    "@types/react-syntax-highlighter": "^15.5.13",
+    "@types/ws": "^8.5.10",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.6.3"
+  },
+  "overrides": {
+    "typescript": "^5.6.3"
+  }
+}

public/index.html

@@ -0,0 +1,526 @@
+<!doctype html>
+<html lang="en">
+
+<head>
+	<meta charset="utf-8" />
+	<link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
+	<meta name="viewport" content="width=device-width, initial-scale=1" />
+	<meta name="theme-color" content="#000000" />
+	<meta name="description" content="Web site created using create-react-app" />
+	<link rel="preconnect" href="https://fonts.googleapis.com" />
+	<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
+	<link href="https://fonts.googleapis.com/css2?family=Space+Mono:ital,wght@0,400;0,700;1,400;1,700&display=swap"
+		rel="stylesheet" />
+	<link
+		href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,[email protected],100..700,0..1,-50..200&display=block"
+		rel="stylesheet" />
+	<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.10.2/p5.js"></script>
+	<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.10.2/addons/p5.sound.min.js"></script>
+	<script src="https://cdn.jsdelivr.net/gh/molleindustria/p5.play/lib/p5.play.js"></script>
+	<link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />
+	<!--
+      Notice the use of %PUBLIC_URL% in the tags above.
+      It will be replaced with the URL of the `public` folder during the build.
+      Only files inside the `public` folder can be referenced from the HTML.
+
+      Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
+      work correctly both with client-side routing and a non-root public URL.
+      Learn how to configure a non-root public URL by running `npm run build`.
+    -->
+	<title>Multimodal Live - Console</title>
+</head>
+
+<body>
+	<noscript>You need to enable JavaScript to run this app.</noscript>
+	<div id="root"></div>
+	<script>
+		let mySketch;
+		let osc; // oscillator for the tone
+		let freq = 261; // base frequency of the tone
+		let selectedCircleIndex = -1;
+		let currentLevel = 1;
+		let targetCircles = [];
+		let matchThreshold = 20; // threshold for position and size matching
+		let levelComplete = false;
+		let score = 0;
+		let moveCount = 0; // track number of moves in current level
+		let totalMoves = 0; // track total moves across all levels
+		let matchedCircles = []; // track which circles are matched
+		let matchAnimationTime = []; // track animation time for each circle
+		let gameComplete = false; // track if the entire game is complete
+		let gameOver = false;
+		let timeLeft = 30;
+		let timerStarted = false;
+		let timerInterval = null;
+		const FINAL_LEVEL = 5; // number of levels in the game
+		const VIBRANT_COLORS = [
+			'#FF0000', // Red
+			'#00FF00', // Lime
+			'#0000FF', // Blue
+			'#FF00FF', // Magenta
+			'#00FFFF', // Cyan
+			'#FFD700', // Gold
+			'#FF4500', // OrangeRed
+			'#32CD32', // LimeGreen
+			'#8A2BE2', // BlueViolet
+			'#FF1493'  // DeepPink
+		];
+		
+		// Function to get n unique random colors from the VIBRANT_COLORS array
+		function getRandomColors(n) {
+			const shuffled = [...VIBRANT_COLORS];
+			for (let i = shuffled.length - 1; i > 0; i--) {
+				const j = Math.floor(Math.random() * (i + 1));
+				[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
+			}
+			return shuffled.slice(0, n);
+		}
+
+		function calculateLevelScore() {
+			// Base score for completing level
+			const baseScore = 1000;
+			// Penalty for number of moves (encourage efficiency)
+			const movePenalty = Math.min(moveCount * 10, 500); // cap penalty at 500
+			return Math.max(baseScore - movePenalty, 100); // minimum 100 points
+		}
+
+		function isCircleMatched(circle, target) {
+			const maxDistance = target.radius * 0.20; // 5% of target radius for position tolerance
+			const maxSizeDiff = target.radius * 0.20; // 5% of target radius for size tolerance
+			
+			const distX = Math.abs(circle.x - target.x);
+			const distY = Math.abs(circle.y - target.y);
+			const distRadius = Math.abs(circle.radius - target.radius);
+			
+			console.log(`Checking circle ${circle.color}:`);
+			console.log(`  Position: (${circle.x.toFixed(2)}, ${circle.y.toFixed(2)}) vs target (${target.x.toFixed(2)}, ${target.y.toFixed(2)})`);
+			console.log(`  Distance: X=${distX.toFixed(2)}, Y=${distY.toFixed(2)} (max allowed: ${maxDistance.toFixed(2)})`);
+			console.log(`  Radius: ${circle.radius.toFixed(2)} vs ${target.radius.toFixed(2)} (diff: ${distRadius.toFixed(2)}, max allowed: ${maxSizeDiff.toFixed(2)})`);
+			
+			const isMatched = distX <= maxDistance && 
+							 distY <= maxDistance && 
+							 distRadius <= maxSizeDiff;
+							 
+			console.log(`  MATCH: ${isMatched}`);
+			return isMatched;
+		}
+
+		function startTimer() {
+			if (!timerStarted) {
+				timerStarted = true;
+				timeLeft = 30;
+				timerInterval = setInterval(() => {
+					timeLeft--;
+					if (timeLeft <= 0) {
+						clearInterval(timerInterval);
+						gameOver = true;
+					}
+				}, 1000);
+			}
+		}
+
+		function resetGame() {
+			currentLevel = 1;
+			score = 0;
+			moveCount = 0;
+			totalMoves = 0;
+			gameComplete = false;
+			gameOver = false;
+			timerStarted = false;
+			timeLeft = 30;
+			if (timerInterval) {
+				clearInterval(timerInterval);
+			}
+			matchedCircles = new Array(currentLevel).fill(false);
+			matchAnimationTime = new Array(currentLevel).fill(0);
+			generateTargets();
+		}
+
+		function checkMatch() {
+			console.log("\n=== Checking Matches ===");
+			// Check each circle individually
+			let allMatched = true;
+			let anyNewMatch = false;
+			
+			for(let i = 0; i < window.circles.length; i++) {
+				const wasMatched = matchedCircles[i];
+				const newMatch = isCircleMatched(window.circles[i], targetCircles[i]);
+				
+				if (newMatch !== matchedCircles[i]) {
+					console.log(`Circle ${window.circles[i].color} match status changed: ${wasMatched} -> ${newMatch}`);
+				}
+				
+				matchedCircles[i] = newMatch;
+				
+				if (matchedCircles[i] && !wasMatched) {
+					// New match - play a tone and start animation
+					anyNewMatch = true;
+					matchAnimationTime[i] = 0;
+					if (osc) {
+						osc.freq(440 + i * 100); // Different tone for each circle
+						osc.start();
+						osc.amp(0.2);
+						osc.fade(0, 0.3);
+					}
+				}
+				
+				if (!matchedCircles[i]) {
+					allMatched = false;
+				}
+			}
+			
+			console.log("Current match status:", matchedCircles);
+			console.log("All matched:", allMatched);
+			
+			if(allMatched && !levelComplete) {
+				console.log("*** LEVEL COMPLETE! ***");
+				levelComplete = true;
+				currentLevel++;
+				score += calculateLevelScore();
+				totalMoves += moveCount;
+
+				// Reset timer for next level
+				timerStarted = false;
+				if (timerInterval) {
+					clearInterval(timerInterval);
+				}
+				timeLeft = 30;
+
+				// Play success sound
+				if (osc) {
+					osc.freq(880); // Higher note for level complete
+					osc.start();
+					osc.amp(0.3);
+					osc.fade(0, 0.5);
+				}
+
+				if (currentLevel > FINAL_LEVEL) {
+					gameComplete = true;
+					// Play victory sound
+					if (osc) {
+						osc.freq(1320); // Even higher note for game complete
+						osc.start();
+						osc.amp(0.4);
+						osc.fade(0, 1.0);
+					}
+				} else {
+					setTimeout(() => {
+						levelComplete = false;
+						generateTargets();
+						matchedCircles = new Array(currentLevel).fill(false);
+						matchAnimationTime = new Array(currentLevel).fill(0);
+					}, 2000); // Changed to 2 seconds
+				}
+			}
+		}
+
+		function generateTargets() {
+			// Reset move counter for new level
+			moveCount = 0;
+			matchedCircles = new Array(currentLevel).fill(false);
+			matchAnimationTime = new Array(currentLevel).fill(0);
+			
+			// Calculate the middle 75% of the screen
+			const margin = {
+				x: window.innerWidth * 0.125,  // 12.5% margin on each side
+				y: window.innerHeight * 0.125  // 12.5% margin on each side
+			};
+			const playArea = {
+				width: window.innerWidth * 0.75,   // 75% of screen width
+				height: window.innerHeight * 0.75   // 75% of screen height
+			};
+			
+			// Get random colors for this level
+			const levelColors = getRandomColors(currentLevel);
+			
+			// Generate new random positions for target circles
+			targetCircles = [];
+			window.circles = [];
+			
+			// Add circles based on current level (one more circle per level)
+			for (let i = 0; i < currentLevel; i++) {
+				const color = levelColors[i];
+				const target = {
+					color: color,
+					x: margin.x + Math.random() * playArea.width,
+					y: margin.y + Math.random() * playArea.height,
+					radius: Math.random() * 50 + 75
+				};
+				targetCircles.push(target);
+				
+				// Create corresponding movable circle, positioned in a line in the middle 75% of screen
+				const startX = margin.x + (playArea.width * (i + 1) / (currentLevel + 1));
+				window.circles.push({
+					color: color,
+					x: startX,
+					y: window.innerHeight/2,
+					radius: 100
+				});
+			}
+			
+			// Update current circles copy
+			window.circlesCurrent = window.circles.map((c) => ({ ...c }));
+		}
+
+		window.get_circles = function () {
+			return {
+				circles: window.circles,
+				targets: targetCircles,
+				level: currentLevel,
+				isComplete: levelComplete
+			};
+		};
+
+		window.change_circle = function (args) {
+			if (!timerStarted) {
+				startTimer();
+			}
+			moveCount++; // increment move counter
+			// Play the tone here
+			if (osc) {
+				osc.start();
+				osc.freq(freq);
+				osc.amp(0.3);
+				osc.fade(0, 0.2);
+			}
+
+			window.circlesCurrent = window.circles.map((c) => ({ ...c }))
+
+			const color = args.color;
+			const findIndex = window.circles.findIndex(
+				(c) => c.color.toLowerCase() === color.toLowerCase(),
+			);
+			window.circles.splice(findIndex, 1, args);
+
+			checkMatch();
+		};
+		
+		window.circles = [
+			{color: "#00FF00", x: window.innerWidth/2, y: window.innerHeight/2, radius: 100},
+		];
+
+		// Generate initial target positions
+		generateTargets();
+
+		// make a copy of it
+		window.circlesCurrent = window.circles.map((c) => ({ ...c }))
+
+		window.initSketch = function (container) {
+			console.log("initialize sketch in public/index.html");
+
+			console.log(container);
+			if (mySketch) {
+				return;
+			}
+			mySketch = new p5((p) => {
+				p.setup = function () {
+					console.log(p);
+					
+					p.createCanvas(window.innerWidth, window.innerHeight);
+					container.innerHTMl = "";
+					container.appendChild(p._renderer.canvas);
+
+					// Create the oscillator here, after p5.sound has been initialized
+					osc = new p5.Oscillator('sine');
+					osc.amp(0); // Start with zero amplitude
+				};
+
+				function getSelectedCircleIndex() {
+					for (let i = 0; i < circles.length; i++) {
+						const circle = circles[i];
+						if (
+							p.mouseX > circle.x - circle.radius &&
+							p.mouseX < circle.x + circle.radius &&
+							p.mouseY > circle.y - circle.radius &&
+							p.mouseY < circle.y + circle.radius
+						) {
+							console.log("clicked : " + i);
+							return i;
+						}
+					}
+					return -1;
+				}
+
+				p.mousePressed = function () {
+					selectedCircleIndex = getSelectedCircleIndex();
+				};
+
+				p.mouseDragged = function () {
+					if (selectedCircleIndex > -1) {
+						moveCount++;
+						const circle = circles[selectedCircleIndex];
+						circle.x = p.mouseX;
+						circle.y = p.mouseY;
+						checkMatch();
+					}
+				};
+
+				p.mouseReleased = function () {
+					selectedCircleIndex = -1;
+				};
+
+				p.draw = function draw() {
+					p.background(0);
+					p.blendMode(p.BLEND);
+					
+					if (gameComplete || gameOver) {
+						// Draw modal overlay
+						p.background(0, 200);
+						
+						// Create centered content box
+						const boxWidth = 500;
+						const boxHeight = 400;
+						const boxX = p.width/2 - boxWidth/2;
+						const boxY = p.height/2 - boxHeight/2;
+						
+						// Draw box background
+						p.fill(20);
+						p.stroke(255);
+						p.strokeWeight(2);
+						p.rect(boxX, boxY, boxWidth, boxHeight, 20);
+						
+						// Draw content
+						p.fill(255);
+						p.noStroke();
+						p.textFont('Space Mono');
+						p.textAlign(p.CENTER, p.CENTER);
+						
+						// Title
+						p.textSize(48);
+						p.text(gameOver ? 'Time\'s Up!' : 'Game Complete!', p.width/2, boxY + 80);
+						
+						if (!gameOver) {
+							// Game complete stats
+							p.textSize(24);
+							p.fill(200);
+							p.text(`Final Score: ${score}`, p.width/2, boxY + 160);
+							p.text(`Total Moves: ${totalMoves}`, p.width/2, boxY + 200);
+							p.text(`Average Moves Per Level: ${(totalMoves/FINAL_LEVEL).toFixed(1)}`, p.width/2, boxY + 240);
+							
+							// Message
+							p.textSize(18);
+							p.fill(150);
+							const efficiency = totalMoves < FINAL_LEVEL * 10 ? "Amazing efficiency!" :
+											totalMoves < FINAL_LEVEL * 20 ? "Great job!" :
+											"Well done!";
+							p.text(efficiency, p.width/2, boxY + 280);
+						} else {
+							// Game over message
+							p.textSize(24);
+							p.fill(200);
+							p.text(`Total Levels Completed: ${currentLevel - 1}`, p.width/2, boxY + 160);
+							p.text(`Total Moves Made: ${totalMoves + moveCount}`, p.width/2, boxY + 200);
+							p.text(`Circles Matched: ${matchedCircles.filter(m => m).length}/${currentLevel}`, p.width/2, boxY + 240);
+							
+							p.textSize(18);
+							p.fill(150);
+							p.text('Keep trying! You can do it!', p.width/2, boxY + 280);
+						}
+						
+						// Draw try again/play again button
+						const buttonWidth = 200;
+						const buttonHeight = 50;
+						const buttonX = p.width/2 - buttonWidth/2;
+						const buttonY = boxY + boxHeight - 80;
+						
+						// Check if mouse is over button
+						const mouseOverButton = p.mouseX > buttonX && p.mouseX < buttonX + buttonWidth &&
+											 p.mouseY > buttonY && p.mouseY < buttonY + buttonHeight;
+						
+						// Draw button
+						p.fill(mouseOverButton ? 40 : 30);
+						p.stroke(255);
+						p.strokeWeight(2);
+						p.rect(buttonX, buttonY, buttonWidth, buttonHeight, 10);
+						
+						// Button text
+						p.fill(255);
+						p.noStroke();
+						p.textSize(24);
+						p.text(gameOver ? 'Try Again' : 'Play Again', p.width/2, buttonY + buttonHeight/2);
+						
+						// Add click handler for button
+						if (mouseOverButton && p.mouseIsPressed) {
+							resetGame();
+						}
+						
+						return; // Don't draw the rest of the game
+					}
+					
+					// Draw target circles first (semi-transparent)
+					for(let i = 0; i < targetCircles.length; i++) {
+						const target = targetCircles[i];
+						p.noStroke();
+						p.blendMode(p.SCREEN);
+						const targetColor = p.color(target.color);
+						targetColor.setAlpha(50);
+						p.fill(targetColor);
+						p.circle(target.x, target.y, target.radius * 2);
+					}
+					
+					// Update animation times
+					for(let i = 0; i < matchAnimationTime.length; i++) {
+						if (matchedCircles[i]) {
+							matchAnimationTime[i] = (matchAnimationTime[i] + 0.1) % (Math.PI * 2);
+						}
+					}
+					
+					// Draw the movable circles
+					for(let i = 0; i < window.circles.length; i++) {
+						const circle = window.circles[i];
+						p.noStroke();
+						p.blendMode(p.SCREEN);
+						p.fill(circle.color);
+						
+						// ease towards the final position
+						const easing = 0.15;
+						circlesCurrent[i].x = circlesCurrent[i].x + (circle.x - circlesCurrent[i].x) * easing;
+						circlesCurrent[i].y = circlesCurrent[i].y + (circle.y - circlesCurrent[i].y) * easing;
+						circlesCurrent[i].radius = circlesCurrent[i].radius + (circle.radius - circlesCurrent[i].radius) * easing;
+						
+						// If circle is matched, add white border and pulse animation
+						if (matchedCircles[i]) {
+							const pulseAmount = Math.sin(matchAnimationTime[i]) * 5;
+							p.stroke(255);
+							p.strokeWeight(2);
+							p.circle(circlesCurrent[i].x, circlesCurrent[i].y, (circlesCurrent[i].radius * 2) + pulseAmount);
+							p.noStroke();
+							p.circle(circlesCurrent[i].x, circlesCurrent[i].y, circlesCurrent[i].radius * 2);
+						} else {
+							p.circle(circlesCurrent[i].x, circlesCurrent[i].y, circlesCurrent[i].radius * 2);
+						}
+					}
+
+					// Draw UI elements with BLEND mode
+					p.blendMode(p.BLEND);
+					
+					// Draw dark semi-transparent background for UI
+					p.noStroke();
+					p.fill(0, 180);
+					p.rect(0, 20, p.width, 100);
+					
+					// Draw level and timer
+					p.fill(255);
+					p.textFont('Space Mono');
+					p.textSize(24);
+					p.textAlign(p.CENTER, p.TOP);
+					p.text(`Level: ${window.circles.length} | Time Left: ${timeLeft}s`, p.width/2, 40);
+					
+					// Draw instructions
+					p.textSize(14);
+					p.fill(180);
+					p.text('Speak to Gemini to move the circle to their place', p.width/2, 75);
+					
+					if(levelComplete) {
+						p.textSize(32);
+						p.textAlign(p.CENTER, p.CENTER);
+						p.fill(255);
+						p.text('Level Complete!', p.width/2, 170);
+					}
+				};
+			});
+		};
+	</script>
+</body>
+
+</html>

public/robots.txt

@@ -0,0 +1,3 @@
+# https://www.robotstxt.org/robotstxt.html
+User-agent: *
+Disallow:

scripts/start-network.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Get the IP address
+IP=$(ipconfig getifaddr en0)
+if [ -z "$IP" ]; then
+    # Try wifi interface if en0 doesn't work
+    IP=$(ipconfig getifaddr en1)
+fi
+
+# Function to display the message at the bottom
+display_ip() {
+    while true; do
+        # Move to bottom of screen
+        tput cup $(($(tput lines)-4)) 0
+        # Clear 4 lines
+        tput el; echo
+        tput el; echo -e "\033[33m🌐 Network Access URL for iOS:"
+        tput el; echo -e "https://$IP:3000"
+        tput el; echo -e "==========================================\033[0m"
+        sleep 1
+    done
+}
+
+# Start the display in background
+display_ip &
+DISPLAY_PID=$!
+
+# Cleanup on script exit
+trap 'kill $DISPLAY_PID; tput cnorm; clear' EXIT
+
+# Start the React app with HTTPS and network access
+HTTPS=true HOST=0.0.0.0 react-scripts start 

server/index.js

@@ -0,0 +1,117 @@
+const express = require('express');
+const path = require('node:path');
+const { WebSocketServer, WebSocket } = require('ws');
+const http = require('node:http');
+require('dotenv').config();
+
+const app = express();
+const server = http.createServer(app);
+const wss = new WebSocketServer({ server });
+
+// Serve static files from the React app build directory
+app.use(express.static(path.join(__dirname, '../build')));
+
+// Make API key available to the WebSocket server but not to the client
+const GEMINI_API_KEY = process.env.GEMINI_API_KEY;
+
+if (!GEMINI_API_KEY) {
+  console.error('GEMINI_API_KEY environment variable is not set!');
+  process.exit(1);
+}
+
+// Create a WebSocket connection to Gemini for each client
+const createGeminiWebSocket = (clientWs) => {
+  const geminiWs = new WebSocket(
+    `wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContent?key=${GEMINI_API_KEY}`
+  );
+
+  // Set up event handlers before connecting
+  geminiWs.on('open', () => {
+    console.log('Connected to Gemini API');
+    // If there's a pending setup message, send it now
+    if (geminiWs.pendingSetup) {
+      console.log('Sending pending setup:', geminiWs.pendingSetup);
+      geminiWs.send(JSON.stringify(geminiWs.pendingSetup));
+      geminiWs.pendingSetup = null;
+    }
+  });
+
+  geminiWs.on('message', (data) => {
+    try {
+      // Convert the message to a Blob before sending to client
+      const message = data.toString();
+      console.log('Received from Gemini:', message);
+      
+      // Create a Blob from the message
+      const blob = Buffer.from(message);
+      clientWs.send(blob, { binary: true });
+    } catch (error) {
+      console.error('Error handling Gemini message:', error);
+    }
+  });
+
+  geminiWs.on('error', (error) => {
+    console.error('Gemini WebSocket error:', error);
+  });
+
+  geminiWs.on('close', (code, reason) => {
+    console.log('Gemini WebSocket closed:', code, reason.toString());
+  });
+
+  return geminiWs;
+};
+
+wss.on('connection', (ws) => {
+  console.log('Client connected');
+  let geminiWs = null;
+
+  ws.on('message', async (message) => {
+    try {
+      const data = JSON.parse(message);
+      console.log('Received from client:', data);
+
+      // Initialize Gemini connection when receiving setup message
+      if (data.setup) {
+        console.log('Initializing Gemini connection with config:', data.setup);
+        geminiWs = createGeminiWebSocket(ws);
+        
+        // Store setup message to send once connection is established
+        if (geminiWs.readyState !== WebSocket.OPEN) {
+          geminiWs.pendingSetup = data;
+        } else {
+          geminiWs.send(JSON.stringify(data));
+        }
+        return;
+      }
+
+      // Forward message to Gemini if connection exists
+      if (geminiWs && geminiWs.readyState === WebSocket.OPEN) {
+        console.log('Forwarding to Gemini:', data);
+        geminiWs.send(JSON.stringify(data));
+      } else if (geminiWs) {
+        console.log('Waiting for Gemini connection to be ready...');
+      } else {
+        console.error('No Gemini connection established');
+      }
+    } catch (error) {
+      console.error('Error processing message:', error);
+    }
+  });
+
+  ws.on('close', () => {
+    console.log('Client disconnected');
+    if (geminiWs) {
+      geminiWs.close();
+    }
+  });
+});
+
+// Handle any remaining requests by returning the React app
+app.get('*', (req, res) => {
+  res.sendFile(path.join(__dirname, '../build', 'index.html'));
+});
+
+const PORT = process.env.PORT || 3001;
+server.listen(PORT, () => {
+  console.log(`Server is running on port ${PORT}`);
+}); 

src/App.scss

@@ -0,0 +1,182 @@
+:root {
+  --text: white;
+  --gray-200: #b4b8bb;
+  --gray-300: #80868b;
+  --gray-500: #5f6368;
+  --gray-600: #444444;
+  --gray-700: #202020;
+  --gray-800: #171717;
+  --gray-900: #111111;
+  --gray-1000: #0a0a0a;
+  --border-stroke: #444444;
+  --accent-blue: rgb(161, 228, 242);
+  --accent-blue-active-bg: #001233;
+  --accent-blue-active: #98beff;
+  --accent-blue-headers: #448dff;
+  --accent-green: rgb(168, 218, 181);
+
+  --midnight-blue: rgb(0, 18, 51);
+  --blue-30: #99beff;
+
+  --accent-red: #ff4600;
+
+  --background: var(--gray-900);
+  --color: var(--text);
+
+  scrollbar-color: var(--gray-600) var(--gray-900);
+  scrollbar-width: thin;
+
+  --font-family: "Space Mono", monospace;
+
+  /* */
+  --Neutral-00: #000;
+  --Neutral-5: #181a1b;
+  --Neutral-10: #1c1f21;
+  --Neutral-15: #232729;
+  --Neutral-20: #2a2f31;
+  --Neutral-30: #404547;
+  --Neutral-50: #707577;
+  --Neutral-60: #888d8f;
+  --Neutral-80: #c3c6c7;
+  --Neutral-90: #e1e2e3;
+
+  --Green-500: #0d9c53;
+  --Green-700: #025022;
+
+  --Blue-500: #1f94ff;
+  --Blue-800: #0f3557;
+
+  --Red-400: #ff9c7a;
+  --Red-500: #ff4600;
+  --Red-600: #e03c00;
+  --Red-700: #bd3000;
+}
+
+body {
+  font-family: "Space Mono", monospace;
+  background: var(--Neutral-30);
+}
+
+.material-symbols-outlined {
+  &.filled {
+    font-variation-settings:
+      "FILL" 1,
+      "wght" 400,
+      "GRAD" 0,
+      "opsz" 24;
+  }
+}
+
+.space-mono-regular {
+  font-family: "Space Mono", monospace;
+  font-weight: 400;
+  font-style: normal;
+}
+
+.space-mono-bold {
+  font-family: "Space Mono", monospace;
+  font-weight: 700;
+  font-style: normal;
+}
+
+.space-mono-regular-italic {
+  font-family: "Space Mono", monospace;
+  font-weight: 400;
+  font-style: italic;
+}
+
+.space-mono-bold-italic {
+  font-family: "Space Mono", monospace;
+  font-weight: 700;
+  font-style: italic;
+}
+
+.hidden {
+  display: none;
+}
+
+.flex {
+  display: flex;
+}
+
+.h-screen-full {
+  height: 100vh;
+}
+
+.w-screen-full {
+  width: 100vw;
+}
+
+.flex-col {
+  flex-direction: column;
+}
+
+@media (prefers-reduced-motion: no-preference) {}
+
+.streaming-console {
+  background: var(--Neutral-5);
+  color: var(--gray-300);
+  display: flex;
+  height: 100vh;
+  width: 100vw;
+
+  a,
+  a:visited,
+  a:active {
+    color: var(--gray-300);
+  }
+
+  .disabled {
+    pointer-events: none;
+
+    >* {
+      pointer-events: none;
+    }
+  }
+
+  main {
+    position: relative;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    justify-content: center;
+    flex-grow: 1;
+    gap: 1rem;
+    max-width: 100%;
+    overflow: hidden;
+  }
+
+  .main-app-area {
+    display: flex;
+    flex: 1;
+    align-items: center;
+    justify-content: center;
+  }
+
+  .function-call {
+    position: absolute;
+    top: 0;
+    width: 100%;
+    height: 50%;
+    overflow-y: auto;
+  }
+}
+
+/* video player */
+.stream {
+  flex-grow: 1;
+  max-width: 90%;
+  border-radius: 32px;
+  max-height: fit-content;
+
+  &.float {
+    position: absolute;
+    bottom: 34px;
+    right: 20px;
+    z-index: 10000;
+  }
+
+  &.minimal {
+    max-width: 320px;
+  }
+}

src/App.test.tsx

@@ -0,0 +1,25 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { render, screen } from '@testing-library/react';
+import App from './App';
+
+test('renders learn react link', () => {
+  render(<App />);
+  const linkElement = screen.getByText(/learn react/i);
+  expect(linkElement).toBeInTheDocument();
+});

src/App.tsx

@@ -0,0 +1,81 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { useRef, useState, useEffect } from "react";
+import "./App.scss";
+import { LiveAPIProvider } from "./contexts/LiveAPIContext";
+import SidePanel from "./components/side-panel/SidePanel";
+import { P5Sketch } from "./components/p5/P5Sketch";
+import ControlTray from "./components/control-tray/ControlTray";
+import { IOSModal } from "./components/ios-modal/IOSModal";
+import { isIOS } from "./lib/platform";
+import cn from "classnames";
+
+function App() {
+  // this video reference is used for displaying the active stream, whether that is the webcam or screen capture
+  // feel free to style as you see fit
+  const videoRef = useRef<HTMLVideoElement>(null);
+  // either the screen capture, the video or null, if null we hide it
+  const [videoStream, setVideoStream] = useState<MediaStream | null>(null);
+  const [showIOSModal, setShowIOSModal] = useState(false);
+
+  useEffect(() => {
+    // Only show the modal on iOS devices
+    if (isIOS()) {
+      setShowIOSModal(true);
+    }
+  }, []);
+
+  return (
+    <div className="App">
+      <LiveAPIProvider>
+        <div className="streaming-console">
+          <SidePanel />
+          <main>
+            <div className="main-app-area">
+              {/* APP goes here */}
+              <P5Sketch />
+            </div>
+
+            <video
+              style={{ background: "black" }}
+              className={cn("stream float minimal", {
+                hidden: !videoRef.current || !videoStream,
+              })}
+              ref={videoRef}
+              autoPlay
+              playsInline
+            />
+            <ControlTray
+              videoRef={videoRef}
+              supportsVideo={false}
+              onVideoStreamChange={setVideoStream}
+            >
+              {/* put your own buttons here */}
+            </ControlTray>
+          </main>
+        </div>
+      </LiveAPIProvider>
+
+      <IOSModal 
+        isOpen={showIOSModal} 
+        onClose={() => setShowIOSModal(false)} 
+      />
+    </div>
+  );
+}
+
+export default App;

src/components/audio-pulse/AudioPulse.tsx

@@ -0,0 +1,67 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import "./audio-pulse.scss";
+import React from "react";
+import { useEffect, useRef } from "react";
+import c from "classnames";
+
+const lineCount = 3;
+
+export type AudioPulseProps = {
+  active: boolean;
+  volume: number;
+  hover?: boolean;
+};
+
+export default function AudioPulse({ active, volume, hover }: AudioPulseProps) {
+  const lines = useRef<HTMLDivElement[]>([]);
+
+  useEffect(() => {
+    let timeout: number | null = null;
+    const update = () => {
+      lines.current.forEach(
+        (line, i) =>
+        (line.style.height = `${Math.min(
+          24,
+          4 + volume * (i === 1 ? 400 : 60),
+        )}px`),
+      );
+      timeout = window.setTimeout(update, 100);
+    };
+
+    update();
+
+    return () => clearTimeout((timeout as number)!);
+  }, [volume]);
+
+  return (
+    <div className={c("audioPulse", { 
+      active,
+      hover 
+    })}>
+      {Array(lineCount)
+        .fill(null)
+        .map((_, i) => (
+          <div
+            key={i}
+            ref={(el) => (lines.current[i] = el!)}
+            style={{ animationDelay: `${i * 133}ms` }}
+          />
+        ))}
+    </div>
+  );
+}

src/components/audio-pulse/audio-pulse.scss

@@ -0,0 +1,51 @@
+.audioPulse {
+  display: flex;
+  width: 24px;
+  justify-content: space-evenly;
+  align-items: center;
+  transition: all 0.5s;
+
+  & > div {
+    background-color: var(--Neutral-30);
+    border-radius: 1000px;
+    width: 4px;
+    min-height: 4px;
+    border-radius: 1000px;
+    transition: height 0.1s;
+  }
+
+  &.hover > div {
+    animation: hover 1.4s infinite alternate ease-in-out;
+  }
+
+  height: 4px;
+  transition: opacity 0.333s;
+
+  &.active {
+    opacity: 1;
+
+    & > div {
+      background-color: var(--Neutral-80);
+    }
+  }
+}
+
+@keyframes hover {
+  from {
+    transform: translateY(0);
+  }
+
+  to {
+    transform: translateY(-3.5px);
+  }
+}
+
+@keyframes pulse {
+  from {
+    scale: 1 1;
+  }
+
+  to {
+    scale: 1.2 1.2;
+  }
+}

src/components/control-tray/ControlTray.tsx

@@ -0,0 +1,260 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import cn from "classnames";
+
+import { memo, ReactNode, RefObject, useEffect, useRef, useState } from "react";
+import { useLiveAPIContext } from "../../contexts/LiveAPIContext";
+import { UseMediaStreamResult } from "../../hooks/use-media-stream-mux";
+import { useScreenCapture } from "../../hooks/use-screen-capture";
+import { useWebcam } from "../../hooks/use-webcam";
+import { AudioRecorder } from "../../lib/audio-recorder";
+import { audioContext } from "../../lib/utils";
+import { isIOS } from "../../lib/platform";
+import AudioPulse from "../audio-pulse/AudioPulse";
+import "./control-tray.scss";
+
+export type ControlTrayProps = {
+  videoRef: RefObject<HTMLVideoElement>;
+  children?: ReactNode;
+  supportsVideo: boolean;
+  onVideoStreamChange?: (stream: MediaStream | null) => void;
+};
+
+type MediaStreamButtonProps = {
+  isStreaming: boolean;
+  onIcon: string;
+  offIcon: string;
+  start: () => Promise<any>;
+  stop: () => any;
+};
+
+/**
+ * button used for triggering webcam or screen-capture
+ */
+const MediaStreamButton = memo(
+  ({ isStreaming, onIcon, offIcon, start, stop }: MediaStreamButtonProps) =>
+    isStreaming ? (
+      <button className="action-button" onClick={stop}>
+        <span className="material-symbols-outlined">{onIcon}</span>
+      </button>
+    ) : (
+      <button className="action-button" onClick={start}>
+        <span className="material-symbols-outlined">{offIcon}</span>
+      </button>
+    ),
+);
+
+function ControlTray({
+  videoRef,
+  children,
+  onVideoStreamChange = () => {},
+  supportsVideo,
+}: ControlTrayProps) {
+  const videoStreams = [useWebcam(), useScreenCapture()];
+  const [activeVideoStream, setActiveVideoStream] =
+    useState<MediaStream | null>(null);
+  const [webcam, screenCapture] = videoStreams;
+  const [inVolume, setInVolume] = useState(0);
+  const [audioRecorder] = useState(() => new AudioRecorder());
+  const [muted, setMuted] = useState(false);
+  const renderCanvasRef = useRef<HTMLCanvasElement>(null);
+  const connectButtonRef = useRef<HTMLButtonElement>(null);
+  const [simulatedVolume, setSimulatedVolume] = useState(0);
+  const isIOSDevice = isIOS();
+
+  const { client, connected, connect, disconnect, volume } =
+    useLiveAPIContext();
+
+  // Add iOS detection
+  const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
+
+  useEffect(() => {
+    if (!connected && connectButtonRef.current) {
+      connectButtonRef.current.focus();
+    }
+  }, [connected]);
+
+  // Add iOS volume simulation effect
+  useEffect(() => {
+    if (isIOSDevice && connected && !muted) {
+      const interval = setInterval(() => {
+        // Create a smooth pulsing effect
+        const pulse = (Math.sin(Date.now() / 500) + 1) / 2; // Values between 0 and 1
+        setSimulatedVolume(0.02 + pulse * 0.03); // Small range for subtle effect
+      }, 50);
+      
+      return () => clearInterval(interval);
+    }
+  }, [connected, muted, isIOSDevice]);
+
+  useEffect(() => {
+    document.documentElement.style.setProperty(
+      "--volume",
+      `${Math.max(5, Math.min((isIOSDevice ? simulatedVolume : inVolume) * 200, 8))}px`,
+    );
+  }, [inVolume, simulatedVolume, isIOSDevice]);
+
+  useEffect(() => {
+    const onData = (base64: string) => {
+      client.sendRealtimeInput([
+        {
+          mimeType: "audio/pcm;rate=16000",
+          data: base64,
+        },
+      ]);
+    };
+
+    if (connected && !muted && audioRecorder) {
+      audioRecorder.on("data", onData).on("volume", setInVolume).start();
+    } else {
+      audioRecorder.stop();
+    }
+
+    return () => {
+      audioRecorder.off("data", onData).off("volume", setInVolume);
+    };
+  }, [connected, client, muted, audioRecorder]);
+
+  useEffect(() => {
+    if (videoRef.current) {
+      videoRef.current.srcObject = activeVideoStream;
+    }
+
+    let timeoutId = -1;
+
+    function sendVideoFrame() {
+      const video = videoRef.current;
+      const canvas = renderCanvasRef.current;
+
+      if (!video || !canvas) {
+        return;
+      }
+
+      const ctx = canvas.getContext("2d")!;
+      canvas.width = video.videoWidth * 0.25;
+      canvas.height = video.videoHeight * 0.25;
+      if (canvas.width + canvas.height > 0) {
+        ctx.drawImage(videoRef.current, 0, 0, canvas.width, canvas.height);
+        const base64 = canvas.toDataURL("image/jpeg", 1.0);
+        const data = base64.slice(base64.indexOf(",") + 1, Infinity);
+        client.sendRealtimeInput([{ mimeType: "image/jpeg", data }]);
+      }
+      if (connected) {
+        timeoutId = window.setTimeout(sendVideoFrame, 1000 / 0.5);
+      }
+    }
+    if (connected && activeVideoStream !== null) {
+      requestAnimationFrame(sendVideoFrame);
+    }
+    return () => {
+      clearTimeout(timeoutId);
+    };
+  }, [connected, activeVideoStream, client, videoRef]);
+
+  //handler for swapping from one video-stream to the next
+  const changeStreams = (next?: UseMediaStreamResult) => async () => {
+    if (next) {
+      const mediaStream = await next.start();
+      setActiveVideoStream(mediaStream);
+      onVideoStreamChange(mediaStream);
+    } else {
+      setActiveVideoStream(null);
+      onVideoStreamChange(null);
+    }
+
+    videoStreams.filter((msr) => msr !== next).forEach((msr) => msr.stop());
+  };
+
+  return (
+    <section className="control-tray">
+      <canvas style={{ display: "none" }} ref={renderCanvasRef} />
+      <nav className={cn("actions-nav", { disabled: !connected })}>
+        <button
+          className={cn("action-button mic-button")}
+          onClick={() => setMuted(!muted)}
+        >
+          {!muted ? (
+            <span className="material-symbols-outlined filled">mic</span>
+          ) : (
+            <span className="material-symbols-outlined filled">mic_off</span>
+          )}
+        </button>
+
+        <div className="action-button no-action outlined">
+          <AudioPulse volume={volume} active={connected} hover={false} />
+        </div>
+
+        {supportsVideo && (
+          <>
+            <MediaStreamButton
+              isStreaming={screenCapture.isStreaming}
+              start={changeStreams(screenCapture)}
+              stop={changeStreams()}
+              onIcon="cancel_presentation"
+              offIcon="present_to_all"
+            />
+            <MediaStreamButton
+              isStreaming={webcam.isStreaming}
+              start={changeStreams(webcam)}
+              stop={changeStreams()}
+              onIcon="videocam_off"
+              offIcon="videocam"
+            />
+          </>
+        )}
+        {children}
+      </nav>
+
+      <div className={cn("connection-container", { connected })}>
+        <div className="connection-button-container">
+          <button
+            ref={connectButtonRef}
+            className={cn("action-button connect-toggle", { connected })}
+            onClick={async () => {
+              console.log('🔄 Connection button clicked');
+              try {
+                if (connected) {
+                  console.log('📴 Disconnecting...');
+                  await disconnect();
+                  console.log('✅ Disconnected successfully');
+                } else {
+                  console.log('🔌 Starting connection...');
+                  console.log('📱 Device info:', { isIOSDevice, isSafari });
+                  
+                  // We already have mic permissions from the modal, just connect
+                  console.log('📞 Calling connect()...');
+                  await connect();
+                  console.log('✅ Connected successfully');
+                }
+              } catch (err) {
+                console.error('❌ Failed to toggle connection:', err);
+                // Here you could add UI feedback about the error
+              }
+            }}
+          >
+            <span className="material-symbols-outlined filled">
+              {connected ? "pause" : "play_arrow"}
+            </span>
+          </button>
+        </div>
+        <span className="text-indicator">Streaming</span>
+      </div>
+    </section>
+  );
+}
+
+export default memo(ControlTray);

src/components/control-tray/control-tray.scss

@@ -0,0 +1,211 @@
+.action-button {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  background: var(--Neutral-20);
+  color: var(--Neutral-60);
+  font-size: 1.25rem;
+  line-height: 1.75rem;
+  text-transform: lowercase;
+  cursor: pointer;
+  animation: opacity-pulse 3s ease-in infinite;
+  transition: all 0.2s ease-in-out;
+  width: 48px;
+  height: 48px;
+  border-radius: 18px;
+  border: 1px solid rgba(0, 0, 0, 0);
+  user-select: none;
+  cursor: pointer;
+
+  &:focus {
+    border: 2px solid var(--Neutral-20);
+    outline: 2px solid var(--Neutral-80);
+  }
+
+  &.outlined {
+    background: var(--Neutral-2);
+    border: 1px solid var(--Neutral-20);
+  }
+
+  .no-action {
+    pointer-events: none;
+  }
+
+  &:hover {
+    background: rgba(0, 0, 0, 0);
+    border: 1px solid var(--Neutral-20);
+  }
+
+  &.connected {
+    background: var(--Blue-800);
+    color: var(--Blue-500);
+
+    &:hover {
+      border: 1px solid var(--Blue-500);
+    }
+  }
+}
+
+@property --volume {
+  syntax: "length";
+  inherit: false;
+  initial-value: 0px;
+}
+
+.disabled .mic-button,
+.mic-button.disabled {
+  &:before {
+    background: rgba(0, 0, 0, 0);
+  }
+}
+
+.mic-button {
+  position: relative;
+  background-color: var(--accent-red);
+  z-index: 1;
+  color: black;
+  transition: all 0.2s ease-in;
+
+  &:focus {
+    border: 2px solid var(--Neutral-20);
+    outline: 2px solid var(--Red-500);
+  }
+
+  &:hover {
+    background-color: var(--Red-400);
+  }
+
+  &:before {
+    position: absolute;
+    z-index: -1;
+    top: calc(var(--volume) * -1);
+    left: calc(var(--volume) * -1);
+    display: block;
+    content: "";
+    opacity: 0.35;
+    background-color: var(--Red-500);
+    width: calc(100% + var(--volume) * 2);
+    height: calc(100% + var(--volume) * 2);
+    border-radius: 24px;
+    transition: all 0.02s ease-in-out;
+  }
+}
+
+.connect-toggle {
+  &:focus {
+    border: 2px solid var(--Neutral-20);
+    outline: 2px solid var(--Neutral-80);
+  }
+
+  &:not(.connected) {
+    background-color: var(--Blue-500);
+    color: var(--Neutral-5);
+  }
+}
+
+.control-tray {
+  position: absolute;
+  bottom: 0;
+  left: 50%;
+  transform: translate(-50%, 0);
+  display: inline-flex;
+  justify-content: center;
+  align-items: flex-start;
+  gap: 8px;
+  padding-bottom: 18px;
+
+  .ios-error-message {
+    position: absolute;
+    bottom: 100%;
+    left: 50%;
+    transform: translateX(-50%);
+    margin-bottom: 10px;
+    padding: 8px 16px;
+    background-color: rgba(255, 0, 0, 0.1);
+    border: 1px solid var(--Red-500);
+    border-radius: 8px;
+    color: var(--Red-500);
+    font-size: 14px;
+    white-space: nowrap;
+    animation: fadeIn 0.3s ease-in-out;
+  }
+
+  .disabled .action-button,
+  .action-button.disabled {
+    background: rgba(0, 0, 0, 0);
+    border: 1px solid var(--Neutral-30, #404547);
+    color: var(--Neutral-30);
+  }
+
+  .connection-container {
+    display: flex;
+    flex-direction: column;
+    justify-content: center;
+    align-items: center;
+    gap: 4px;
+
+    .connection-button-container {
+      border-radius: 27px;
+      border: 1px solid var(--Neutral-30);
+      background: var(--Neutral-5);
+      padding: 10px;
+    }
+
+    .text-indicator {
+      font-size: 11px;
+      color: var(--Blue-500);
+      user-select: none;
+    }
+
+    &:not(.connected) {
+      .text-indicator {
+        opacity: 0;
+      }
+    }
+  }
+}
+
+.actions-nav {
+  background: var(--Neutral-5);
+  border: 1px solid var(--Neutral-30);
+  border-radius: 27px;
+  display: inline-flex;
+  gap: 12px;
+  align-items: center;
+  overflow: clip;
+  padding: 10px;
+
+  transition: all 0.6s ease-in;
+
+  &>* {
+    display: flex;
+    align-items: center;
+    flex-direction: column;
+    gap: 1rem;
+  }
+}
+
+@keyframes opacity-pulse {
+  0% {
+    opacity: 0.9;
+  }
+
+  50% {
+    opacity: 1;
+  }
+
+  100% {
+    opacity: 0.9;
+  }
+}
+
+@keyframes fadeIn {
+  from {
+    opacity: 0;
+    transform: translate(-50%, 10px);
+  }
+  to {
+    opacity: 1;
+    transform: translate(-50%, 0);
+  }
+}

src/components/ios-modal/IOSModal.scss

@@ -0,0 +1,57 @@
+.ios-modal-overlay {
+  position: fixed;
+  top: 0;
+  left: 0;
+  right: 0;
+  bottom: 0;
+  background-color: rgba(0, 0, 0, 0.5);
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  z-index: 1000;
+}
+
+.ios-modal {
+  background: white;
+  padding: 2rem;
+  border-radius: 8px;
+  max-width: 90%;
+  width: 400px;
+  text-align: center;
+  box-shadow: 0 2px 10px rgba(0, 0, 0, 0.1);
+
+  h2 {
+    margin-top: 0;
+    color: #333;
+  }
+
+  p {
+    margin: 1rem 0;
+    color: #666;
+    line-height: 1.5;
+
+    &.error-message {
+      color: var(--Red-500);
+      background-color: rgba(255, 0, 0, 0.1);
+      padding: 8px;
+      border-radius: 4px;
+      margin: 1rem -8px;
+      font-weight: 500;
+    }
+  }
+}
+
+.ios-modal-button {
+  background-color: #007AFF;
+  color: white;
+  border: none;
+  padding: 0.8rem 2rem;
+  border-radius: 6px;
+  font-size: 1rem;
+  cursor: pointer;
+  margin-top: 1rem;
+
+  &:hover {
+    background-color: #0056b3;
+  }
+} 

src/components/ios-modal/IOSModal.tsx

@@ -0,0 +1,51 @@
+import React, { useState } from 'react';
+import './IOSModal.scss';
+
+interface IOSModalProps {
+  isOpen: boolean;
+  onClose: () => void;
+}
+
+export const IOSModal: React.FC<IOSModalProps> = ({ isOpen, onClose }) => {
+  const [error, setError] = useState<string | null>(null);
+
+  const handleMicPermission = async () => {
+    try {
+      console.log('🎤 Requesting iOS microphone permission from modal...');
+      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      console.log('✅ iOS microphone permission granted from modal!');
+      // Stop the stream since we don't need it yet - we'll request it again when recording starts
+      stream.getTracks().forEach(track => track.stop());
+      setError(null);
+      onClose();
+    } catch (err) {
+      console.error('❌ iOS microphone permission denied from modal:', err);
+      setError(err instanceof Error ? err.message : 'Failed to access microphone');
+    }
+  };
+
+  if (!isOpen) return null;
+
+  return (
+    <div className="ios-modal-overlay">
+      <div className="ios-modal">
+        <h2>Microphone Access Required</h2>
+        <p>
+          To use this app on iOS, we need permission to access your microphone.
+          Please tap "Allow" when prompted.
+        </p>
+        <p>
+          If you've denied permission, you'll need to enable it in your device settings.
+        </p>
+        {error && (
+          <p className="error-message">
+            Error: {error}
+          </p>
+        )}
+        <button onClick={handleMicPermission} className="ios-modal-button">
+          {error ? 'Try Again' : 'Got it'}
+        </button>
+      </div>
+    </div>
+  );
+}; 

src/components/logger/Logger.tsx

@@ -0,0 +1,276 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import "./logger.scss";
+
+import { Part } from "@google/generative-ai";
+import cn from "classnames";
+import { ReactNode } from "react";
+import { useLoggerStore } from "../../lib/store-logger";
+import SyntaxHighlighter from "react-syntax-highlighter";
+import { vs2015 as dark } from "react-syntax-highlighter/dist/esm/styles/hljs";
+import {
+  ClientContentMessage,
+  isClientContentMessage,
+  isInterrupted,
+  isModelTurn,
+  isServerContentMessage,
+  isToolCallCancellationMessage,
+  isToolCallMessage,
+  isToolResponseMessage,
+  isTurnComplete,
+  ModelTurn,
+  ServerContentMessage,
+  StreamingLog,
+  ToolCallCancellationMessage,
+  ToolCallMessage,
+  ToolResponseMessage,
+} from "../../multimodal-live-types";
+
+const formatTime = (d: Date) => d.toLocaleTimeString().slice(0, -3);
+
+const LogEntry = ({
+  log,
+  MessageComponent,
+}: {
+  log: StreamingLog;
+  MessageComponent: ({
+    message,
+  }: {
+    message: StreamingLog["message"];
+  }) => ReactNode;
+}): JSX.Element => (
+  <li
+    className={cn(
+      `plain-log`,
+      `source-${log.type.slice(0, log.type.indexOf("."))}`,
+      {
+        receive: log.type.includes("receive"),
+        send: log.type.includes("send"),
+      },
+    )}
+  >
+    <span className="timestamp">{formatTime(log.date)}</span>
+    <span className="source">{log.type}</span>
+    <span className="message">
+      <MessageComponent message={log.message} />
+    </span>
+    {log.count && <span className="count">{log.count}</span>}
+  </li>
+);
+
+const PlainTextMessage = ({
+  message,
+}: {
+  message: StreamingLog["message"];
+}) => <span>{message as string}</span>;
+
+type Message = { message: StreamingLog["message"] };
+
+const AnyMessage = ({ message }: Message) => (
+  <pre>{JSON.stringify(message, null, "  ")}</pre>
+);
+
+function tryParseCodeExecutionResult(output: string) {
+  try {
+    const json = JSON.parse(output);
+    return JSON.stringify(json, null, "  ");
+  } catch (e) {
+    return output;
+  }
+}
+
+const RenderPart = ({ part }: { part: Part }) =>
+  part.text && part.text.length ? (
+    <p className="part part-text">{part.text}</p>
+  ) : part.executableCode ? (
+    <div className="part part-executableCode">
+      <h5>executableCode: {part.executableCode.language}</h5>
+      <SyntaxHighlighter
+        language={part.executableCode.language.toLowerCase()}
+        style={dark}
+      >
+        {part.executableCode.code}
+      </SyntaxHighlighter>
+    </div>
+  ) : part.codeExecutionResult ? (
+    <div className="part part-codeExecutionResult">
+      <h5>codeExecutionResult: {part.codeExecutionResult.outcome}</h5>
+      <SyntaxHighlighter language="json" style={dark}>
+        {tryParseCodeExecutionResult(part.codeExecutionResult.output)}
+      </SyntaxHighlighter>
+    </div>
+  ) : (
+    <div className="part part-inlinedata">
+      <h5>Inline Data: {part.inlineData?.mimeType}</h5>
+      <SyntaxHighlighter language="plaintext" style={dark}>
+        {part.inlineData?.data || ""}
+      </SyntaxHighlighter>
+    </div>
+  );
+
+const ClientContentLog = ({ message }: Message) => {
+  const { turns, turnComplete } = (message as ClientContentMessage)
+    .clientContent;
+  return (
+    <div className="rich-log client-content user">
+      <h4 className="roler-user">User</h4>
+      {turns.map((turn, i) => (
+        <div key={`message-turn-${i}`}>
+          {turn.parts
+            .filter((part) => !(part.text && part.text === "\n"))
+            .map((part, j) => (
+              <RenderPart part={part} key={`message-turh-${i}-part-${j}`} />
+            ))}
+        </div>
+      ))}
+      {!turnComplete ? <span>turnComplete: false</span> : ""}
+    </div>
+  );
+};
+
+const ToolCallLog = ({ message }: Message) => {
+  const { toolCall } = message as ToolCallMessage;
+  return (
+    <div className={cn("rich-log tool-call")}>
+      {toolCall.functionCalls.map((fc, i) => (
+        <div key={fc.id} className="part part-functioncall">
+          <h5>Function call: {fc.name}</h5>
+          <SyntaxHighlighter language="json" style={dark}>
+            {JSON.stringify(fc, null, "  ")}
+          </SyntaxHighlighter>
+        </div>
+      ))}
+    </div>
+  );
+};
+
+const ToolCallCancellationLog = ({ message }: Message): JSX.Element => (
+  <div className={cn("rich-log tool-call-cancellation")}>
+    <span>
+      {" "}
+      ids:{" "}
+      {(message as ToolCallCancellationMessage).toolCallCancellation.ids.map(
+        (id) => (
+          <span className="inline-code" key={`cancel-${id}`}>
+            "{id}"
+          </span>
+        ),
+      )}
+    </span>
+  </div>
+);
+
+const ToolResponseLog = ({ message }: Message): JSX.Element => (
+  <div className={cn("rich-log tool-response")}>
+    {(message as ToolResponseMessage).toolResponse.functionResponses.map(
+      (fc) => (
+        <div key={`tool-response-${fc.id}`} className="part">
+          <h5>Function Response: {fc.id}</h5>
+          <SyntaxHighlighter language="json" style={dark}>
+            {JSON.stringify(fc.response, null, "  ")}
+          </SyntaxHighlighter>
+        </div>
+      ),
+    )}
+  </div>
+);
+
+const ModelTurnLog = ({ message }: Message): JSX.Element => {
+  const serverContent = (message as ServerContentMessage).serverContent;
+  const { modelTurn } = serverContent as ModelTurn;
+  const { parts } = modelTurn;
+
+  return (
+    <div className="rich-log model-turn model">
+      <h4 className="role-model">Model</h4>
+      {parts
+        .filter((part) => !(part.text && part.text === "\n"))
+        .map((part, j) => (
+          <RenderPart part={part} key={`model-turn-part-${j}`} />
+        ))}
+    </div>
+  );
+};
+
+const CustomPlainTextLog = (msg: string) => () => (
+  <PlainTextMessage message={msg} />
+);
+
+export type LoggerFilterType = "conversations" | "tools" | "none";
+
+export type LoggerProps = {
+  filter: LoggerFilterType;
+};
+
+const filters: Record<LoggerFilterType, (log: StreamingLog) => boolean> = {
+  tools: (log: StreamingLog) =>
+    isToolCallMessage(log.message) ||
+    isToolResponseMessage(log.message) ||
+    isToolCallCancellationMessage(log.message),
+  conversations: (log: StreamingLog) =>
+    isClientContentMessage(log.message) || isServerContentMessage(log.message),
+  none: () => true,
+};
+
+const component = (log: StreamingLog) => {
+  if (typeof log.message === "string") {
+    return PlainTextMessage;
+  }
+  if (isClientContentMessage(log.message)) {
+    return ClientContentLog;
+  }
+  if (isToolCallMessage(log.message)) {
+    return ToolCallLog;
+  }
+  if (isToolCallCancellationMessage(log.message)) {
+    return ToolCallCancellationLog;
+  }
+  if (isToolResponseMessage(log.message)) {
+    return ToolResponseLog;
+  }
+  if (isServerContentMessage(log.message)) {
+    const { serverContent } = log.message;
+    if (isInterrupted(serverContent)) {
+      return CustomPlainTextLog("interrupted");
+    }
+    if (isTurnComplete(serverContent)) {
+      return CustomPlainTextLog("turnComplete");
+    }
+    if (isModelTurn(serverContent)) {
+      return ModelTurnLog;
+    }
+  }
+  return AnyMessage;
+};
+
+export default function Logger({ filter = "none" }: LoggerProps) {
+  const { logs } = useLoggerStore();
+
+  const filterFn = filters[filter];
+
+  return (
+    <div className="logger">
+      <ul className="logger-list">
+        {logs.filter(filterFn).map((log, key) => {
+          return (
+            <LogEntry MessageComponent={component(log)} log={log} key={key} />
+          );
+        })}
+      </ul>
+    </div>
+  );
+}

src/components/logger/logger.scss

@@ -0,0 +1,116 @@
+.logger {
+  color: var(--gray-300);
+  width: 100%;
+  max-width: 100%;
+  display: block;
+
+  .logger-list {
+    padding: 0 0px 0 25px;
+    overflow-x: hidden;
+    width: calc(100% - 45px);
+  }
+
+  .user h4 {
+    color: var(--Green-500);
+  }
+
+  .model h4 {
+    color: var(--Blue-500);
+  }
+
+  .rich-log {
+    display: flex;
+    justify-content: center;
+    gap: 4px;
+
+    pre {
+      overflow-x: auto;
+    }
+
+    display: block;
+
+    h4 {
+      font-size: 14px;
+      text-transform: uppercase;
+      padding: 8px 0;
+      margin: 0;
+    }
+
+    h5 {
+      margin: 0;
+      padding-bottom: 8px;
+      border-bottom: 1px solid var(--Neutral-20);
+    }
+
+    .part {
+      background: var(--Neutral-5);
+      padding: 14px;
+      margin-bottom: 4px;
+      color: var(--Neutral-90);
+      border-radius: 8px;
+    }
+  }
+
+  .plain-log {
+    &>* {
+      padding-right: 4px;
+    }
+  }
+
+  .inline-code:not(:last-child) {
+    font-style: italic;
+
+    &::after {
+      content: ", ";
+    }
+  }
+}
+
+.logger li {
+  display: block;
+  padding: 8px 0;
+  color: var(--Neutral-50, #707577);
+  font-family: "Space Mono";
+  font-size: 14px;
+  font-style: normal;
+  font-weight: 400;
+  line-height: normal;
+}
+
+.logger li .timestamp {
+  width: 70px;
+  flex-grow: 0;
+  flex-shrink: 0;
+  color: var(--Neutral-50);
+}
+
+.logger li .source {
+  flex-shrink: 0;
+  font-weight: bold;
+}
+
+.logger li.source-server,
+.logger li.receive {
+  color: var(--Blue-500);
+}
+
+.logger li.source-client,
+.logger li.send:not(.source-server) {
+  color: var(--Green-500);
+}
+
+.logger li .count {
+  background-color: var(--Neutral-5);
+  font-size: x-small;
+  padding: 0em 0.6em;
+  padding: 0.3em 0.5em;
+  line-height: 1em;
+  vertical-align: middle;
+  border-radius: 8px;
+  color: var(--Blue-500);
+}
+
+.logger li .message {
+  flex-grow: 1;
+  color: var(--Neutral-50);
+}

src/components/logger/mock-logs.ts

@@ -0,0 +1,151 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * this module is just mock data, intended to make it easier to develop and style the logger
+ */
+import type { StreamingLog } from "../../multimodal-live-types";
+
+const soundLogs = (n: number): StreamingLog[] =>
+  new Array(n).fill(0).map(
+    (): StreamingLog => ({
+      date: new Date(),
+      type: "server.audio",
+      message: "buffer (11250)",
+    }),
+  );
+//
+const realtimeLogs = (n: number): StreamingLog[] =>
+  new Array(n).fill(0).map(
+    (): StreamingLog => ({
+      date: new Date(),
+      type: "client.realtimeInput",
+      message: "audio",
+    }),
+  );
+
+export const mockLogs: StreamingLog[] = [
+  {
+    date: new Date(),
+    type: "client.open",
+    message: "connected to socket",
+  },
+  ...realtimeLogs(10),
+  ...soundLogs(10),
+  {
+    date: new Date(),
+    type: "receive.content",
+    message: {
+      serverContent: {
+        interrupted: true,
+      },
+    },
+  },
+  {
+    date: new Date(),
+    type: "receive.content",
+    message: {
+      serverContent: {
+        turnComplete: true,
+      },
+    },
+  },
+  //this one is just a string
+  // {
+  //   date: new Date(),
+  //   type: "server.send",
+  //   message: {
+  //     serverContent: {
+  //       turnComplete: true,
+  //     },
+  //   },
+  // },
+  ...realtimeLogs(10),
+  ...soundLogs(20),
+  {
+    date: new Date(),
+    type: "receive.content",
+    message: {
+      serverContent: {
+        modelTurn: {
+          parts: [{ text: "Hey its text" }, { text: "more" }],
+        },
+      },
+    },
+  },
+  {
+    date: new Date(),
+    type: "client.send",
+    message: {
+      clientContent: {
+        turns: [
+          {
+            role: "User",
+            parts: [
+              {
+                text: "How much wood could a woodchuck chuck if a woodchuck could chuck wood",
+              },
+            ],
+          },
+        ],
+        turnComplete: true,
+      },
+    },
+  },
+  {
+    date: new Date(),
+    type: "server.toolCall",
+    message: {
+      toolCall: {
+        functionCalls: [
+          {
+            id: "akadjlasdfla-askls",
+            name: "take_photo",
+            args: {},
+          },
+          {
+            id: "akldjsjskldsj-102",
+            name: "move_camera",
+            args: { x: 20, y: 4 },
+          },
+        ],
+      },
+    },
+  },
+  {
+    date: new Date(),
+    type: "server.toolCallCancellation",
+    message: {
+      toolCallCancellation: {
+        ids: ["akladfjadslfk", "adkafsdljfsdk"],
+      },
+    },
+  },
+  {
+    date: new Date(),
+    type: "client.toolResponse",
+    message: {
+      toolResponse: {
+        functionResponses: [
+          {
+            response: { success: true },
+            id: "akslaj-10102",
+          },
+        ],
+      },
+    },
+  },
+];

src/components/p5/P5Sketch.tsx

@@ -0,0 +1,156 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { FunctionDeclaration, SchemaType } from "@google/generative-ai";
+import { useEffect, useRef, useState, memo } from "react";
+import vegaEmbed from "vega-embed";
+import { useLiveAPIContext } from "../../contexts/LiveAPIContext";
+import { ToolCall } from "../../multimodal-live-types";
+
+function P5SketchComponent() {
+  const containerRef = useRef<HTMLDivElement>(null);
+  const [jsonString, setJSONString] = useState<string>("");
+  const { client, setConfig } = useLiveAPIContext();
+
+  useEffect(() => {
+    const w = window as any;
+    if (containerRef.current && w.initSketch) {
+      w.initSketch(containerRef.current);
+    }
+  }, [containerRef]);
+
+  useEffect(() => {
+    setConfig({
+      model: "models/gemini-2.0-flash-exp",
+      generationConfig: {
+        responseModalities: "text",
+        //responseModalities: "audio",
+        /*
+        speechConfig: {
+          voiceConfig: { prebuiltVoiceConfig: { voiceName: "Kobe" } },
+        },
+        */
+      },
+      systemInstruction: {
+        parts: [
+          {
+            text: `We have circles on a canvas that we can move and resize. Every time that I ask you something, I want you to use the "get_circles" function to understand where the circles are. Once you receive the response for "get_circles" I want you to use "change_circle" to modify one or multiple circle properties to accomplish our mutual goal. Every time I ask you ask you to do something you should call "get_circles" then "change_circle" after evaluating the response.
+            
+            The circles blend additively using screen mode, so if I ask you mix colors, you can make the color by moving circles to the same position to blend their colors.
+            `
+          },
+        ],
+      },
+      tools: [
+        // there is a free-tier quota for search
+        { googleSearch: {} },
+        {
+          functionDeclarations: [
+            {
+              name: "get_circles",
+              description: "Get a list of circles in my application",
+            },
+            {
+              name: "change_circle",
+              description: "Change one of the circles",
+              parameters: {
+                type: SchemaType.OBJECT,
+                properties: {
+                  color: {
+                    type: SchemaType.STRING,
+                  },
+                  x: {
+                    type: SchemaType.NUMBER,
+                  },
+                  y: {
+                    type: SchemaType.NUMBER,
+                  },
+                  radius: {
+                    type: SchemaType.NUMBER,
+                  },
+                },
+                required: ["color", "x", "y", "radius"],
+              },
+            },
+          ],
+        },
+      ],
+    });
+  }, [setConfig]);
+
+  useEffect(() => {
+    const onToolCall = (toolCall: ToolCall) => {
+      console.log(`got toolcall`, toolCall);
+      toolCall.functionCalls.forEach((fc) => {
+        const w = window as any;
+        const func: Function = w[fc.name];
+        if (fc.name === "get_circles") {
+          setTimeout(
+            () =>
+              client.sendToolResponse({
+                functionResponses: [
+                  {
+                    response: w.get_circles(),
+                    id: fc.id,
+                  },
+                ],
+              }),
+            500,
+          );
+        } else if (func && typeof func === "function") {
+          func(fc.args);
+          client.sendToolResponse({
+            functionResponses: [
+              {
+                response: {
+                  content: { ...w.get_circles() },
+                },
+                id: fc.id,
+              },
+            ],
+          });
+        } else {
+          console.error(`Unhandled function call for ${fc.name}`);
+        }
+      });
+    };
+    client.on("toolcall", onToolCall);
+    return () => {
+      client.off("toolcall", onToolCall);
+    };
+  }, [client]);
+
+  const embedRef = useRef<HTMLDivElement>(null);
+
+  useEffect(() => {
+    if (embedRef.current && jsonString) {
+      vegaEmbed(embedRef.current, JSON.parse(jsonString));
+    }
+  }, [embedRef, jsonString]);
+  return (
+    <div
+      ref={containerRef}
+      style={{
+        width: "100vw",
+        height: "100vh",
+        alignItems: "center",
+        justifyContent: "center",
+        display: "flex",
+      }}
+    />
+  );
+}
+
+export const P5Sketch = memo(P5SketchComponent);

src/components/side-panel/SidePanel.tsx

@@ -0,0 +1,185 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import cn from "classnames";
+import { useEffect, useRef, useState } from "react";
+import { RiSidebarFoldLine, RiSidebarUnfoldLine } from "react-icons/ri";
+import Select from "react-select";
+import { useLiveAPIContext } from "../../contexts/LiveAPIContext";
+import { useLoggerStore } from "../../lib/store-logger";
+import Logger from "../logger/Logger";
+import type { LoggerFilterType } from "../logger/Logger";
+import "./side-panel.scss";
+
+const filterOptions = [
+  { value: "conversations", label: "Conversations" },
+  { value: "tools", label: "Tool Use" },
+  { value: "none", label: "All" },
+];
+
+export default function SidePanel() {
+  const { connected, client } = useLiveAPIContext();
+  const [open, setOpen] = useState(window.innerWidth >= 768);
+  const [isMobile, setIsMobile] = useState(window.innerWidth < 768);
+  const loggerRef = useRef<HTMLDivElement>(null);
+  const loggerLastHeightRef = useRef<number>(-1);
+  const { log, logs } = useLoggerStore();
+
+  // Add effect to handle responsive behavior
+  useEffect(() => {
+    const handleResize = () => {
+      const mobileScreen = window.innerWidth < 768;
+      setIsMobile(mobileScreen);
+      setOpen(!mobileScreen);
+    };
+
+    // Initial check
+    handleResize();
+
+    // Add event listener for window resize
+    window.addEventListener('resize', handleResize);
+
+    // Cleanup
+    return () => window.removeEventListener('resize', handleResize);
+  }, []);
+
+  const [textInput, setTextInput] = useState("");
+  const [selectedOption, setSelectedOption] = useState<{
+    value: string;
+    label: string;
+  } | null>(null);
+  const inputRef = useRef<HTMLTextAreaElement>(null);
+
+  //scroll the log to the bottom when new logs come in
+  useEffect(() => {
+    const el = loggerRef.current;
+    if (el) {
+      const scrollHeight = el.scrollHeight;
+      if (scrollHeight !== loggerLastHeightRef.current) {
+        el.scrollTop = scrollHeight;
+        loggerLastHeightRef.current = scrollHeight;
+      }
+    }
+  }, []);
+
+  // listen for log events and store them
+  useEffect(() => {
+    client.on("log", log);
+    return () => {
+      client.off("log", log);
+    };
+  }, [client, log]);
+
+  const handleSubmit = () => {
+    client.send([{ text: textInput }]);
+
+    setTextInput("");
+    if (inputRef.current) {
+      inputRef.current.innerText = "";
+    }
+  };
+
+  return (
+    <div className={cn("side-panel", { 
+      open,
+      mobile: isMobile
+    })}>
+      <header className="top">
+        <h2>Console</h2>
+        {open ? (
+          <button type="button" className="opener" onClick={() => setOpen(false)}>
+            <RiSidebarFoldLine color="#b4b8bb" />
+          </button>
+        ) : (
+          <button type="button" className="opener" onClick={() => setOpen(true)}>
+            <RiSidebarUnfoldLine color="#b4b8bb" />
+          </button>
+        )}
+      </header>
+      <section className="indicators">
+        <Select
+          className="react-select"
+          classNamePrefix="react-select"
+          styles={{
+            control: (baseStyles) => ({
+              ...baseStyles,
+              background: "var(--Neutral-15)",
+              color: "var(--Neutral-90)",
+              minHeight: "33px",
+              maxHeight: "33px",
+              border: 0,
+            }),
+            option: (styles, { isFocused, isSelected }) => ({
+              ...styles,
+              backgroundColor: isFocused
+                ? "var(--Neutral-30)"
+                : isSelected
+                  ? "var(--Neutral-20)"
+                  : undefined,
+            }),
+          }}
+          defaultValue={selectedOption}
+          options={filterOptions}
+          onChange={(e) => {
+            setSelectedOption(e);
+          }}
+        />
+        <div className={cn("streaming-indicator", { connected })}>
+          {connected
+            ? `🔵${open ? " Streaming" : ""}`
+            : `⏸️${open ? " Paused" : ""}`}
+        </div>
+      </section>
+      <div className="side-panel-container" ref={loggerRef}>
+        <Logger
+          filter={(selectedOption?.value as LoggerFilterType) || "none"}
+        />
+      </div>
+      <div className={cn("input-container", { disabled: !connected })}>
+        <div className="input-content">
+          <textarea
+            className="input-area"
+            ref={inputRef}
+            onKeyDown={(e) => {
+              if (e.key === "Enter" && !e.shiftKey) {
+                e.preventDefault();
+                e.stopPropagation();
+                handleSubmit();
+              }
+            }}
+            onChange={(e) => setTextInput(e.target.value)}
+            value={textInput}
+          />
+          <span
+            className={cn("input-content-placeholder", {
+              hidden: textInput.length,
+            })}
+          >
+            Type&nbsp;something...
+          </span>
+
+          <button
+            type="button"
+            className="send-button material-symbols-outlined filled"
+            onClick={handleSubmit}
+          >
+            send
+          </button>
+        </div>
+      </div>
+    </div>
+  );
+}

src/components/side-panel/side-panel.scss

@@ -0,0 +1,309 @@
+.side-panel {
+  background: var(--Neutral-00);
+  width: 40px; /* when closed */
+  display: flex;
+  flex-direction: column;
+  height: 100vh;
+  transition: all 0.2s ease-in;
+  font-family: Arial, sans-serif;
+  border-right: 1px solid var(--gray-600);
+  color: var(--Neutral-90, #e1e2e3);
+  font-family: var(--font-family);
+  font-size: 13px;
+  font-style: normal;
+  font-weight: 400;
+  line-height: 160%;
+
+  .react-select {
+    background: var(--Neutral-20);
+    color: var(--Neutral-90);
+    width: 193px;
+    height: 30px;
+
+    .react-select__single-value {
+      color: var(--Neutral-90);
+    }
+
+    .react-select__menu {
+      background: var(--Neutral-20);
+      color: var(--Neutral-90);
+    }
+
+    .react-select__option:hover,
+    .react-select__option:focus,
+    .react-select_option:focus-within {
+      background: var(--Neutral-30);
+    }
+  }
+
+  .hidden {
+    display: none !important;
+  }
+
+  &.open {
+    width: 400px;
+    height: 100vh;
+
+    .top h2 {
+      left: 0%;
+      display: block;
+      opacity: 1;
+    }
+  }
+
+  .top {
+    display: flex;
+    width: calc(100% - 45px);
+    justify-content: space-between;
+    align-items: center;
+    padding: 12px 20px 12px 25px;
+    border-bottom: 1px solid var(--Neutral-20);
+
+    h2 {
+      position: relative;
+      color: var(--Neutral-90, #e1e2e3);
+      font-family: "Google Sans";
+      font-size: 21px;
+      font-style: normal;
+      font-weight: 500;
+      line-height: 16px; /* 100% */
+
+      opacity: 0;
+      display: none;
+      left: -100%;
+      transition:
+        opacity 0.2s ease-in,
+        left 0.2s ease-in,
+        display 0.2s ease-in;
+      transition-behavior: allow-discrete;
+
+      @starting-style {
+        left: 0%;
+        opacity: 1;
+      }
+    }
+  }
+
+  .opener {
+    height: 30px;
+    transition: transform 0.2s ease-in;
+  }
+
+  &:not(.open) {
+    .side-panel-container {
+      opacity: 0;
+      display: none;
+      transition: all 0.2s ease-in allow-discrete;
+      transition-delay: 0.1s;
+    }
+
+    .indicators .streaming-indicator {
+      width: 30px;
+      opacity: 0;
+    }
+
+    .opener {
+      transform: translate(-50%, 0);
+    }
+
+    .input-container {
+      opacity: 0;
+      display: none;
+      transition: all 0.2s ease-in allow-discrete;
+    }
+  }
+
+  .indicators {
+    display: flex;
+    padding: 24px 25px;
+    justify-content: flex-end;
+    gap: 21px;
+    .streaming-indicator {
+      user-select: none;
+      border-radius: 4px;
+      border: 1px solid var(--Neutral-20, #2a2f31);
+      background: var(--Neutral-10, #1c1f21);
+      display: flex;
+      width: 136px;
+      height: 30px;
+      padding-left: 4px;
+      justify-content: center;
+      align-items: center;
+      gap: 6px;
+      flex-shrink: 0;
+      text-align: center;
+      font-family: "Space Mono";
+      font-size: 14px;
+      font-style: normal;
+      font-weight: 400;
+      line-height: normal;
+      transition: width 0.2s ease-in;
+
+      &.connected {
+        color: var(--Blue-500, #0d9c53);
+      }
+    }
+  }
+
+  .side-panel-container {
+    align-self: flex-end;
+    width: 400px;
+    flex-grow: 1;
+    overflow-x: hidden;
+    overflow-y: auto;
+    /*scrollbar-gutter: stable both-edges;*/
+  }
+
+  .input-container {
+    height: 50px;
+    flex-grow: 0;
+    flex-shrink: 0;
+    border-top: 1px solid var(--Neutral-20);
+    padding: 14px 25px;
+    overflow: hidden;
+
+    .input-content {
+      position: relative;
+      background: var(--Neutral-10);
+      border: 1px solid var(--Neutral-15);
+      height: 22px;
+      border-radius: 10px;
+      padding: 11px 18px;
+
+      .send-button {
+        position: absolute;
+        top: 50%;
+        right: 0;
+        transform: translate(0, -50%);
+        background: none;
+        border: 0;
+        color: var(--Neutral-20);
+        cursor: pointer;
+        transition: color 0.1s ease-in;
+        z-index: 2;
+
+        &:hover {
+          color: var(--Neutral-60);
+        }
+      }
+
+      .input-area {
+        background: none;
+        color: var(--Neutral-90);
+        field-sizing: content;
+        position: absolute;
+        top: 0;
+        left: 0;
+        z-index: 2;
+        display: inline-block;
+        width: calc(100% - 72px);
+        max-height: 20px;
+        outline: none;
+        --webkit-box-flex: 1;
+        flex: 1;
+        word-break: break-word;
+        overflow: auto;
+        padding: 14px 18px;
+        border: 0;
+        resize: none;
+      }
+
+      .input-content-placeholder {
+        position: absolute;
+        left: 0;
+        top: 0;
+        display: flex;
+        align-items: center;
+        z-index: 1;
+        height: 100%;
+        width: 100%;
+        pointer-events: none;
+        user-select: none;
+        padding: 0px 18px;
+        white-space: pre-wrap;
+      }
+    }
+  }
+
+  @media screen and (max-width: 768px) {
+    position: absolute;
+    z-index: 1000;
+    
+    &:not(.open) {
+      width: 100%;
+      height: 40px;
+      border-right: none;
+      border-bottom: 1px solid var(--gray-600);
+
+      .top {
+        width: 100%;
+        padding: 12px;
+        justify-content: flex-end;
+        border-bottom: none;
+
+        .opener {
+          transform: rotate(90deg);
+          position: absolute;
+          right: 12px;
+          top: 50%;
+          transform: translateY(-50%) rotate(90deg);
+        }
+      }
+
+      .indicators {
+        display: none;
+      }
+    }
+    
+    &.open {
+      width: 100%;
+      max-width: 100%;
+    }
+
+    .side-panel-container {
+      width: 100%;
+    }
+
+    .react-select {
+      width: 140px;
+    }
+
+    .indicators {
+      padding: 24px 15px;
+      
+      .streaming-indicator {
+        width: 110px;
+      }
+    }
+  }
+}
+
+.side-panel-responses,
+.side-panel-requests {
+  flex-grow: 1;
+  flex-shrink: 1;
+  overflow-x: hidden;
+  overflow-y: auto;
+  width: 100%;
+  display: block;
+  margin-left: 8px;
+}
+
+.top {
+  width: 100%;
+  flex-grow: 0;
+  flex-shrink: 0;
+  height: 30px;
+  display: flex;
+  align-self: flex-end;
+  align-items: center;
+  transition: all 0.2s ease-in;
+}
+.top button {
+  background: transparent;
+  border: 0;
+  cursor: pointer;
+  font-size: 1.25rem;
+  line-height: 1.75rem;
+  padding: 4px;
+}

src/contexts/LiveAPIContext.tsx

@@ -0,0 +1,48 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createContext, FC, type ReactNode, useContext } from "react";
+import { useLiveAPI, type UseLiveAPIResults } from "../hooks/use-live-api";
+
+const LiveAPIContext = createContext<UseLiveAPIResults | undefined>(undefined);
+
+export type LiveAPIProviderProps = {
+  children: ReactNode;
+  url?: string;
+};
+
+export const LiveAPIProvider: FC<LiveAPIProviderProps> = ({
+  url = process.env.NODE_ENV === 'development' 
+    ? `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//localhost:3001/ws`
+    : `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`,
+  children,
+}) => {
+  const liveAPI = useLiveAPI({ url });
+
+  return (
+    <LiveAPIContext.Provider value={liveAPI}>
+      {children}
+    </LiveAPIContext.Provider>
+  );
+};
+
+export const useLiveAPIContext = () => {
+  const context = useContext(LiveAPIContext);
+  if (!context) {
+    throw new Error("useLiveAPIContext must be used within a LiveAPIProvider");
+  }
+  return context;
+};

src/hooks/use-live-api.ts

@@ -0,0 +1,116 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import {
+  MultimodalLiveAPIClientConnection,
+  MultimodalLiveClient,
+} from "../lib/multimodal-live-client";
+import { LiveConfig } from "../multimodal-live-types";
+import { AudioStreamer } from "../lib/audio-streamer";
+import { audioContext } from "../lib/utils";
+import VolMeterWorket from "../lib/worklets/vol-meter";
+
+export type UseLiveAPIResults = {
+  client: MultimodalLiveClient;
+  setConfig: (config: LiveConfig) => void;
+  config: LiveConfig;
+  connected: boolean;
+  connect: () => Promise<void>;
+  disconnect: () => Promise<void>;
+  volume: number;
+};
+
+export function useLiveAPI({
+  url,
+  apiKey,
+}: MultimodalLiveAPIClientConnection): UseLiveAPIResults {
+  const client = useMemo(
+    () => new MultimodalLiveClient({ url, apiKey }),
+    [url, apiKey],
+  );
+  const audioStreamerRef = useRef<AudioStreamer | null>(null);
+
+  const [connected, setConnected] = useState(false);
+  const [config, setConfig] = useState<LiveConfig>({
+    model: "models/gemini-2.0-flash-exp",
+  });
+  const [volume, setVolume] = useState(0);
+
+  // register audio for streaming server -> speakers
+  useEffect(() => {
+    if (!audioStreamerRef.current) {
+      audioContext({ id: "audio-out" }).then((audioCtx: AudioContext) => {
+        audioStreamerRef.current = new AudioStreamer(audioCtx);
+        audioStreamerRef.current
+          .addWorklet<any>("vumeter-out", VolMeterWorket, (ev: any) => {
+            setVolume(ev.data.volume);
+          })
+          .then(() => {
+            // Successfully added worklet
+          });
+      });
+    }
+  }, [audioStreamerRef]);
+
+  useEffect(() => {
+    const onClose = () => {
+      setConnected(false);
+    };
+
+    const stopAudioStreamer = () => audioStreamerRef.current?.stop();
+
+    const onAudio = (data: ArrayBuffer) =>
+      audioStreamerRef.current?.addPCM16(new Uint8Array(data));
+
+    client
+      .on("close", onClose)
+      .on("interrupted", stopAudioStreamer)
+      .on("audio", onAudio);
+
+    return () => {
+      client
+        .off("close", onClose)
+        .off("interrupted", stopAudioStreamer)
+        .off("audio", onAudio);
+    };
+  }, [client]);
+
+  const connect = useCallback(async () => {
+    console.log(config);
+    if (!config) {
+      throw new Error("config has not been set");
+    }
+    client.disconnect();
+    await client.connect(config);
+    setConnected(true);
+  }, [client, setConnected, config]);
+
+  const disconnect = useCallback(async () => {
+    client.disconnect();
+    setConnected(false);
+  }, [setConnected, client]);
+
+  return {
+    client,
+    config,
+    setConfig,
+    connected,
+    connect,
+    disconnect,
+    volume,
+  };
+}

src/hooks/use-media-stream-mux.ts

@@ -0,0 +1,23 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type UseMediaStreamResult = {
+  type: "webcam" | "screen";
+  start: () => Promise<MediaStream>;
+  stop: () => void;
+  isStreaming: boolean;
+  stream: MediaStream | null;
+};

src/hooks/use-screen-capture.ts

@@ -0,0 +1,72 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { useState, useEffect } from "react";
+import { UseMediaStreamResult } from "./use-media-stream-mux";
+
+export function useScreenCapture(): UseMediaStreamResult {
+  const [stream, setStream] = useState<MediaStream | null>(null);
+  const [isStreaming, setIsStreaming] = useState(false);
+
+  useEffect(() => {
+    const handleStreamEnded = () => {
+      setIsStreaming(false);
+      setStream(null);
+    };
+    if (stream) {
+      stream
+        .getTracks()
+        .forEach((track) => track.addEventListener("ended", handleStreamEnded));
+      return () => {
+        stream
+          .getTracks()
+          .forEach((track) =>
+            track.removeEventListener("ended", handleStreamEnded),
+          );
+      };
+    }
+  }, [stream]);
+
+  const start = async () => {
+    // const controller = new CaptureController();
+    // controller.setFocusBehavior("no-focus-change");
+    const mediaStream = await navigator.mediaDevices.getDisplayMedia({
+      video: true,
+      // controller
+    });
+    setStream(mediaStream);
+    setIsStreaming(true);
+    return mediaStream;
+  };
+
+  const stop = () => {
+    if (stream) {
+      stream.getTracks().forEach((track) => track.stop());
+      setStream(null);
+      setIsStreaming(false);
+    }
+  };
+
+  const result: UseMediaStreamResult = {
+    type: "screen",
+    start,
+    stop,
+    isStreaming,
+    stream,
+  };
+
+  return result;
+}

src/hooks/use-webcam.ts

@@ -0,0 +1,69 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { useState, useEffect } from "react";
+import { UseMediaStreamResult } from "./use-media-stream-mux";
+
+export function useWebcam(): UseMediaStreamResult {
+  const [stream, setStream] = useState<MediaStream | null>(null);
+  const [isStreaming, setIsStreaming] = useState(false);
+
+  useEffect(() => {
+    const handleStreamEnded = () => {
+      setIsStreaming(false);
+      setStream(null);
+    };
+    if (stream) {
+      stream
+        .getTracks()
+        .forEach((track) => track.addEventListener("ended", handleStreamEnded));
+      return () => {
+        stream
+          .getTracks()
+          .forEach((track) =>
+            track.removeEventListener("ended", handleStreamEnded),
+          );
+      };
+    }
+  }, [stream]);
+
+  const start = async () => {
+    const mediaStream = await navigator.mediaDevices.getUserMedia({
+      video: true,
+    });
+    setStream(mediaStream);
+    setIsStreaming(true);
+    return mediaStream;
+  };
+
+  const stop = () => {
+    if (stream) {
+      stream.getTracks().forEach((track) => track.stop());
+      setStream(null);
+      setIsStreaming(false);
+    }
+  };
+
+  const result: UseMediaStreamResult = {
+    type: "webcam",
+    start,
+    stop,
+    isStreaming,
+    stream,
+  };
+
+  return result;
+}

src/index.css

@@ -0,0 +1,13 @@
+body {
+  margin: 0;
+  font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
+    'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
+    sans-serif;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+
+code {
+  font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
+    monospace;
+}

src/index.tsx

@@ -0,0 +1,35 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import './index.css';
+import App from './App';
+import reportWebVitals from './reportWebVitals';
+
+const root = ReactDOM.createRoot(
+  document.getElementById('root') as HTMLElement
+);
+root.render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>
+);
+
+// If you want to start measuring performance in your app, pass a function
+// to log results (for example: reportWebVitals(console.log))
+// or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals
+reportWebVitals();

src/lib/audio-recorder.ts

@@ -0,0 +1,417 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { audioContext } from "./utils";
+import AudioRecordingWorklet from "./worklets/audio-processing";
+import SafariAudioRecordingWorklet from "./worklets/safari-audio-processing";
+import VolMeterWorket from "./worklets/vol-meter";
+
+import { createWorketFromSrc } from "./audioworklet-registry";
+import EventEmitter from "eventemitter3";
+
+function arrayBufferToBase64(buffer: ArrayBuffer) {
+  var binary = "";
+  var bytes = new Uint8Array(buffer);
+  var len = bytes.byteLength;
+  for (var i = 0; i < len; i++) {
+    binary += String.fromCharCode(bytes[i]);
+  }
+  return window.btoa(binary);
+}
+
+// Add Safari-specific audio context creation
+async function createSafariAudioContext(sampleRate: number): Promise<AudioContext> {
+  console.log('Creating Safari audio context with options:', { sampleRate });
+  
+  // Safari requires webkit prefix
+  const AudioContextClass = (window as any).webkitAudioContext || window.AudioContext;
+  console.log('Using AudioContext class:', AudioContextClass.name);
+  
+  const ctx = new AudioContextClass({
+    sampleRate,
+    latencyHint: 'interactive'
+  });
+  
+  console.log('Safari AudioContext initial state:', {
+    state: ctx.state,
+    sampleRate: ctx.sampleRate,
+    baseLatency: ctx.baseLatency,
+    destination: ctx.destination,
+  });
+
+  // Safari requires user interaction to start audio context
+  if (ctx.state === 'suspended') {
+    console.log('Attempting to resume suspended Safari audio context...');
+    try {
+      await ctx.resume();
+      console.log('Successfully resumed Safari audio context:', ctx.state);
+    } catch (err) {
+      console.error('Failed to resume Safari audio context:', err);
+      throw err;
+    }
+  }
+
+  return ctx;
+}
+
+export class AudioRecorder extends EventEmitter {
+  stream: MediaStream | undefined;
+  audioContext: AudioContext | undefined;
+  source: MediaStreamAudioSourceNode | undefined;
+  recording: boolean = false;
+  recordingWorklet: AudioWorkletNode | undefined;
+  vuWorklet: AudioWorkletNode | undefined;
+
+  private starting: Promise<void> | null = null;
+
+  // Add browser detection
+  isSafari: boolean;
+  isIOS: boolean;
+
+  constructor(public sampleRate = 16000) {
+    super();
+    this.isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
+    this.isIOS = /iPad|iPhone|iPod/.test(navigator.userAgent) && !(window as any).MSStream;
+    console.log('AudioRecorder initialized:', { 
+      isSafari: this.isSafari, 
+      isIOS: this.isIOS,
+      sampleRate: this.sampleRate,
+      userAgent: navigator.userAgent,
+      webAudioSupport: !!(window.AudioContext || (window as any).webkitAudioContext),
+      mediaDevicesSupport: !!navigator.mediaDevices
+    });
+  }
+
+  async start() {
+    if (!navigator.mediaDevices?.getUserMedia) {
+      console.error('MediaDevices API not available:', { 
+        mediaDevices: !!navigator.mediaDevices,
+        getUserMedia: !!(navigator.mediaDevices && navigator.mediaDevices.getUserMedia)
+      });
+      throw new Error("Could not request user media");
+    }
+
+    console.log('Starting AudioRecorder with full environment info:', {
+      userAgent: navigator.userAgent,
+      platform: navigator.platform,
+      vendor: navigator.vendor,
+      audioWorkletSupport: !!(window.AudioWorklet),
+      sampleRate: this.sampleRate,
+      existingAudioContext: !!this.audioContext,
+      existingStream: !!this.stream,
+      isSafari: this.isSafari
+    });
+
+    this.starting = new Promise(async (resolve, reject) => {
+      try {
+        if (this.isSafari) {
+          // Safari implementation
+          console.log('Safari detected - using Safari-specific audio initialization');
+          
+          // 1. First get audio permissions
+          console.log('Requesting audio permissions first for Safari...');
+          const constraints = {
+            audio: {
+              echoCancellation: false,
+              noiseSuppression: false,
+              autoGainControl: false,
+              sampleRate: this.sampleRate,
+              channelCount: 1
+            }
+          };
+          console.log('Safari audio constraints:', constraints);
+          
+          try {
+            this.stream = await navigator.mediaDevices.getUserMedia(constraints);
+            const track = this.stream.getAudioTracks()[0];
+            console.log('Safari audio permissions granted:', {
+              track: track.label,
+              settings: track.getSettings(),
+              constraints: track.getConstraints(),
+              enabled: track.enabled,
+              muted: track.muted,
+              readyState: track.readyState
+            });
+          } catch (err) {
+            console.error('Failed to get Safari audio permissions:', err);
+            throw err;
+          }
+
+          // 2. Create and initialize audio context
+          try {
+            this.audioContext = await createSafariAudioContext(this.sampleRate);
+            console.log('Safari audio context ready:', {
+              state: this.audioContext.state,
+              currentTime: this.audioContext.currentTime
+            });
+          } catch (err) {
+            console.error('Failed to initialize Safari audio context:', err);
+            throw err;
+          }
+
+          // 3. Create and connect audio source
+          try {
+            console.log('Creating Safari audio source...');
+            this.source = this.audioContext.createMediaStreamSource(this.stream);
+            console.log('Safari audio source created successfully:', {
+              numberOfInputs: this.source.numberOfInputs,
+              numberOfOutputs: this.source.numberOfOutputs,
+              channelCount: this.source.channelCount
+            });
+          } catch (err) {
+            console.error('Failed to create Safari audio source:', err);
+            throw err;
+          }
+
+          // 4. Load and create worklet
+          try {
+            const workletName = "audio-recorder-worklet";
+            console.log('Loading Safari audio worklet...');
+            const src = createWorketFromSrc(workletName, SafariAudioRecordingWorklet);
+            await this.audioContext.audioWorklet.addModule(src);
+            console.log('Safari audio worklet module loaded');
+
+            this.recordingWorklet = new AudioWorkletNode(
+              this.audioContext,
+              workletName,
+              {
+                numberOfInputs: 1,
+                numberOfOutputs: 1,
+                channelCount: 1,
+                processorOptions: {
+                  sampleRate: this.sampleRate
+                }
+              }
+            );
+
+            // Add detailed error handlers
+            this.recordingWorklet.onprocessorerror = (event) => {
+              console.error('Safari AudioWorklet processor error:', event);
+            };
+
+            this.recordingWorklet.port.onmessageerror = (event) => {
+              console.error('Safari AudioWorklet message error:', event);
+            };
+
+            // Add data handler with detailed logging
+            this.recordingWorklet.port.onmessage = (ev: MessageEvent) => {
+              const data = ev.data.data;
+              console.log('Safari AudioWorklet message received:', {
+                eventType: ev.data.event,
+                hasData: !!data,
+                dataType: data ? typeof data : null,
+                timestamp: Date.now()
+              });
+
+              if (data?.int16arrayBuffer) {
+                console.log('Processing Safari audio chunk:', {
+                  byteLength: data.int16arrayBuffer.byteLength,
+                  timestamp: Date.now()
+                });
+                const arrayBufferString = arrayBufferToBase64(data.int16arrayBuffer);
+                this.emit("data", arrayBufferString);
+              } else {
+                console.warn('Invalid Safari audio chunk received:', ev.data);
+              }
+            };
+
+            console.log('Safari AudioWorkletNode created successfully');
+          } catch (err) {
+            console.error('Failed to setup Safari audio worklet:', err);
+            throw err;
+          }
+
+          // 5. Connect nodes
+          try {
+            console.log('Connecting Safari audio nodes...');
+            this.source.connect(this.recordingWorklet);
+            console.log('Safari audio nodes connected successfully');
+          } catch (err) {
+            console.error('Failed to connect Safari audio nodes:', err);
+            throw err;
+          }
+
+        } else {
+          // Chrome/other browsers implementation
+          console.log('Non-Safari browser detected - using standard audio initialization');
+          
+          // Get media stream first for Chrome
+          const constraints = {
+            audio: {
+              echoCancellation: true,
+              noiseSuppression: true,
+              autoGainControl: true,
+              sampleRate: this.sampleRate
+            }
+          };
+          console.log('Chrome audio constraints:', constraints);
+          
+          try {
+            this.stream = await navigator.mediaDevices.getUserMedia(constraints);
+            const track = this.stream.getAudioTracks()[0];
+            console.log('Chrome audio permissions granted:', {
+              track: track.label,
+              settings: track.getSettings()
+            });
+          } catch (err) {
+            console.error('Failed to get Chrome audio permissions:', err);
+            throw err;
+          }
+
+          // Create audio context after getting stream for Chrome
+          try {
+            console.log('Creating Chrome audio context...');
+            this.audioContext = await audioContext({ sampleRate: this.sampleRate });
+            console.log('Chrome audio context created:', {
+              state: this.audioContext.state,
+              sampleRate: this.audioContext.sampleRate
+            });
+          } catch (err) {
+            console.error('Failed to create Chrome audio context:', err);
+            throw err;
+          }
+
+          // Create media stream source
+          try {
+            console.log('Creating Chrome audio source...');
+            this.source = this.audioContext.createMediaStreamSource(this.stream);
+            console.log('Chrome audio source created');
+          } catch (err) {
+            console.error('Failed to create Chrome audio source:', err);
+            throw err;
+          }
+
+          // Load and create standard worklet
+          try {
+            const workletName = "audio-recorder-worklet";
+            console.log('Loading Chrome audio worklet...');
+            const src = createWorketFromSrc(workletName, AudioRecordingWorklet);
+            await this.audioContext.audioWorklet.addModule(src);
+            console.log('Chrome audio worklet loaded');
+
+            this.recordingWorklet = new AudioWorkletNode(
+              this.audioContext,
+              workletName,
+              {
+                numberOfInputs: 1,
+                numberOfOutputs: 1,
+                channelCount: 1,
+                processorOptions: {
+                  sampleRate: this.sampleRate
+                }
+              }
+            );
+
+            // Add error handlers
+            this.recordingWorklet.onprocessorerror = (event) => {
+              console.error('Chrome AudioWorklet processor error:', event);
+            };
+
+            this.recordingWorklet.port.onmessageerror = (event) => {
+              console.error('Chrome AudioWorklet message error:', event);
+            };
+
+            // Add data handler
+            this.recordingWorklet.port.onmessage = async (ev: MessageEvent) => {
+              const arrayBuffer = ev.data.data?.int16arrayBuffer;
+              if (arrayBuffer) {
+                const arrayBufferString = arrayBufferToBase64(arrayBuffer);
+                this.emit("data", arrayBufferString);
+              } else {
+                console.warn('Invalid Chrome audio chunk received:', ev.data);
+              }
+            };
+
+            console.log('Chrome AudioWorkletNode created');
+          } catch (err) {
+            console.error('Failed to setup Chrome audio worklet:', err);
+            throw err;
+          }
+
+          // Connect nodes
+          try {
+            console.log('Connecting Chrome audio nodes...');
+            this.source.connect(this.recordingWorklet);
+            console.log('Chrome audio nodes connected');
+
+            // Set up VU meter
+            const vuWorkletName = "vu-meter";
+            await this.audioContext.audioWorklet.addModule(
+              createWorketFromSrc(vuWorkletName, VolMeterWorket),
+            );
+            this.vuWorklet = new AudioWorkletNode(this.audioContext, vuWorkletName);
+            this.vuWorklet.port.onmessage = (ev: MessageEvent) => {
+              this.emit("volume", ev.data.volume);
+            };
+            this.source.connect(this.vuWorklet);
+            console.log('Chrome VU meter connected');
+          } catch (err) {
+            console.error('Failed to connect Chrome audio nodes:', err);
+            throw err;
+          }
+        }
+
+        this.recording = true;
+        console.log('Recording started successfully');
+        resolve();
+        this.starting = null;
+      } catch (error) {
+        console.error('Failed to start recording:', error);
+        this.stop();
+        reject(error);
+        this.starting = null;
+      }
+    });
+    return this.starting;
+  }
+
+  stop() {
+    console.log('Stopping audio recorder...');
+    // its plausible that stop would be called before start completes
+    // such as if the websocket immediately hangs up
+    const handleStop = () => {
+      try {
+        if (this.source) {
+          console.log('Disconnecting audio source...');
+          this.source.disconnect();
+        }
+        if (this.stream) {
+          console.log('Stopping media stream tracks...');
+          this.stream.getTracks().forEach(track => {
+            track.stop();
+            console.log('Stopped track:', track.label);
+          });
+        }
+        if (this.audioContext && this.isSafari) {
+          console.log('Closing Safari audio context...');
+          this.audioContext.close();
+        }
+        this.stream = undefined;
+        this.recordingWorklet = undefined;
+        this.vuWorklet = undefined;
+        console.log('Audio recorder stopped successfully');
+      } catch (err) {
+        console.error('Error while stopping audio recorder:', err);
+      }
+    };
+    if (this.starting) {
+      console.log('Stop called while starting - waiting for start to complete...');
+      this.starting.then(handleStop);
+      return;
+    }
+    handleStop();
+  }
+}

src/lib/audio-streamer.ts

@@ -0,0 +1,270 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  createWorketFromSrc,
+  registeredWorklets,
+} from "./audioworklet-registry";
+
+export class AudioStreamer {
+  public audioQueue: Float32Array[] = [];
+  private isPlaying: boolean = false;
+  private sampleRate: number = 24000;
+  private bufferSize: number = 7680;
+  private processingBuffer: Float32Array = new Float32Array(0);
+  private scheduledTime: number = 0;
+  public gainNode: GainNode;
+  public source: AudioBufferSourceNode;
+  private isStreamComplete: boolean = false;
+  private checkInterval: number | null = null;
+  private initialBufferTime: number = 0.1; //0.1 // 100ms initial buffer
+  private endOfQueueAudioSource: AudioBufferSourceNode | null = null;
+
+  public onComplete = () => {};
+
+  constructor(public context: AudioContext) {
+    this.gainNode = this.context.createGain();
+    this.source = this.context.createBufferSource();
+    this.gainNode.connect(this.context.destination);
+    this.addPCM16 = this.addPCM16.bind(this);
+  }
+
+  async addWorklet<T extends (d: any) => void>(
+    workletName: string,
+    workletSrc: string,
+    handler: T,
+  ): Promise<this> {
+    let workletsRecord = registeredWorklets.get(this.context);
+    if (workletsRecord && workletsRecord[workletName]) {
+      // the worklet already exists on this context
+      // add the new handler to it
+      workletsRecord[workletName].handlers.push(handler);
+      return Promise.resolve(this);
+      //throw new Error(`Worklet ${workletName} already exists on context`);
+    }
+
+    if (!workletsRecord) {
+      registeredWorklets.set(this.context, {});
+      workletsRecord = registeredWorklets.get(this.context)!;
+    }
+
+    // create new record to fill in as becomes available
+    workletsRecord[workletName] = { handlers: [handler] };
+
+    const src = createWorketFromSrc(workletName, workletSrc);
+    await this.context.audioWorklet.addModule(src);
+    const worklet = new AudioWorkletNode(this.context, workletName);
+
+    //add the node into the map
+    workletsRecord[workletName].node = worklet;
+
+    return this;
+  }
+
+  addPCM16(chunk: Uint8Array) {
+    const float32Array = new Float32Array(chunk.length / 2);
+    const dataView = new DataView(chunk.buffer);
+
+    for (let i = 0; i < chunk.length / 2; i++) {
+      try {
+        const int16 = dataView.getInt16(i * 2, true);
+        float32Array[i] = int16 / 32768;
+      } catch (e) {
+        console.error(e);
+        // console.log(
+        //   `dataView.length: ${dataView.byteLength},  i * 2: ${i * 2}`,
+        // );
+      }
+    }
+
+    const newBuffer = new Float32Array(
+      this.processingBuffer.length + float32Array.length,
+    );
+    newBuffer.set(this.processingBuffer);
+    newBuffer.set(float32Array, this.processingBuffer.length);
+    this.processingBuffer = newBuffer;
+
+    while (this.processingBuffer.length >= this.bufferSize) {
+      const buffer = this.processingBuffer.slice(0, this.bufferSize);
+      this.audioQueue.push(buffer);
+      this.processingBuffer = this.processingBuffer.slice(this.bufferSize);
+    }
+
+    if (!this.isPlaying) {
+      this.isPlaying = true;
+      // Initialize scheduledTime only when we start playing
+      this.scheduledTime = this.context.currentTime + this.initialBufferTime;
+      this.scheduleNextBuffer();
+    }
+  }
+
+  private createAudioBuffer(audioData: Float32Array): AudioBuffer {
+    const audioBuffer = this.context.createBuffer(
+      1,
+      audioData.length,
+      this.sampleRate,
+    );
+    audioBuffer.getChannelData(0).set(audioData);
+    return audioBuffer;
+  }
+
+  private scheduleNextBuffer() {
+    const SCHEDULE_AHEAD_TIME = 0.2;
+
+    while (
+      this.audioQueue.length > 0 &&
+      this.scheduledTime < this.context.currentTime + SCHEDULE_AHEAD_TIME
+    ) {
+      const audioData = this.audioQueue.shift()!;
+      const audioBuffer = this.createAudioBuffer(audioData);
+      const source = this.context.createBufferSource();
+
+      if (this.audioQueue.length === 0) {
+        if (this.endOfQueueAudioSource) {
+          this.endOfQueueAudioSource.onended = null;
+        }
+        this.endOfQueueAudioSource = source;
+        source.onended = () => {
+          if (
+            !this.audioQueue.length &&
+            this.endOfQueueAudioSource === source
+          ) {
+            this.endOfQueueAudioSource = null;
+            this.onComplete();
+          }
+        };
+      }
+
+      source.buffer = audioBuffer;
+      source.connect(this.gainNode);
+
+      const worklets = registeredWorklets.get(this.context);
+
+      if (worklets) {
+        Object.entries(worklets).forEach(([workletName, graph]) => {
+          const { node, handlers } = graph;
+          if (node) {
+            source.connect(node);
+            node.port.onmessage = function (ev: MessageEvent) {
+              handlers.forEach((handler) => {
+                handler.call(node.port, ev);
+              });
+            };
+            node.connect(this.context.destination);
+          }
+        });
+      }
+
+      // i added this trying to fix clicks
+      // this.gainNode.gain.setValueAtTime(0, 0);
+      // this.gainNode.gain.linearRampToValueAtTime(1, 1);
+
+      // Ensure we never schedule in the past
+      const startTime = Math.max(this.scheduledTime, this.context.currentTime);
+      source.start(startTime);
+
+      this.scheduledTime = startTime + audioBuffer.duration;
+    }
+
+    if (this.audioQueue.length === 0 && this.processingBuffer.length === 0) {
+      if (this.isStreamComplete) {
+        this.isPlaying = false;
+        if (this.checkInterval) {
+          clearInterval(this.checkInterval);
+          this.checkInterval = null;
+        }
+      } else {
+        if (!this.checkInterval) {
+          this.checkInterval = window.setInterval(() => {
+            if (
+              this.audioQueue.length > 0 ||
+              this.processingBuffer.length >= this.bufferSize
+            ) {
+              this.scheduleNextBuffer();
+            }
+          }, 100) as unknown as number;
+        }
+      }
+    } else {
+      const nextCheckTime =
+        (this.scheduledTime - this.context.currentTime) * 1000;
+      setTimeout(
+        () => this.scheduleNextBuffer(),
+        Math.max(0, nextCheckTime - 50),
+      );
+    }
+  }
+
+  stop() {
+    this.isPlaying = false;
+    this.isStreamComplete = true;
+    this.audioQueue = [];
+    this.processingBuffer = new Float32Array(0);
+    this.scheduledTime = this.context.currentTime;
+
+    if (this.checkInterval) {
+      clearInterval(this.checkInterval);
+      this.checkInterval = null;
+    }
+
+    this.gainNode.gain.linearRampToValueAtTime(
+      0,
+      this.context.currentTime + 0.1,
+    );
+
+    setTimeout(() => {
+      this.gainNode.disconnect();
+      this.gainNode = this.context.createGain();
+      this.gainNode.connect(this.context.destination);
+    }, 200);
+  }
+
+  async resume() {
+    if (this.context.state === "suspended") {
+      await this.context.resume();
+    }
+    this.isStreamComplete = false;
+    this.scheduledTime = this.context.currentTime + this.initialBufferTime;
+    this.gainNode.gain.setValueAtTime(1, this.context.currentTime);
+  }
+
+  complete() {
+    this.isStreamComplete = true;
+    if (this.processingBuffer.length > 0) {
+      this.audioQueue.push(this.processingBuffer);
+      this.processingBuffer = new Float32Array(0);
+      if (this.isPlaying) {
+        this.scheduleNextBuffer();
+      }
+    } else {
+      this.onComplete();
+    }
+  }
+}
+
+// // Usage example:
+// const audioStreamer = new AudioStreamer();
+//
+// // In your streaming code:
+// function handleChunk(chunk: Uint8Array) {
+//   audioStreamer.handleChunk(chunk);
+// }
+//
+// // To start playing (call this in response to a user interaction)
+// await audioStreamer.resume();
+//
+// // To stop playing
+// // audioStreamer.stop();

src/lib/audioworklet-registry.ts

@@ -0,0 +1,43 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A registry to map attached worklets by their audio-context
+ * any module using `audioContext.audioWorklet.addModule(` should register the worklet here
+ */
+export type WorkletGraph = {
+  node?: AudioWorkletNode;
+  handlers: Array<(this: MessagePort, ev: MessageEvent) => any>;
+};
+
+export const registeredWorklets: Map<
+  AudioContext,
+  Record<string, WorkletGraph>
+> = new Map();
+
+export const createWorketFromSrc = (
+  workletName: string,
+  workletSrc: string,
+) => {
+  const script = new Blob(
+    [`registerProcessor("${workletName}", ${workletSrc})`],
+    {
+      type: "application/javascript",
+    },
+  );
+
+  return URL.createObjectURL(script);
+};

src/lib/multimodal-live-client.ts

@@ -0,0 +1,313 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Content, GenerativeContentBlob, Part } from "@google/generative-ai";
+import { EventEmitter } from "eventemitter3";
+import { difference } from "lodash";
+import {
+  ClientContentMessage,
+  isInterrupted,
+  isModelTurn,
+  isServerContentMessage,
+  isSetupCompleteMessage,
+  isToolCallCancellationMessage,
+  isToolCallMessage,
+  isTurnComplete,
+  LiveIncomingMessage,
+  ModelTurn,
+  RealtimeInputMessage,
+  ServerContent,
+  SetupMessage,
+  StreamingLog,
+  ToolCall,
+  ToolCallCancellation,
+  ToolResponseMessage,
+  type LiveConfig,
+} from "../multimodal-live-types";
+import { blobToJSON, base64ToArrayBuffer } from "./utils";
+
+/**
+ * the events that this client will emit
+ */
+interface MultimodalLiveClientEventTypes {
+  open: () => void;
+  log: (log: StreamingLog) => void;
+  close: (event: CloseEvent) => void;
+  audio: (data: ArrayBuffer) => void;
+  content: (data: ServerContent) => void;
+  interrupted: () => void;
+  setupcomplete: () => void;
+  turncomplete: () => void;
+  toolcall: (toolCall: ToolCall) => void;
+  toolcallcancellation: (toolcallCancellation: ToolCallCancellation) => void;
+}
+
+export type MultimodalLiveAPIClientConnection = {
+  url?: string;
+  apiKey?: string;
+};
+
+/**
+ * A event-emitting class that manages the connection to the websocket and emits
+ * events to the rest of the application.
+ * If you dont want to use react you can still use this.
+ */
+export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEventTypes> {
+  public ws: WebSocket | null = null;
+  protected config: LiveConfig | null = null;
+  public url: string;
+
+  constructor({ url, apiKey }: MultimodalLiveAPIClientConnection = {}) {
+    super();
+    this.url = url || `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`;
+    this.send = this.send.bind(this);
+  }
+
+  log(type: string, message: StreamingLog["message"]) {
+    const log: StreamingLog = {
+      date: new Date(),
+      type,
+      message,
+    };
+    this.emit("log", log);
+  }
+
+  connect(config: LiveConfig): Promise<boolean> {
+    this.config = config;
+
+    const ws = new WebSocket(this.url);
+
+    ws.addEventListener("message", async (evt: MessageEvent) => {
+      if (evt.data instanceof Blob) {
+        this.receive(evt.data);
+      } else {
+        console.log("non blob message", evt);
+      }
+    });
+    return new Promise((resolve, reject) => {
+      const onError = (ev: Event) => {
+        this.disconnect(ws);
+        const message = `Could not connect to "${this.url}"`;
+        this.log(`server.${ev.type}`, message);
+        reject(new Error(message));
+      };
+      ws.addEventListener("error", onError);
+      ws.addEventListener("open", (ev: Event) => {
+        if (!this.config) {
+          reject("Invalid config sent to `connect(config)`");
+          return;
+        }
+        this.log(`client.${ev.type}`, `connected to socket`);
+        this.emit("open");
+
+        this.ws = ws;
+
+        const setupMessage: SetupMessage = {
+          setup: this.config,
+        };
+        this._sendDirect(setupMessage);
+        this.log("client.send", "setup");
+
+        ws.removeEventListener("error", onError);
+        ws.addEventListener("close", (ev: CloseEvent) => {
+          console.log(ev);
+          this.disconnect(ws);
+          let reason = ev.reason || "";
+          if (reason.toLowerCase().includes("error")) {
+            const prelude = "ERROR]";
+            const preludeIndex = reason.indexOf(prelude);
+            if (preludeIndex > 0) {
+              reason = reason.slice(
+                preludeIndex + prelude.length + 1,
+                Infinity,
+              );
+            }
+          }
+          this.log(
+            `server.${ev.type}`,
+            `disconnected ${reason ? `with reason: ${reason}` : ``}`,
+          );
+          this.emit("close", ev);
+        });
+        resolve(true);
+      });
+    });
+  }
+
+  disconnect(ws?: WebSocket) {
+    // could be that this is an old websocket and theres already a new instance
+    // only close it if its still the correct reference
+    if ((!ws || this.ws === ws) && this.ws) {
+      this.ws.close();
+      this.ws = null;
+      this.log("client.close", `Disconnected`);
+      return true;
+    }
+    return false;
+  }
+
+  protected async receive(blob: Blob) {
+    const response: LiveIncomingMessage = (await blobToJSON(
+      blob,
+    )) as LiveIncomingMessage;
+    if (isToolCallMessage(response)) {
+      this.log("server.toolCall", response);
+      this.emit("toolcall", response.toolCall);
+      return;
+    }
+    if (isToolCallCancellationMessage(response)) {
+      this.log("receive.toolCallCancellation", response);
+      this.emit("toolcallcancellation", response.toolCallCancellation);
+      return;
+    }
+
+    if (isSetupCompleteMessage(response)) {
+      this.log("server.send", "setupComplete");
+      this.emit("setupcomplete");
+      return;
+    }
+
+    // this json also might be `contentUpdate { interrupted: true }`
+    // or contentUpdate { end_of_turn: true }
+    if (isServerContentMessage(response)) {
+      const { serverContent } = response;
+      if (isInterrupted(serverContent)) {
+        this.log("receive.serverContent", "interrupted");
+        this.emit("interrupted");
+        return;
+      }
+      if (isTurnComplete(serverContent)) {
+        this.log("server.send", "turnComplete");
+        this.emit("turncomplete");
+        //plausible theres more to the message, continue
+      }
+
+      if (isModelTurn(serverContent)) {
+        let parts: Part[] = serverContent.modelTurn.parts;
+
+        // when its audio that is returned for modelTurn
+        const audioParts = parts.filter(
+          (p) => p.inlineData && p.inlineData.mimeType.startsWith("audio/pcm"),
+        );
+        const base64s = audioParts.map((p) => p.inlineData?.data);
+
+        // strip the audio parts out of the modelTurn
+        const otherParts = difference(parts, audioParts);
+        // console.log("otherParts", otherParts);
+
+        base64s.forEach((b64) => {
+          if (b64) {
+            const data = base64ToArrayBuffer(b64);
+            this.emit("audio", data);
+            this.log(`server.audio`, `buffer (${data.byteLength})`);
+          }
+        });
+        if (!otherParts.length) {
+          return;
+        }
+
+        parts = otherParts;
+
+        const content: ModelTurn = { modelTurn: { parts } };
+        this.emit("content", content);
+        this.log(`server.content`, response);
+      }
+    } else {
+      console.log("received unmatched message", response);
+    }
+  }
+
+  /**
+   * send realtimeInput, this is base64 chunks of "audio/pcm" and/or "image/jpg"
+   */
+  sendRealtimeInput(chunks: GenerativeContentBlob[]) {
+    let hasAudio = false;
+    let hasVideo = false;
+    for (let i = 0; i < chunks.length; i++) {
+      const ch = chunks[i];
+      if (ch.mimeType.includes("audio")) {
+        hasAudio = true;
+      }
+      if (ch.mimeType.includes("image")) {
+        hasVideo = true;
+      }
+      if (hasAudio && hasVideo) {
+        break;
+      }
+    }
+    const message =
+      hasAudio && hasVideo
+        ? "audio + video"
+        : hasAudio
+          ? "audio"
+          : hasVideo
+            ? "video"
+            : "unknown";
+
+    const data: RealtimeInputMessage = {
+      realtimeInput: {
+        mediaChunks: chunks,
+      },
+    };
+    this._sendDirect(data);
+    this.log(`client.realtimeInput`, message);
+  }
+
+  /**
+   *  send a response to a function call and provide the id of the functions you are responding to
+   */
+  sendToolResponse(toolResponse: ToolResponseMessage["toolResponse"]) {
+    const message: ToolResponseMessage = {
+      toolResponse,
+    };
+
+    this._sendDirect(message);
+    this.log(`client.toolResponse`, message);
+  }
+
+  /**
+   * send normal content parts such as { text }
+   */
+  send(parts: Part | Part[], turnComplete: boolean = true) {
+    parts = Array.isArray(parts) ? parts : [parts];
+    const content: Content = {
+      role: "user",
+      parts,
+    };
+
+    const clientContentRequest: ClientContentMessage = {
+      clientContent: {
+        turns: [content],
+        turnComplete,
+      },
+    };
+
+    this._sendDirect(clientContentRequest);
+    this.log(`client.send`, clientContentRequest);
+  }
+
+  /**
+   *  used internally to send all messages
+   *  don't use directly unless trying to send an unsupported message type
+   */
+  _sendDirect(request: object) {
+    if (!this.ws) {
+      throw new Error("WebSocket is not connected");
+    }
+    const str = JSON.stringify(request);
+    this.ws.send(str);
+  }
+}

src/lib/platform.ts

@@ -0,0 +1,6 @@
+export const isIOS = (): boolean => {
+  const userAgent = window.navigator.userAgent.toLowerCase();
+  return /iphone|ipad|ipod/.test(userAgent) || 
+         // Detect iPad on iOS 13+ (which reports as Mac)
+         (userAgent.includes('mac') && 'ontouchend' in document);
+}; 

src/lib/store-logger.ts

@@ -0,0 +1,65 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { create } from "zustand";
+import { StreamingLog } from "../multimodal-live-types";
+import { mockLogs } from "../components/logger/mock-logs";
+
+interface StoreLoggerState {
+  maxLogs: number;
+  logs: StreamingLog[];
+  log: (streamingLog: StreamingLog) => void;
+  clearLogs: () => void;
+}
+
+export const useLoggerStore = create<StoreLoggerState>((set, get) => ({
+  maxLogs: 500,
+  logs: [], //mockLogs,
+  log: ({ date, type, message }: StreamingLog) => {
+    set((state) => {
+      const prevLog = state.logs.at(-1);
+      if (prevLog && prevLog.type === type && prevLog.message === message) {
+        return {
+          logs: [
+            ...state.logs.slice(0, -1),
+            {
+              date,
+              type,
+              message,
+              count: prevLog.count ? prevLog.count + 1 : 1,
+            } as StreamingLog,
+          ],
+        };
+      }
+      return {
+        logs: [
+          ...state.logs.slice(-(get().maxLogs - 1)),
+          {
+            date,
+            type,
+            message,
+          } as StreamingLog,
+        ],
+      };
+    });
+  },
+
+  clearLogs: () => {
+    console.log("clear log");
+    set({ logs: [] });
+  },
+  setMaxLogs: (n: number) => set({ maxLogs: n }),
+}));

src/lib/utils.ts

@@ -0,0 +1,86 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type GetAudioContextOptions = AudioContextOptions & {
+  id?: string;
+};
+
+const map: Map<string, AudioContext> = new Map();
+
+export const audioContext: (
+  options?: GetAudioContextOptions,
+) => Promise<AudioContext> = (() => {
+  const didInteract = new Promise((res) => {
+    window.addEventListener("pointerdown", res, { once: true });
+    window.addEventListener("keydown", res, { once: true });
+  });
+
+  return async (options?: GetAudioContextOptions) => {
+    try {
+      const a = new Audio();
+      a.src =
+        "data:audio/wav;base64,UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA";
+      await a.play();
+      if (options?.id && map.has(options.id)) {
+        const ctx = map.get(options.id);
+        if (ctx) {
+          return ctx;
+        }
+      }
+      const ctx = new AudioContext(options);
+      if (options?.id) {
+        map.set(options.id, ctx);
+      }
+      return ctx;
+    } catch (e) {
+      await didInteract;
+      if (options?.id && map.has(options.id)) {
+        const ctx = map.get(options.id);
+        if (ctx) {
+          return ctx;
+        }
+      }
+      const ctx = new AudioContext(options);
+      if (options?.id) {
+        map.set(options.id, ctx);
+      }
+      return ctx;
+    }
+  };
+})();
+
+export const blobToJSON = (blob: Blob) =>
+  new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = () => {
+      if (reader.result) {
+        const json = JSON.parse(reader.result as string);
+        resolve(json);
+      } else {
+        reject("oops");
+      }
+    };
+    reader.readAsText(blob);
+  });
+
+export function base64ToArrayBuffer(base64: string) {
+  var binaryString = atob(base64);
+  var bytes = new Uint8Array(binaryString.length);
+  for (let i = 0; i < binaryString.length; i++) {
+    bytes[i] = binaryString.charCodeAt(i);
+  }
+  return bytes.buffer;
+}

src/lib/worklets/audio-processing.ts

@@ -0,0 +1,73 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const AudioRecordingWorklet = `
+class AudioProcessingWorklet extends AudioWorkletProcessor {
+
+  // send and clear buffer every 2048 samples, 
+  // which at 16khz is about 8 times a second
+  buffer = new Int16Array(2048);
+
+  // current write index
+  bufferWriteIndex = 0;
+
+  constructor() {
+    super();
+    this.hasAudio = false;
+  }
+
+  /**
+   * @param inputs Float32Array[][] [input#][channel#][sample#] so to access first inputs 1st channel inputs[0][0]
+   * @param outputs Float32Array[][]
+   */
+  process(inputs) {
+    if (inputs[0].length) {
+      const channel0 = inputs[0][0];
+      this.processChunk(channel0);
+    }
+    return true;
+  }
+
+  sendAndClearBuffer(){
+    this.port.postMessage({
+      event: "chunk",
+      data: {
+        int16arrayBuffer: this.buffer.slice(0, this.bufferWriteIndex).buffer,
+      },
+    });
+    this.bufferWriteIndex = 0;
+  }
+
+  processChunk(float32Array) {
+    const l = float32Array.length;
+    
+    for (let i = 0; i < l; i++) {
+      // convert float32 -1 to 1 to int16 -32768 to 32767
+      const int16Value = float32Array[i] * 32768;
+      this.buffer[this.bufferWriteIndex++] = int16Value;
+      if(this.bufferWriteIndex >= this.buffer.length) {
+        this.sendAndClearBuffer();
+      }
+    }
+
+    if(this.bufferWriteIndex >= this.buffer.length) {
+      this.sendAndClearBuffer();
+    }
+  }
+}
+`;
+
+export default AudioRecordingWorklet;

src/lib/worklets/safari-audio-processing.ts

@@ -0,0 +1,99 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const SafariAudioRecordingWorklet = `
+class AudioProcessingWorklet extends AudioWorkletProcessor {
+  // Safari seems to work better with smaller buffer sizes
+  // and more frequent updates
+  buffer = new Int16Array(1024);
+  bufferWriteIndex = 0;
+  lastProcessTime = 0;
+  sampleRate = 0;
+
+  constructor(options) {
+    super();
+    console.log('Safari AudioProcessingWorklet constructed with options:', options);
+    this.sampleRate = options.processorOptions?.sampleRate || sampleRate;
+    console.log('Using sample rate:', this.sampleRate);
+  }
+
+  process(inputs) {
+    // Log processing details periodically
+    const now = currentTime;
+    if (now - this.lastProcessTime > 1) {
+      console.log('Safari AudioProcessingWorklet processing:', {
+        inputChannels: inputs[0]?.length,
+        inputSamples: inputs[0]?.[0]?.length,
+        bufferWriteIndex: this.bufferWriteIndex,
+        time: now
+      });
+      this.lastProcessTime = now;
+    }
+
+    if (!inputs[0]?.length) {
+      console.warn('No input channels available');
+      return true;
+    }
+
+    const channel0 = inputs[0][0];
+    if (!channel0?.length) {
+      console.warn('Empty input channel');
+      return true;
+    }
+
+    this.processChunk(channel0);
+    return true;
+  }
+
+  sendAndClearBuffer() {
+    if (this.bufferWriteIndex > 0) {
+      this.port.postMessage({
+        event: "chunk",
+        data: {
+          int16arrayBuffer: this.buffer.slice(0, this.bufferWriteIndex).buffer,
+        },
+      });
+      this.bufferWriteIndex = 0;
+    }
+  }
+
+  processChunk(float32Array) {
+    // Safari can sometimes send empty arrays or undefined
+    if (!float32Array?.length) {
+      return;
+    }
+
+    const l = float32Array.length;
+    for (let i = 0; i < l; i++) {
+      // Convert float32 -1 to 1 to int16 -32768 to 32767
+      // Add some additional gain for Safari which tends to be quieter
+      const int16Value = Math.max(-32768, Math.min(32767, float32Array[i] * 32768 * 1.5));
+      this.buffer[this.bufferWriteIndex++] = int16Value;
+      
+      if (this.bufferWriteIndex >= this.buffer.length) {
+        this.sendAndClearBuffer();
+      }
+    }
+
+    // Make sure to send any remaining data
+    if (this.bufferWriteIndex > 0) {
+      this.sendAndClearBuffer();
+    }
+  }
+}
+`;
+
+export default SafariAudioRecordingWorklet; 

src/lib/worklets/vol-meter.ts

@@ -0,0 +1,65 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const VolMeterWorket = `
+  class VolMeter extends AudioWorkletProcessor {
+    volume
+    updateIntervalInMS
+    nextUpdateFrame
+
+    constructor() {
+      super()
+      this.volume = 0
+      this.updateIntervalInMS = 25
+      this.nextUpdateFrame = this.updateIntervalInMS
+      this.port.onmessage = event => {
+        if (event.data.updateIntervalInMS) {
+          this.updateIntervalInMS = event.data.updateIntervalInMS
+        }
+      }
+    }
+
+    get intervalInFrames() {
+      return (this.updateIntervalInMS / 1000) * sampleRate
+    }
+
+    process(inputs) {
+      const input = inputs[0]
+
+      if (input.length > 0) {
+        const samples = input[0]
+        let sum = 0
+        let rms = 0
+
+        for (let i = 0; i < samples.length; ++i) {
+          sum += samples[i] * samples[i]
+        }
+
+        rms = Math.sqrt(sum / samples.length)
+        this.volume = Math.max(rms, this.volume * 0.7)
+
+        this.nextUpdateFrame -= samples.length
+        if (this.nextUpdateFrame < 0) {
+          this.nextUpdateFrame += this.intervalInFrames
+          this.port.postMessage({volume: this.volume})
+        }
+      }
+
+      return true
+    }
+  }`;
+
+export default VolMeterWorket;

src/multimodal-live-types.ts

@@ -0,0 +1,242 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type {
+  Content,
+  FunctionCall,
+  GenerationConfig,
+  GenerativeContentBlob,
+  Part,
+  Tool,
+} from "@google/generative-ai";
+
+/**
+ * this module contains type-definitions and Type-Guards
+ */
+
+// Type-definitions
+
+/* outgoing types */
+
+/**
+ * the config to initiate the session
+ */
+export type LiveConfig = {
+  model: string;
+  systemInstruction?: { parts: Part[] };
+  generationConfig?: Partial<LiveGenerationConfig>;
+  tools?: Array<Tool | { googleSearch: {} } | { codeExecution: {} }>;
+};
+
+export type LiveGenerationConfig = GenerationConfig & {
+  responseModalities: "text" | "audio" | "image";
+  speechConfig?: {
+    voiceConfig?: {
+      prebuiltVoiceConfig?: {
+        voiceName: "Puck" | "Charon" | "Kore" | "Fenrir" | "Aoede" | string;
+      };
+    };
+  };
+};
+
+export type LiveOutgoingMessage =
+  | SetupMessage
+  | ClientContentMessage
+  | RealtimeInputMessage
+  | ToolResponseMessage;
+
+export type SetupMessage = {
+  setup: LiveConfig;
+};
+
+export type ClientContentMessage = {
+  clientContent: {
+    turns: Content[];
+    turnComplete: boolean;
+  };
+};
+
+export type RealtimeInputMessage = {
+  realtimeInput: {
+    mediaChunks: GenerativeContentBlob[];
+  };
+};
+
+export type ToolResponseMessage = {
+  toolResponse: {
+    functionResponses: LiveFunctionResponse[];
+  };
+};
+
+export type ToolResponse = ToolResponseMessage["toolResponse"];
+
+export type LiveFunctionResponse = {
+  response: object;
+  id: string;
+};
+
+/** Incoming types */
+
+export type LiveIncomingMessage =
+  | ToolCallCancellationMessage
+  | ToolCallMessage
+  | ServerContentMessage
+  | SetupCompleteMessage;
+
+export type SetupCompleteMessage = { setupComplete: {} };
+
+export type ServerContentMessage = {
+  serverContent: ServerContent;
+};
+
+export type ServerContent = ModelTurn | TurnComplete | Interrupted;
+
+export type ModelTurn = {
+  modelTurn: {
+    parts: Part[];
+  };
+};
+
+export type TurnComplete = { turnComplete: boolean };
+
+export type Interrupted = { interrupted: true };
+
+export type ToolCallCancellationMessage = {
+  toolCallCancellation: {
+    ids: string[];
+  };
+};
+
+export type ToolCallCancellation =
+  ToolCallCancellationMessage["toolCallCancellation"];
+
+export type ToolCallMessage = {
+  toolCall: ToolCall;
+};
+
+export type LiveFunctionCall = FunctionCall & {
+  id: string;
+};
+
+/**
+ * A `toolCall` message
+ */
+export type ToolCall = {
+  functionCalls: LiveFunctionCall[];
+};
+
+/** log types */
+export type StreamingLog = {
+  date: Date;
+  type: string;
+  count?: number;
+  message: string | LiveOutgoingMessage | LiveIncomingMessage;
+};
+
+// Type-Guards
+
+const prop = (a: any, prop: string, kind: string = "object") =>
+  typeof a === "object" && typeof a[prop] === "object";
+
+// outgoing messages
+export const isSetupMessage = (a: unknown): a is SetupMessage =>
+  prop(a, "setup");
+
+export const isClientContentMessage = (a: unknown): a is ClientContentMessage =>
+  prop(a, "clientContent");
+
+export const isRealtimeInputMessage = (a: unknown): a is RealtimeInputMessage =>
+  prop(a, "realtimeInput");
+
+export const isToolResponseMessage = (a: unknown): a is ToolResponseMessage =>
+  prop(a, "toolResponse");
+
+// incoming messages
+export const isSetupCompleteMessage = (a: unknown): a is SetupCompleteMessage =>
+  prop(a, "setupComplete");
+
+export const isServerContentMessage = (a: any): a is ServerContentMessage =>
+  prop(a, "serverContent");
+
+export const isToolCallMessage = (a: any): a is ToolCallMessage =>
+  prop(a, "toolCall");
+
+export const isToolCallCancellationMessage = (
+  a: unknown,
+): a is ToolCallCancellationMessage =>
+  prop(a, "toolCallCancellation") &&
+  isToolCallCancellation((a as any).toolCallCancellation);
+
+export const isModelTurn = (a: any): a is ModelTurn =>
+  typeof (a as ModelTurn).modelTurn === "object";
+
+export const isTurnComplete = (a: any): a is TurnComplete =>
+  typeof (a as TurnComplete).turnComplete === "boolean";
+
+export const isInterrupted = (a: any): a is Interrupted =>
+  (a as Interrupted).interrupted;
+
+export function isToolCall(value: unknown): value is ToolCall {
+  if (!value || typeof value !== "object") return false;
+
+  const candidate = value as Record<string, unknown>;
+
+  return (
+    Array.isArray(candidate.functionCalls) &&
+    candidate.functionCalls.every((call) => isLiveFunctionCall(call))
+  );
+}
+
+export function isToolResponse(value: unknown): value is ToolResponse {
+  if (!value || typeof value !== "object") return false;
+
+  const candidate = value as Record<string, unknown>;
+
+  return (
+    Array.isArray(candidate.functionResponses) &&
+    candidate.functionResponses.every((resp) => isLiveFunctionResponse(resp))
+  );
+}
+
+export function isLiveFunctionCall(value: unknown): value is LiveFunctionCall {
+  if (!value || typeof value !== "object") return false;
+
+  const candidate = value as Record<string, unknown>;
+
+  return (
+    typeof candidate.name === "string" &&
+    typeof candidate.id === "string" &&
+    typeof candidate.args === "object" &&
+    candidate.args !== null
+  );
+}
+
+export function isLiveFunctionResponse(
+  value: unknown,
+): value is LiveFunctionResponse {
+  if (!value || typeof value !== "object") return false;
+
+  const candidate = value as Record<string, unknown>;
+
+  return (
+    typeof candidate.response === "object" && typeof candidate.id === "string"
+  );
+}
+
+export const isToolCallCancellation = (
+  a: unknown,
+): a is ToolCallCancellationMessage["toolCallCancellation"] =>
+  typeof a === "object" && Array.isArray((a as any).ids);