(null);
+
+ const handleMicPermission = async () => {
+ try {
+ console.log('🎤 Requesting iOS microphone permission from modal...');
+ const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+ console.log('✅ iOS microphone permission granted from modal!');
+ // Stop the stream since we don't need it yet - we'll request it again when recording starts
+ stream.getTracks().forEach(track => track.stop());
+ setError(null);
+ onClose();
+ } catch (err) {
+ console.error('❌ iOS microphone permission denied from modal:', err);
+ setError(err instanceof Error ? err.message : 'Failed to access microphone');
+ }
+ };
+
+ if (!isOpen) return null;
+
+ return (
+
+
+
Microphone Access Required
+
+ To use this app on iOS, we need permission to access your microphone.
+ Please tap "Allow" when prompted.
+
+
+ If you've denied permission, you'll need to enable it in your device settings.
+
+ {error && (
+
+ Error: {error}
+
+ )}
+
+ {error ? 'Try Again' : 'Got it'}
+
+
+
+ );
+};
\ No newline at end of file
diff --git a/src/components/logger/Logger.tsx b/src/components/logger/Logger.tsx
new file mode 100644
index 0000000000000000000000000000000000000000..e86520eb43d6db17ff5527f5f439c6dbbc339b33
--- /dev/null
+++ b/src/components/logger/Logger.tsx
@@ -0,0 +1,273 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import "./logger.scss";
+
+import { Part } from "@google/generative-ai";
+import cn from "classnames";
+import { ReactNode } from "react";
+import { useLoggerStore } from "../../lib/store-logger";
+import SyntaxHighlighter from "react-syntax-highlighter";
+import { vs2015 as dark } from "react-syntax-highlighter/dist/esm/styles/hljs";
+import {
+ ClientContentMessage,
+ isClientContentMessage,
+ isInterrupted,
+ isModelTurn,
+ isServerContentMessage,
+ isToolCallCancellationMessage,
+ isToolCallMessage,
+ isToolResponseMessage,
+ isTurnComplete,
+ ModelTurn,
+ ServerContentMessage,
+ StreamingLog,
+ ToolCallCancellationMessage,
+ ToolCallMessage,
+ ToolResponseMessage,
+} from "../../multimodal-live-types";
+
+const formatTime = (d: Date) => d.toLocaleTimeString().slice(0, -3);
+
+const LogEntry = ({
+ log,
+ MessageComponent,
+}: {
+ log: StreamingLog;
+ MessageComponent: ({
+ message,
+ }: {
+ message: StreamingLog["message"];
+ }) => ReactNode;
+}): JSX.Element => (
+
+ {formatTime(log.date)}
+ {log.type}
+
+
+
+ {log.count && {log.count} }
+
+);
+
+const PlainTextMessage = ({
+ message,
+}: {
+ message: StreamingLog["message"];
+}) => {message as string} ;
+
+type Message = { message: StreamingLog["message"] };
+
+const AnyMessage = ({ message }: Message) => (
+ {JSON.stringify(message, null, " ")}
+);
+
+function tryParseCodeExecutionResult(output: string) {
+ try {
+ const json = JSON.parse(output);
+ return JSON.stringify(json, null, " ");
+ } catch (e) {
+ return output;
+ }
+}
+
+const RenderPart = ({ part }: { part: Part }) =>
+ part.text && part.text.length ? (
+ {part.text}
+ ) : part.executableCode ? (
+
+
executableCode: {part.executableCode.language}
+
+ {part.executableCode.code}
+
+
+ ) : part.codeExecutionResult ? (
+
+
codeExecutionResult: {part.codeExecutionResult.outcome}
+
+ {tryParseCodeExecutionResult(part.codeExecutionResult.output)}
+
+
+ ) : (
+
+
Inline Data: {part.inlineData?.mimeType}
+
+ );
+
+const ClientContentLog = ({ message }: Message) => {
+ const { turns, turnComplete } = (message as ClientContentMessage)
+ .clientContent;
+ return (
+
+
User
+ {turns.map((turn, i) => (
+
+ {turn.parts
+ .filter((part) => !(part.text && part.text === "\n"))
+ .map((part, j) => (
+
+ ))}
+
+ ))}
+ {!turnComplete ?
turnComplete: false : ""}
+
+ );
+};
+
+const ToolCallLog = ({ message }: Message) => {
+ const { toolCall } = message as ToolCallMessage;
+ return (
+
+ {toolCall.functionCalls.map((fc, i) => (
+
+
Function call: {fc.name}
+
+ {JSON.stringify(fc, null, " ")}
+
+
+ ))}
+
+ );
+};
+
+const ToolCallCancellationLog = ({ message }: Message): JSX.Element => (
+
+
+ {" "}
+ ids:{" "}
+ {(message as ToolCallCancellationMessage).toolCallCancellation.ids.map(
+ (id) => (
+
+ "{id}"
+
+ ),
+ )}
+
+
+);
+
+const ToolResponseLog = ({ message }: Message): JSX.Element => (
+
+ {(message as ToolResponseMessage).toolResponse.functionResponses.map(
+ (fc) => (
+
+
Function Response: {fc.id}
+
+ {JSON.stringify(fc.response, null, " ")}
+
+
+ ),
+ )}
+
+);
+
+const ModelTurnLog = ({ message }: Message): JSX.Element => {
+ const serverContent = (message as ServerContentMessage).serverContent;
+ const { modelTurn } = serverContent as ModelTurn;
+ const { parts } = modelTurn;
+
+ return (
+
+
Model
+ {parts
+ .filter((part) => !(part.text && part.text === "\n"))
+ .map((part, j) => (
+
+ ))}
+
+ );
+};
+
+const CustomPlainTextLog = (msg: string) => () => (
+
+);
+
+export type LoggerFilterType = "conversations" | "tools" | "none";
+
+export type LoggerProps = {
+ filter: LoggerFilterType;
+};
+
+const filters: Record boolean> = {
+ tools: (log: StreamingLog) =>
+ isToolCallMessage(log.message) ||
+ isToolResponseMessage(log.message) ||
+ isToolCallCancellationMessage(log.message),
+ conversations: (log: StreamingLog) =>
+ isClientContentMessage(log.message) || isServerContentMessage(log.message),
+ none: () => true,
+};
+
+const component = (log: StreamingLog) => {
+ if (typeof log.message === "string") {
+ return PlainTextMessage;
+ }
+ if (isClientContentMessage(log.message)) {
+ return ClientContentLog;
+ }
+ if (isToolCallMessage(log.message)) {
+ return ToolCallLog;
+ }
+ if (isToolCallCancellationMessage(log.message)) {
+ return ToolCallCancellationLog;
+ }
+ if (isToolResponseMessage(log.message)) {
+ return ToolResponseLog;
+ }
+ if (isServerContentMessage(log.message)) {
+ const { serverContent } = log.message;
+ if (isInterrupted(serverContent)) {
+ return CustomPlainTextLog("interrupted");
+ }
+ if (isTurnComplete(serverContent)) {
+ return CustomPlainTextLog("turnComplete");
+ }
+ if (isModelTurn(serverContent)) {
+ return ModelTurnLog;
+ }
+ }
+ return AnyMessage;
+};
+
+export default function Logger({ filter = "none" }: LoggerProps) {
+ const { logs } = useLoggerStore();
+
+ const filterFn = filters[filter];
+
+ return (
+
+
+ {logs.filter(filterFn).map((log, key) => {
+ return (
+
+ );
+ })}
+
+
+ );
+}
diff --git a/src/components/logger/logger.scss b/src/components/logger/logger.scss
new file mode 100644
index 0000000000000000000000000000000000000000..914d7502e46206cfceef90b1a4d88be2b274d922
--- /dev/null
+++ b/src/components/logger/logger.scss
@@ -0,0 +1,116 @@
+.logger {
+ color: var(--gray-300);
+ width: 100%;
+ max-width: 100%;
+ display: block;
+
+ .logger-list {
+ padding: 0 0px 0 25px;
+ overflow-x: hidden;
+ width: calc(100% - 45px);
+ }
+
+ .user h4 {
+ color: var(--Green-500);
+ }
+
+ .model h4 {
+ color: var(--Blue-500);
+ }
+
+ .rich-log {
+ display: flex;
+ justify-content: center;
+ gap: 4px;
+
+ pre {
+ overflow-x: auto;
+ }
+
+ display: block;
+
+ h4 {
+ font-size: 14px;
+ text-transform: uppercase;
+ padding: 8px 0;
+ margin: 0;
+ }
+
+ h5 {
+ margin: 0;
+ padding-bottom: 8px;
+ border-bottom: 1px solid var(--Neutral-20);
+ }
+
+ .part {
+ background: var(--Neutral-5);
+ padding: 14px;
+ margin-bottom: 4px;
+ color: var(--Neutral-90);
+ border-radius: 8px;
+ }
+ }
+
+ .plain-log {
+ &>* {
+ padding-right: 4px;
+ }
+ }
+
+ .inline-code:not(:last-child) {
+ font-style: italic;
+
+ &::after {
+ content: ", ";
+ }
+ }
+}
+
+.logger li {
+ display: block;
+ padding: 8px 0;
+ color: var(--Neutral-50, #707577);
+ font-family: "Space Mono";
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 400;
+ line-height: normal;
+}
+
+.logger li .timestamp {
+ width: 70px;
+ flex-grow: 0;
+ flex-shrink: 0;
+ color: var(--Neutral-50);
+}
+
+.logger li .source {
+ flex-shrink: 0;
+ font-weight: bold;
+}
+
+.logger li.source-server,
+.logger li.receive {
+ color: var(--Blue-500);
+}
+
+.logger li.source-client,
+.logger li.send:not(.source-server) {
+ color: var(--Green-500);
+}
+
+.logger li .count {
+ background-color: var(--Neutral-5);
+ font-size: x-small;
+ padding: 0em 0.6em;
+ padding: 0.3em 0.5em;
+ line-height: 1em;
+ vertical-align: middle;
+ border-radius: 8px;
+ color: var(--Blue-500);
+}
+
+.logger li .message {
+ flex-grow: 1;
+ color: var(--Neutral-50);
+}
diff --git a/src/components/logger/mock-logs.ts b/src/components/logger/mock-logs.ts
new file mode 100644
index 0000000000000000000000000000000000000000..06ac1056fa609b8f53b01d1104e536e46f97c7c4
--- /dev/null
+++ b/src/components/logger/mock-logs.ts
@@ -0,0 +1,151 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * this module is just mock data, intended to make it easier to develop and style the logger
+ */
+import type { StreamingLog } from "../../multimodal-live-types";
+
+const soundLogs = (n: number): StreamingLog[] =>
+ new Array(n).fill(0).map(
+ (): StreamingLog => ({
+ date: new Date(),
+ type: "server.audio",
+ message: "buffer (11250)",
+ }),
+ );
+//
+const realtimeLogs = (n: number): StreamingLog[] =>
+ new Array(n).fill(0).map(
+ (): StreamingLog => ({
+ date: new Date(),
+ type: "client.realtimeInput",
+ message: "audio",
+ }),
+ );
+
+export const mockLogs: StreamingLog[] = [
+ {
+ date: new Date(),
+ type: "client.open",
+ message: "connected to socket",
+ },
+ ...realtimeLogs(10),
+ ...soundLogs(10),
+ {
+ date: new Date(),
+ type: "receive.content",
+ message: {
+ serverContent: {
+ interrupted: true,
+ },
+ },
+ },
+ {
+ date: new Date(),
+ type: "receive.content",
+ message: {
+ serverContent: {
+ turnComplete: true,
+ },
+ },
+ },
+ //this one is just a string
+ // {
+ // date: new Date(),
+ // type: "server.send",
+ // message: {
+ // serverContent: {
+ // turnComplete: true,
+ // },
+ // },
+ // },
+ ...realtimeLogs(10),
+ ...soundLogs(20),
+ {
+ date: new Date(),
+ type: "receive.content",
+ message: {
+ serverContent: {
+ modelTurn: {
+ parts: [{ text: "Hey its text" }, { text: "more" }],
+ },
+ },
+ },
+ },
+ {
+ date: new Date(),
+ type: "client.send",
+ message: {
+ clientContent: {
+ turns: [
+ {
+ role: "User",
+ parts: [
+ {
+ text: "How much wood could a woodchuck chuck if a woodchuck could chuck wood",
+ },
+ ],
+ },
+ ],
+ turnComplete: true,
+ },
+ },
+ },
+ {
+ date: new Date(),
+ type: "server.toolCall",
+ message: {
+ toolCall: {
+ functionCalls: [
+ {
+ id: "akadjlasdfla-askls",
+ name: "take_photo",
+ args: {},
+ },
+ {
+ id: "akldjsjskldsj-102",
+ name: "move_camera",
+ args: { x: 20, y: 4 },
+ },
+ ],
+ },
+ },
+ },
+ {
+ date: new Date(),
+ type: "server.toolCallCancellation",
+ message: {
+ toolCallCancellation: {
+ ids: ["akladfjadslfk", "adkafsdljfsdk"],
+ },
+ },
+ },
+ {
+ date: new Date(),
+ type: "client.toolResponse",
+ message: {
+ toolResponse: {
+ functionResponses: [
+ {
+ response: { success: true },
+ id: "akslaj-10102",
+ },
+ ],
+ },
+ },
+ },
+];
diff --git a/src/components/side-panel/SidePanel.tsx b/src/components/side-panel/SidePanel.tsx
new file mode 100644
index 0000000000000000000000000000000000000000..7f5d4381eb1d498d5b23bfb0b3ba5fc9aa060e7c
--- /dev/null
+++ b/src/components/side-panel/SidePanel.tsx
@@ -0,0 +1,185 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import cn from "classnames";
+import { useEffect, useRef, useState } from "react";
+import { RiSidebarFoldLine, RiSidebarUnfoldLine } from "react-icons/ri";
+import Select from "react-select";
+import { useLiveAPIContext } from "../../contexts/LiveAPIContext";
+import { useLoggerStore } from "../../lib/store-logger";
+import Logger from "../logger/Logger";
+import type { LoggerFilterType } from "../logger/Logger";
+import "./side-panel.scss";
+
+const filterOptions = [
+ { value: "conversations", label: "Conversations" },
+ { value: "tools", label: "Tool Use" },
+ { value: "none", label: "All" },
+];
+
+export default function SidePanel() {
+ const { connected, client } = useLiveAPIContext();
+ const [open, setOpen] = useState(window.innerWidth >= 768);
+ const [isMobile, setIsMobile] = useState(window.innerWidth < 768);
+ const loggerRef = useRef(null);
+ const loggerLastHeightRef = useRef(-1);
+ const { log, logs } = useLoggerStore();
+
+ // Add effect to handle responsive behavior
+ useEffect(() => {
+ const handleResize = () => {
+ const mobileScreen = window.innerWidth < 768;
+ setIsMobile(mobileScreen);
+ setOpen(!mobileScreen);
+ };
+
+ // Initial check
+ handleResize();
+
+ // Add event listener for window resize
+ window.addEventListener('resize', handleResize);
+
+ // Cleanup
+ return () => window.removeEventListener('resize', handleResize);
+ }, []);
+
+ const [textInput, setTextInput] = useState("");
+ const [selectedOption, setSelectedOption] = useState<{
+ value: string;
+ label: string;
+ } | null>(null);
+ const inputRef = useRef(null);
+
+ //scroll the log to the bottom when new logs come in
+ useEffect(() => {
+ const el = loggerRef.current;
+ if (el) {
+ const scrollHeight = el.scrollHeight;
+ if (scrollHeight !== loggerLastHeightRef.current) {
+ el.scrollTop = scrollHeight;
+ loggerLastHeightRef.current = scrollHeight;
+ }
+ }
+ }, []);
+
+ // listen for log events and store them
+ useEffect(() => {
+ client.on("log", log);
+ return () => {
+ client.off("log", log);
+ };
+ }, [client, log]);
+
+ const handleSubmit = () => {
+ client.send([{ text: textInput }]);
+
+ setTextInput("");
+ if (inputRef.current) {
+ inputRef.current.innerText = "";
+ }
+ };
+
+ return (
+
+
+ Console
+ {open ? (
+ setOpen(false)}>
+
+
+ ) : (
+ setOpen(true)}>
+
+
+ )}
+
+
+ ({
+ ...baseStyles,
+ background: "var(--Neutral-15)",
+ color: "var(--Neutral-90)",
+ minHeight: "33px",
+ maxHeight: "33px",
+ border: 0,
+ }),
+ option: (styles, { isFocused, isSelected }) => ({
+ ...styles,
+ backgroundColor: isFocused
+ ? "var(--Neutral-30)"
+ : isSelected
+ ? "var(--Neutral-20)"
+ : undefined,
+ }),
+ }}
+ defaultValue={selectedOption}
+ options={filterOptions}
+ onChange={(e) => {
+ setSelectedOption(e);
+ }}
+ />
+
+ {connected
+ ? `🔵${open ? " Streaming" : ""}`
+ : `⏸️${open ? " Paused" : ""}`}
+
+
+
+
+
+
+
+ );
+}
diff --git a/src/components/side-panel/side-panel.scss b/src/components/side-panel/side-panel.scss
new file mode 100644
index 0000000000000000000000000000000000000000..74d3c441b88ae1111eddfeee7a95eb100539a12a
--- /dev/null
+++ b/src/components/side-panel/side-panel.scss
@@ -0,0 +1,309 @@
+.side-panel {
+ background: var(--Neutral-00);
+ width: 40px; /* when closed */
+ display: flex;
+ flex-direction: column;
+ height: 100vh;
+ transition: all 0.2s ease-in;
+ font-family: Arial, sans-serif;
+ border-right: 1px solid var(--gray-600);
+ color: var(--Neutral-90, #e1e2e3);
+ font-family: var(--font-family);
+ font-size: 13px;
+ font-style: normal;
+ font-weight: 400;
+ line-height: 160%;
+
+ .react-select {
+ background: var(--Neutral-20);
+ color: var(--Neutral-90);
+ width: 193px;
+ height: 30px;
+
+ .react-select__single-value {
+ color: var(--Neutral-90);
+ }
+
+ .react-select__menu {
+ background: var(--Neutral-20);
+ color: var(--Neutral-90);
+ }
+
+ .react-select__option:hover,
+ .react-select__option:focus,
+ .react-select_option:focus-within {
+ background: var(--Neutral-30);
+ }
+ }
+
+ .hidden {
+ display: none !important;
+ }
+
+ &.open {
+ width: 400px;
+ height: 100vh;
+
+ .top h2 {
+ left: 0%;
+ display: block;
+ opacity: 1;
+ }
+ }
+
+ .top {
+ display: flex;
+ width: calc(100% - 45px);
+ justify-content: space-between;
+ align-items: center;
+ padding: 12px 20px 12px 25px;
+ border-bottom: 1px solid var(--Neutral-20);
+
+ h2 {
+ position: relative;
+ color: var(--Neutral-90, #e1e2e3);
+ font-family: "Google Sans";
+ font-size: 21px;
+ font-style: normal;
+ font-weight: 500;
+ line-height: 16px; /* 100% */
+
+ opacity: 0;
+ display: none;
+ left: -100%;
+ transition:
+ opacity 0.2s ease-in,
+ left 0.2s ease-in,
+ display 0.2s ease-in;
+ transition-behavior: allow-discrete;
+
+ @starting-style {
+ left: 0%;
+ opacity: 1;
+ }
+ }
+ }
+
+ .opener {
+ height: 30px;
+ transition: transform 0.2s ease-in;
+ }
+
+ &:not(.open) {
+ .side-panel-container {
+ opacity: 0;
+ display: none;
+ transition: all 0.2s ease-in allow-discrete;
+ transition-delay: 0.1s;
+ }
+
+ .indicators .streaming-indicator {
+ width: 30px;
+ opacity: 0;
+ }
+
+ .opener {
+ transform: translate(-50%, 0);
+ }
+
+ .input-container {
+ opacity: 0;
+ display: none;
+ transition: all 0.2s ease-in allow-discrete;
+ }
+ }
+
+ .indicators {
+ display: flex;
+ padding: 24px 25px;
+ justify-content: flex-end;
+ gap: 21px;
+ .streaming-indicator {
+ user-select: none;
+ border-radius: 4px;
+ border: 1px solid var(--Neutral-20, #2a2f31);
+ background: var(--Neutral-10, #1c1f21);
+ display: flex;
+ width: 136px;
+ height: 30px;
+ padding-left: 4px;
+ justify-content: center;
+ align-items: center;
+ gap: 6px;
+ flex-shrink: 0;
+ text-align: center;
+ font-family: "Space Mono";
+ font-size: 14px;
+ font-style: normal;
+ font-weight: 400;
+ line-height: normal;
+ transition: width 0.2s ease-in;
+
+ &.connected {
+ color: var(--Blue-500, #0d9c53);
+ }
+ }
+ }
+
+ .side-panel-container {
+ align-self: flex-end;
+ width: 400px;
+ flex-grow: 1;
+ overflow-x: hidden;
+ overflow-y: auto;
+ /*scrollbar-gutter: stable both-edges;*/
+ }
+
+ .input-container {
+ height: 50px;
+ flex-grow: 0;
+ flex-shrink: 0;
+ border-top: 1px solid var(--Neutral-20);
+ padding: 14px 25px;
+ overflow: hidden;
+
+ .input-content {
+ position: relative;
+ background: var(--Neutral-10);
+ border: 1px solid var(--Neutral-15);
+ height: 22px;
+ border-radius: 10px;
+ padding: 11px 18px;
+
+ .send-button {
+ position: absolute;
+ top: 50%;
+ right: 0;
+ transform: translate(0, -50%);
+ background: none;
+ border: 0;
+ color: var(--Neutral-20);
+ cursor: pointer;
+ transition: color 0.1s ease-in;
+ z-index: 2;
+
+ &:hover {
+ color: var(--Neutral-60);
+ }
+ }
+
+ .input-area {
+ background: none;
+ color: var(--Neutral-90);
+ field-sizing: content;
+ position: absolute;
+ top: 0;
+ left: 0;
+ z-index: 2;
+ display: inline-block;
+ width: calc(100% - 72px);
+ max-height: 20px;
+ outline: none;
+ --webkit-box-flex: 1;
+ flex: 1;
+ word-break: break-word;
+ overflow: auto;
+ padding: 14px 18px;
+ border: 0;
+ resize: none;
+ }
+
+ .input-content-placeholder {
+ position: absolute;
+ left: 0;
+ top: 0;
+ display: flex;
+ align-items: center;
+ z-index: 1;
+ height: 100%;
+ width: 100%;
+ pointer-events: none;
+ user-select: none;
+ padding: 0px 18px;
+ white-space: pre-wrap;
+ }
+ }
+ }
+
+ @media screen and (max-width: 768px) {
+ position: absolute;
+ z-index: 1000;
+
+ &:not(.open) {
+ width: 100%;
+ height: 40px;
+ border-right: none;
+ border-bottom: 1px solid var(--gray-600);
+
+ .top {
+ width: 100%;
+ padding: 12px;
+ justify-content: flex-end;
+ border-bottom: none;
+
+ .opener {
+ transform: rotate(90deg);
+ position: absolute;
+ right: 12px;
+ top: 50%;
+ transform: translateY(-50%) rotate(90deg);
+ }
+ }
+
+ .indicators {
+ display: none;
+ }
+ }
+
+ &.open {
+ width: 100%;
+ max-width: 100%;
+ }
+
+ .side-panel-container {
+ width: 100%;
+ }
+
+ .react-select {
+ width: 140px;
+ }
+
+ .indicators {
+ padding: 24px 15px;
+
+ .streaming-indicator {
+ width: 110px;
+ }
+ }
+ }
+}
+
+.side-panel-responses,
+.side-panel-requests {
+ flex-grow: 1;
+ flex-shrink: 1;
+ overflow-x: hidden;
+ overflow-y: auto;
+ width: 100%;
+ display: block;
+ margin-left: 8px;
+}
+
+.top {
+ width: 100%;
+ flex-grow: 0;
+ flex-shrink: 0;
+ height: 30px;
+ display: flex;
+ align-self: flex-end;
+ align-items: center;
+ transition: all 0.2s ease-in;
+}
+.top button {
+ background: transparent;
+ border: 0;
+ cursor: pointer;
+ font-size: 1.25rem;
+ line-height: 1.75rem;
+ padding: 4px;
+}
diff --git a/src/contexts/LiveAPIContext.tsx b/src/contexts/LiveAPIContext.tsx
new file mode 100644
index 0000000000000000000000000000000000000000..a8de633fd51f59e702d6f54f52c34c4f39743540
--- /dev/null
+++ b/src/contexts/LiveAPIContext.tsx
@@ -0,0 +1,46 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createContext, FC, type ReactNode, useContext } from "react";
+import { useLiveAPI, type UseLiveAPIResults } from "../hooks/use-live-api";
+
+const LiveAPIContext = createContext(undefined);
+
+export type LiveAPIProviderProps = {
+ children: ReactNode;
+ url?: string;
+};
+
+export const LiveAPIProvider: FC = ({
+ url = `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`,
+ children,
+}) => {
+ const liveAPI = useLiveAPI({ url });
+
+ return (
+
+ {children}
+
+ );
+};
+
+export const useLiveAPIContext = () => {
+ const context = useContext(LiveAPIContext);
+ if (!context) {
+ throw new Error("useLiveAPIContext must be used within a LiveAPIProvider");
+ }
+ return context;
+};
diff --git a/src/hooks/use-live-api.ts b/src/hooks/use-live-api.ts
new file mode 100644
index 0000000000000000000000000000000000000000..278ca1210babe5803a6d2d98160c46159fc11eae
--- /dev/null
+++ b/src/hooks/use-live-api.ts
@@ -0,0 +1,116 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import {
+ MultimodalLiveAPIClientConnection,
+ MultimodalLiveClient,
+} from "../lib/multimodal-live-client";
+import { LiveConfig } from "../multimodal-live-types";
+import { AudioStreamer } from "../lib/audio-streamer";
+import { audioContext } from "../lib/utils";
+import VolMeterWorket from "../lib/worklets/vol-meter";
+
+export type UseLiveAPIResults = {
+ client: MultimodalLiveClient;
+ setConfig: (config: LiveConfig) => void;
+ config: LiveConfig;
+ connected: boolean;
+ connect: () => Promise;
+ disconnect: () => Promise;
+ volume: number;
+};
+
+export function useLiveAPI({
+ url,
+ apiKey,
+}: MultimodalLiveAPIClientConnection): UseLiveAPIResults {
+ const client = useMemo(
+ () => new MultimodalLiveClient({ url, apiKey }),
+ [url, apiKey],
+ );
+ const audioStreamerRef = useRef(null);
+
+ const [connected, setConnected] = useState(false);
+ const [config, setConfig] = useState({
+ model: "models/gemini-2.0-flash-exp",
+ });
+ const [volume, setVolume] = useState(0);
+
+ // register audio for streaming server -> speakers
+ useEffect(() => {
+ if (!audioStreamerRef.current) {
+ audioContext({ id: "audio-out" }).then((audioCtx: AudioContext) => {
+ audioStreamerRef.current = new AudioStreamer(audioCtx);
+ audioStreamerRef.current
+ .addWorklet("vumeter-out", VolMeterWorket, (ev: any) => {
+ setVolume(ev.data.volume);
+ })
+ .then(() => {
+ // Successfully added worklet
+ });
+ });
+ }
+ }, [audioStreamerRef]);
+
+ useEffect(() => {
+ const onClose = () => {
+ setConnected(false);
+ };
+
+ const stopAudioStreamer = () => audioStreamerRef.current?.stop();
+
+ const onAudio = (data: ArrayBuffer) =>
+ audioStreamerRef.current?.addPCM16(new Uint8Array(data));
+
+ client
+ .on("close", onClose)
+ .on("interrupted", stopAudioStreamer)
+ .on("audio", onAudio);
+
+ return () => {
+ client
+ .off("close", onClose)
+ .off("interrupted", stopAudioStreamer)
+ .off("audio", onAudio);
+ };
+ }, [client]);
+
+ const connect = useCallback(async () => {
+ console.log(config);
+ if (!config) {
+ throw new Error("config has not been set");
+ }
+ client.disconnect();
+ await client.connect(config);
+ setConnected(true);
+ }, [client, setConnected, config]);
+
+ const disconnect = useCallback(async () => {
+ client.disconnect();
+ setConnected(false);
+ }, [setConnected, client]);
+
+ return {
+ client,
+ config,
+ setConfig,
+ connected,
+ connect,
+ disconnect,
+ volume,
+ };
+}
diff --git a/src/hooks/use-media-stream-mux.ts b/src/hooks/use-media-stream-mux.ts
new file mode 100644
index 0000000000000000000000000000000000000000..084144575b79481e199021ce46cc71f8070f43a3
--- /dev/null
+++ b/src/hooks/use-media-stream-mux.ts
@@ -0,0 +1,23 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type UseMediaStreamResult = {
+ type: "webcam" | "screen";
+ start: () => Promise;
+ stop: () => void;
+ isStreaming: boolean;
+ stream: MediaStream | null;
+};
diff --git a/src/hooks/use-screen-capture.ts b/src/hooks/use-screen-capture.ts
new file mode 100644
index 0000000000000000000000000000000000000000..33e8576fdd9d909f597a06cb0f99d7d650a827b6
--- /dev/null
+++ b/src/hooks/use-screen-capture.ts
@@ -0,0 +1,77 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { useState, useEffect } from "react";
+import { UseMediaStreamResult } from "./use-media-stream-mux";
+
+export function useScreenCapture(): UseMediaStreamResult {
+ const [stream, setStream] = useState(null);
+ const [isStreaming, setIsStreaming] = useState(false);
+
+ useEffect(() => {
+ const handleStreamEnded = () => {
+ setIsStreaming(false);
+ setStream(null);
+ };
+ if (stream) {
+ stream
+ .getTracks()
+ .forEach((track) => track.addEventListener("ended", handleStreamEnded));
+ return () => {
+ stream
+ .getTracks()
+ .forEach((track) =>
+ track.removeEventListener("ended", handleStreamEnded),
+ );
+ };
+ }
+ }, [stream]);
+
+ const start = async () => {
+ try {
+ // const controller = new CaptureController();
+ // controller.setFocusBehavior("no-focus-change");
+ const mediaStream = await navigator.mediaDevices.getDisplayMedia({
+ video: true,
+ // controller
+ });
+ setStream(mediaStream);
+ setIsStreaming(true);
+ return mediaStream;
+ } catch (err) {
+ console.error('Failed to start screen capture:', err);
+ return null;
+ }
+ };
+
+ const stop = () => {
+ if (stream) {
+ stream.getTracks().forEach((track) => track.stop());
+ setStream(null);
+ setIsStreaming(false);
+ }
+ };
+
+ const result: UseMediaStreamResult = {
+ type: "screen",
+ start,
+ stop,
+ isStreaming,
+ stream,
+ };
+
+ return result;
+}
diff --git a/src/hooks/use-webcam.ts b/src/hooks/use-webcam.ts
new file mode 100644
index 0000000000000000000000000000000000000000..18472361f701f9b3c8d5129ac13cb4c3685dabc3
--- /dev/null
+++ b/src/hooks/use-webcam.ts
@@ -0,0 +1,123 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { useState, useEffect } from "react";
+import { UseMediaStreamResult } from "./use-media-stream-mux";
+
+export function useWebcam(): UseMediaStreamResult {
+ const [stream, setStream] = useState(null);
+ const [isStreaming, setIsStreaming] = useState(false);
+ const [availableCameras, setAvailableCameras] = useState([]);
+ const [currentCameraIndex, setCurrentCameraIndex] = useState(-1);
+
+ // Get list of available cameras on mount
+ useEffect(() => {
+ async function getCameras() {
+ try {
+ // First request permission to ensure we can enumerate video devices
+ await navigator.mediaDevices.getUserMedia({ video: true })
+ .then(stream => {
+ // Stop the stream immediately, we just needed permission
+ stream.getTracks().forEach(track => track.stop());
+ });
+
+ const devices = await navigator.mediaDevices.enumerateDevices();
+ const videoDevices = devices.filter(device => device.kind === 'videoinput');
+ setAvailableCameras(videoDevices);
+ console.log('Available cameras:', videoDevices);
+ } catch (err) {
+ console.error('Error getting cameras:', err);
+ }
+ }
+ getCameras();
+ }, []);
+
+ useEffect(() => {
+ const handleStreamEnded = () => {
+ setIsStreaming(false);
+ setStream(null);
+ };
+ if (stream) {
+ stream
+ .getTracks()
+ .forEach((track) => track.addEventListener("ended", handleStreamEnded));
+ return () => {
+ stream
+ .getTracks()
+ .forEach((track) =>
+ track.removeEventListener("ended", handleStreamEnded),
+ );
+ };
+ }
+ }, [stream]);
+
+ const start = async () => {
+ // If we're already streaming, cycle to next camera
+ if (isStreaming) {
+ const nextIndex = (currentCameraIndex + 1) % (availableCameras.length);
+ setCurrentCameraIndex(nextIndex);
+
+ // Stop current stream
+ if (stream) {
+ stream.getTracks().forEach((track) => track.stop());
+ }
+
+ // If we've cycled through all cameras, stop streaming
+ if (nextIndex === 0) {
+ setStream(null);
+ setIsStreaming(false);
+ return null;
+ }
+
+ const deviceId = availableCameras[nextIndex].deviceId;
+ const mediaStream = await navigator.mediaDevices.getUserMedia({
+ video: { deviceId: { exact: deviceId } }
+ });
+ setStream(mediaStream);
+ setIsStreaming(true);
+ return mediaStream;
+ } else {
+ // Start with first camera
+ setCurrentCameraIndex(0);
+ const deviceId = availableCameras[0]?.deviceId;
+ const mediaStream = await navigator.mediaDevices.getUserMedia({
+ video: deviceId ? { deviceId: { exact: deviceId } } : true
+ });
+ setStream(mediaStream);
+ setIsStreaming(true);
+ return mediaStream;
+ }
+ };
+
+ const stop = () => {
+ if (stream) {
+ stream.getTracks().forEach((track) => track.stop());
+ setStream(null);
+ setIsStreaming(false);
+ setCurrentCameraIndex(-1);
+ }
+ };
+
+ const result: UseMediaStreamResult = {
+ type: "webcam",
+ start,
+ stop,
+ isStreaming,
+ stream,
+ };
+
+ return result;
+}
diff --git a/src/index.css b/src/index.css
new file mode 100644
index 0000000000000000000000000000000000000000..ec2585e8c0bb8188184ed1e0703c4c8f2a8419b0
--- /dev/null
+++ b/src/index.css
@@ -0,0 +1,13 @@
+body {
+ margin: 0;
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
+ 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
+ sans-serif;
+ -webkit-font-smoothing: antialiased;
+ -moz-osx-font-smoothing: grayscale;
+}
+
+code {
+ font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
+ monospace;
+}
diff --git a/src/index.tsx b/src/index.tsx
new file mode 100644
index 0000000000000000000000000000000000000000..1239f5e9713e93af199de4517e52e3e89b85c3e1
--- /dev/null
+++ b/src/index.tsx
@@ -0,0 +1,35 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import './index.css';
+import App from './App';
+import reportWebVitals from './reportWebVitals';
+
+const root = ReactDOM.createRoot(
+ document.getElementById('root') as HTMLElement
+);
+root.render(
+
+
+
+);
+
+// If you want to start measuring performance in your app, pass a function
+// to log results (for example: reportWebVitals(console.log))
+// or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals
+reportWebVitals();
diff --git a/src/lib/audio-recorder.ts b/src/lib/audio-recorder.ts
new file mode 100644
index 0000000000000000000000000000000000000000..c4780f212a3131d9b74c03cc7d6a1590ee0e55ec
--- /dev/null
+++ b/src/lib/audio-recorder.ts
@@ -0,0 +1,417 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { audioContext } from "./utils";
+import AudioRecordingWorklet from "./worklets/audio-processing";
+import SafariAudioRecordingWorklet from "./worklets/safari-audio-processing";
+import VolMeterWorket from "./worklets/vol-meter";
+
+import { createWorketFromSrc } from "./audioworklet-registry";
+import EventEmitter from "eventemitter3";
+
+function arrayBufferToBase64(buffer: ArrayBuffer) {
+ var binary = "";
+ var bytes = new Uint8Array(buffer);
+ var len = bytes.byteLength;
+ for (var i = 0; i < len; i++) {
+ binary += String.fromCharCode(bytes[i]);
+ }
+ return window.btoa(binary);
+}
+
+// Add Safari-specific audio context creation
+async function createSafariAudioContext(sampleRate: number): Promise {
+ console.log('Creating Safari audio context with options:', { sampleRate });
+
+ // Safari requires webkit prefix
+ const AudioContextClass = (window as any).webkitAudioContext || window.AudioContext;
+ console.log('Using AudioContext class:', AudioContextClass.name);
+
+ const ctx = new AudioContextClass({
+ sampleRate,
+ latencyHint: 'interactive'
+ });
+
+ console.log('Safari AudioContext initial state:', {
+ state: ctx.state,
+ sampleRate: ctx.sampleRate,
+ baseLatency: ctx.baseLatency,
+ destination: ctx.destination,
+ });
+
+ // Safari requires user interaction to start audio context
+ if (ctx.state === 'suspended') {
+ console.log('Attempting to resume suspended Safari audio context...');
+ try {
+ await ctx.resume();
+ console.log('Successfully resumed Safari audio context:', ctx.state);
+ } catch (err) {
+ console.error('Failed to resume Safari audio context:', err);
+ throw err;
+ }
+ }
+
+ return ctx;
+}
+
+export class AudioRecorder extends EventEmitter {
+ stream: MediaStream | undefined;
+ audioContext: AudioContext | undefined;
+ source: MediaStreamAudioSourceNode | undefined;
+ recording: boolean = false;
+ recordingWorklet: AudioWorkletNode | undefined;
+ vuWorklet: AudioWorkletNode | undefined;
+
+ private starting: Promise | null = null;
+
+ // Add browser detection
+ isSafari: boolean;
+ isIOS: boolean;
+
+ constructor(public sampleRate = 16000) {
+ super();
+ this.isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
+ this.isIOS = /iPad|iPhone|iPod/.test(navigator.userAgent) && !(window as any).MSStream;
+ console.log('AudioRecorder initialized:', {
+ isSafari: this.isSafari,
+ isIOS: this.isIOS,
+ sampleRate: this.sampleRate,
+ userAgent: navigator.userAgent,
+ webAudioSupport: !!(window.AudioContext || (window as any).webkitAudioContext),
+ mediaDevicesSupport: !!navigator.mediaDevices
+ });
+ }
+
+ async start() {
+ if (!navigator.mediaDevices?.getUserMedia) {
+ console.error('MediaDevices API not available:', {
+ mediaDevices: !!navigator.mediaDevices,
+ getUserMedia: !!(navigator.mediaDevices && navigator.mediaDevices.getUserMedia)
+ });
+ throw new Error("Could not request user media");
+ }
+
+ console.log('Starting AudioRecorder with full environment info:', {
+ userAgent: navigator.userAgent,
+ platform: navigator.platform,
+ vendor: navigator.vendor,
+ audioWorkletSupport: !!(window.AudioWorklet),
+ sampleRate: this.sampleRate,
+ existingAudioContext: !!this.audioContext,
+ existingStream: !!this.stream,
+ isSafari: this.isSafari
+ });
+
+ this.starting = new Promise(async (resolve, reject) => {
+ try {
+ if (this.isSafari) {
+ // Safari implementation
+ console.log('Safari detected - using Safari-specific audio initialization');
+
+ // 1. First get audio permissions
+ console.log('Requesting audio permissions first for Safari...');
+ const constraints = {
+ audio: {
+ echoCancellation: false,
+ noiseSuppression: false,
+ autoGainControl: false,
+ sampleRate: this.sampleRate,
+ channelCount: 1
+ }
+ };
+ console.log('Safari audio constraints:', constraints);
+
+ try {
+ this.stream = await navigator.mediaDevices.getUserMedia(constraints);
+ const track = this.stream.getAudioTracks()[0];
+ console.log('Safari audio permissions granted:', {
+ track: track.label,
+ settings: track.getSettings(),
+ constraints: track.getConstraints(),
+ enabled: track.enabled,
+ muted: track.muted,
+ readyState: track.readyState
+ });
+ } catch (err) {
+ console.error('Failed to get Safari audio permissions:', err);
+ throw err;
+ }
+
+ // 2. Create and initialize audio context
+ try {
+ this.audioContext = await createSafariAudioContext(this.sampleRate);
+ console.log('Safari audio context ready:', {
+ state: this.audioContext.state,
+ currentTime: this.audioContext.currentTime
+ });
+ } catch (err) {
+ console.error('Failed to initialize Safari audio context:', err);
+ throw err;
+ }
+
+ // 3. Create and connect audio source
+ try {
+ console.log('Creating Safari audio source...');
+ this.source = this.audioContext.createMediaStreamSource(this.stream);
+ console.log('Safari audio source created successfully:', {
+ numberOfInputs: this.source.numberOfInputs,
+ numberOfOutputs: this.source.numberOfOutputs,
+ channelCount: this.source.channelCount
+ });
+ } catch (err) {
+ console.error('Failed to create Safari audio source:', err);
+ throw err;
+ }
+
+ // 4. Load and create worklet
+ try {
+ const workletName = "audio-recorder-worklet";
+ console.log('Loading Safari audio worklet...');
+ const src = createWorketFromSrc(workletName, SafariAudioRecordingWorklet);
+ await this.audioContext.audioWorklet.addModule(src);
+ console.log('Safari audio worklet module loaded');
+
+ this.recordingWorklet = new AudioWorkletNode(
+ this.audioContext,
+ workletName,
+ {
+ numberOfInputs: 1,
+ numberOfOutputs: 1,
+ channelCount: 1,
+ processorOptions: {
+ sampleRate: this.sampleRate
+ }
+ }
+ );
+
+ // Add detailed error handlers
+ this.recordingWorklet.onprocessorerror = (event) => {
+ console.error('Safari AudioWorklet processor error:', event);
+ };
+
+ this.recordingWorklet.port.onmessageerror = (event) => {
+ console.error('Safari AudioWorklet message error:', event);
+ };
+
+ // Add data handler with detailed logging
+ this.recordingWorklet.port.onmessage = (ev: MessageEvent) => {
+ const data = ev.data.data;
+ console.log('Safari AudioWorklet message received:', {
+ eventType: ev.data.event,
+ hasData: !!data,
+ dataType: data ? typeof data : null,
+ timestamp: Date.now()
+ });
+
+ if (data?.int16arrayBuffer) {
+ console.log('Processing Safari audio chunk:', {
+ byteLength: data.int16arrayBuffer.byteLength,
+ timestamp: Date.now()
+ });
+ const arrayBufferString = arrayBufferToBase64(data.int16arrayBuffer);
+ this.emit("data", arrayBufferString);
+ } else {
+ console.warn('Invalid Safari audio chunk received:', ev.data);
+ }
+ };
+
+ console.log('Safari AudioWorkletNode created successfully');
+ } catch (err) {
+ console.error('Failed to setup Safari audio worklet:', err);
+ throw err;
+ }
+
+ // 5. Connect nodes
+ try {
+ console.log('Connecting Safari audio nodes...');
+ this.source.connect(this.recordingWorklet);
+ console.log('Safari audio nodes connected successfully');
+ } catch (err) {
+ console.error('Failed to connect Safari audio nodes:', err);
+ throw err;
+ }
+
+ } else {
+ // Chrome/other browsers implementation
+ console.log('Non-Safari browser detected - using standard audio initialization');
+
+ // Get media stream first for Chrome
+ const constraints = {
+ audio: {
+ echoCancellation: true,
+ noiseSuppression: true,
+ autoGainControl: true,
+ sampleRate: this.sampleRate
+ }
+ };
+ console.log('Chrome audio constraints:', constraints);
+
+ try {
+ this.stream = await navigator.mediaDevices.getUserMedia(constraints);
+ const track = this.stream.getAudioTracks()[0];
+ console.log('Chrome audio permissions granted:', {
+ track: track.label,
+ settings: track.getSettings()
+ });
+ } catch (err) {
+ console.error('Failed to get Chrome audio permissions:', err);
+ throw err;
+ }
+
+ // Create audio context after getting stream for Chrome
+ try {
+ console.log('Creating Chrome audio context...');
+ this.audioContext = await audioContext({ sampleRate: this.sampleRate });
+ console.log('Chrome audio context created:', {
+ state: this.audioContext.state,
+ sampleRate: this.audioContext.sampleRate
+ });
+ } catch (err) {
+ console.error('Failed to create Chrome audio context:', err);
+ throw err;
+ }
+
+ // Create media stream source
+ try {
+ console.log('Creating Chrome audio source...');
+ this.source = this.audioContext.createMediaStreamSource(this.stream);
+ console.log('Chrome audio source created');
+ } catch (err) {
+ console.error('Failed to create Chrome audio source:', err);
+ throw err;
+ }
+
+ // Load and create standard worklet
+ try {
+ const workletName = "audio-recorder-worklet";
+ console.log('Loading Chrome audio worklet...');
+ const src = createWorketFromSrc(workletName, AudioRecordingWorklet);
+ await this.audioContext.audioWorklet.addModule(src);
+ console.log('Chrome audio worklet loaded');
+
+ this.recordingWorklet = new AudioWorkletNode(
+ this.audioContext,
+ workletName,
+ {
+ numberOfInputs: 1,
+ numberOfOutputs: 1,
+ channelCount: 1,
+ processorOptions: {
+ sampleRate: this.sampleRate
+ }
+ }
+ );
+
+ // Add error handlers
+ this.recordingWorklet.onprocessorerror = (event) => {
+ console.error('Chrome AudioWorklet processor error:', event);
+ };
+
+ this.recordingWorklet.port.onmessageerror = (event) => {
+ console.error('Chrome AudioWorklet message error:', event);
+ };
+
+ // Add data handler
+ this.recordingWorklet.port.onmessage = async (ev: MessageEvent) => {
+ const arrayBuffer = ev.data.data?.int16arrayBuffer;
+ if (arrayBuffer) {
+ const arrayBufferString = arrayBufferToBase64(arrayBuffer);
+ this.emit("data", arrayBufferString);
+ } else {
+ console.warn('Invalid Chrome audio chunk received:', ev.data);
+ }
+ };
+
+ console.log('Chrome AudioWorkletNode created');
+ } catch (err) {
+ console.error('Failed to setup Chrome audio worklet:', err);
+ throw err;
+ }
+
+ // Connect nodes
+ try {
+ console.log('Connecting Chrome audio nodes...');
+ this.source.connect(this.recordingWorklet);
+ console.log('Chrome audio nodes connected');
+
+ // Set up VU meter
+ const vuWorkletName = "vu-meter";
+ await this.audioContext.audioWorklet.addModule(
+ createWorketFromSrc(vuWorkletName, VolMeterWorket),
+ );
+ this.vuWorklet = new AudioWorkletNode(this.audioContext, vuWorkletName);
+ this.vuWorklet.port.onmessage = (ev: MessageEvent) => {
+ this.emit("volume", ev.data.volume);
+ };
+ this.source.connect(this.vuWorklet);
+ console.log('Chrome VU meter connected');
+ } catch (err) {
+ console.error('Failed to connect Chrome audio nodes:', err);
+ throw err;
+ }
+ }
+
+ this.recording = true;
+ console.log('Recording started successfully');
+ resolve();
+ this.starting = null;
+ } catch (error) {
+ console.error('Failed to start recording:', error);
+ this.stop();
+ reject(error);
+ this.starting = null;
+ }
+ });
+ return this.starting;
+ }
+
+ stop() {
+ console.log('Stopping audio recorder...');
+ // its plausible that stop would be called before start completes
+ // such as if the websocket immediately hangs up
+ const handleStop = () => {
+ try {
+ if (this.source) {
+ console.log('Disconnecting audio source...');
+ this.source.disconnect();
+ }
+ if (this.stream) {
+ console.log('Stopping media stream tracks...');
+ this.stream.getTracks().forEach(track => {
+ track.stop();
+ console.log('Stopped track:', track.label);
+ });
+ }
+ if (this.audioContext && this.isSafari) {
+ console.log('Closing Safari audio context...');
+ this.audioContext.close();
+ }
+ this.stream = undefined;
+ this.recordingWorklet = undefined;
+ this.vuWorklet = undefined;
+ console.log('Audio recorder stopped successfully');
+ } catch (err) {
+ console.error('Error while stopping audio recorder:', err);
+ }
+ };
+ if (this.starting) {
+ console.log('Stop called while starting - waiting for start to complete...');
+ this.starting.then(handleStop);
+ return;
+ }
+ handleStop();
+ }
+}
diff --git a/src/lib/audio-streamer.ts b/src/lib/audio-streamer.ts
new file mode 100644
index 0000000000000000000000000000000000000000..b89de83a1fb9cb9dfcbc2a54ffcabf35719cd67d
--- /dev/null
+++ b/src/lib/audio-streamer.ts
@@ -0,0 +1,270 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+ createWorketFromSrc,
+ registeredWorklets,
+} from "./audioworklet-registry";
+
+export class AudioStreamer {
+ public audioQueue: Float32Array[] = [];
+ private isPlaying: boolean = false;
+ private sampleRate: number = 24000;
+ private bufferSize: number = 7680;
+ private processingBuffer: Float32Array = new Float32Array(0);
+ private scheduledTime: number = 0;
+ public gainNode: GainNode;
+ public source: AudioBufferSourceNode;
+ private isStreamComplete: boolean = false;
+ private checkInterval: number | null = null;
+ private initialBufferTime: number = 0.1; //0.1 // 100ms initial buffer
+ private endOfQueueAudioSource: AudioBufferSourceNode | null = null;
+
+ public onComplete = () => {};
+
+ constructor(public context: AudioContext) {
+ this.gainNode = this.context.createGain();
+ this.source = this.context.createBufferSource();
+ this.gainNode.connect(this.context.destination);
+ this.addPCM16 = this.addPCM16.bind(this);
+ }
+
+ async addWorklet void>(
+ workletName: string,
+ workletSrc: string,
+ handler: T,
+ ): Promise {
+ let workletsRecord = registeredWorklets.get(this.context);
+ if (workletsRecord && workletsRecord[workletName]) {
+ // the worklet already exists on this context
+ // add the new handler to it
+ workletsRecord[workletName].handlers.push(handler);
+ return Promise.resolve(this);
+ //throw new Error(`Worklet ${workletName} already exists on context`);
+ }
+
+ if (!workletsRecord) {
+ registeredWorklets.set(this.context, {});
+ workletsRecord = registeredWorklets.get(this.context)!;
+ }
+
+ // create new record to fill in as becomes available
+ workletsRecord[workletName] = { handlers: [handler] };
+
+ const src = createWorketFromSrc(workletName, workletSrc);
+ await this.context.audioWorklet.addModule(src);
+ const worklet = new AudioWorkletNode(this.context, workletName);
+
+ //add the node into the map
+ workletsRecord[workletName].node = worklet;
+
+ return this;
+ }
+
+ addPCM16(chunk: Uint8Array) {
+ const float32Array = new Float32Array(chunk.length / 2);
+ const dataView = new DataView(chunk.buffer);
+
+ for (let i = 0; i < chunk.length / 2; i++) {
+ try {
+ const int16 = dataView.getInt16(i * 2, true);
+ float32Array[i] = int16 / 32768;
+ } catch (e) {
+ console.error(e);
+ // console.log(
+ // `dataView.length: ${dataView.byteLength}, i * 2: ${i * 2}`,
+ // );
+ }
+ }
+
+ const newBuffer = new Float32Array(
+ this.processingBuffer.length + float32Array.length,
+ );
+ newBuffer.set(this.processingBuffer);
+ newBuffer.set(float32Array, this.processingBuffer.length);
+ this.processingBuffer = newBuffer;
+
+ while (this.processingBuffer.length >= this.bufferSize) {
+ const buffer = this.processingBuffer.slice(0, this.bufferSize);
+ this.audioQueue.push(buffer);
+ this.processingBuffer = this.processingBuffer.slice(this.bufferSize);
+ }
+
+ if (!this.isPlaying) {
+ this.isPlaying = true;
+ // Initialize scheduledTime only when we start playing
+ this.scheduledTime = this.context.currentTime + this.initialBufferTime;
+ this.scheduleNextBuffer();
+ }
+ }
+
+ private createAudioBuffer(audioData: Float32Array): AudioBuffer {
+ const audioBuffer = this.context.createBuffer(
+ 1,
+ audioData.length,
+ this.sampleRate,
+ );
+ audioBuffer.getChannelData(0).set(audioData);
+ return audioBuffer;
+ }
+
+ private scheduleNextBuffer() {
+ const SCHEDULE_AHEAD_TIME = 0.2;
+
+ while (
+ this.audioQueue.length > 0 &&
+ this.scheduledTime < this.context.currentTime + SCHEDULE_AHEAD_TIME
+ ) {
+ const audioData = this.audioQueue.shift()!;
+ const audioBuffer = this.createAudioBuffer(audioData);
+ const source = this.context.createBufferSource();
+
+ if (this.audioQueue.length === 0) {
+ if (this.endOfQueueAudioSource) {
+ this.endOfQueueAudioSource.onended = null;
+ }
+ this.endOfQueueAudioSource = source;
+ source.onended = () => {
+ if (
+ !this.audioQueue.length &&
+ this.endOfQueueAudioSource === source
+ ) {
+ this.endOfQueueAudioSource = null;
+ this.onComplete();
+ }
+ };
+ }
+
+ source.buffer = audioBuffer;
+ source.connect(this.gainNode);
+
+ const worklets = registeredWorklets.get(this.context);
+
+ if (worklets) {
+ Object.entries(worklets).forEach(([workletName, graph]) => {
+ const { node, handlers } = graph;
+ if (node) {
+ source.connect(node);
+ node.port.onmessage = function (ev: MessageEvent) {
+ handlers.forEach((handler) => {
+ handler.call(node.port, ev);
+ });
+ };
+ node.connect(this.context.destination);
+ }
+ });
+ }
+
+ // i added this trying to fix clicks
+ // this.gainNode.gain.setValueAtTime(0, 0);
+ // this.gainNode.gain.linearRampToValueAtTime(1, 1);
+
+ // Ensure we never schedule in the past
+ const startTime = Math.max(this.scheduledTime, this.context.currentTime);
+ source.start(startTime);
+
+ this.scheduledTime = startTime + audioBuffer.duration;
+ }
+
+ if (this.audioQueue.length === 0 && this.processingBuffer.length === 0) {
+ if (this.isStreamComplete) {
+ this.isPlaying = false;
+ if (this.checkInterval) {
+ clearInterval(this.checkInterval);
+ this.checkInterval = null;
+ }
+ } else {
+ if (!this.checkInterval) {
+ this.checkInterval = window.setInterval(() => {
+ if (
+ this.audioQueue.length > 0 ||
+ this.processingBuffer.length >= this.bufferSize
+ ) {
+ this.scheduleNextBuffer();
+ }
+ }, 100) as unknown as number;
+ }
+ }
+ } else {
+ const nextCheckTime =
+ (this.scheduledTime - this.context.currentTime) * 1000;
+ setTimeout(
+ () => this.scheduleNextBuffer(),
+ Math.max(0, nextCheckTime - 50),
+ );
+ }
+ }
+
+ stop() {
+ this.isPlaying = false;
+ this.isStreamComplete = true;
+ this.audioQueue = [];
+ this.processingBuffer = new Float32Array(0);
+ this.scheduledTime = this.context.currentTime;
+
+ if (this.checkInterval) {
+ clearInterval(this.checkInterval);
+ this.checkInterval = null;
+ }
+
+ this.gainNode.gain.linearRampToValueAtTime(
+ 0,
+ this.context.currentTime + 0.1,
+ );
+
+ setTimeout(() => {
+ this.gainNode.disconnect();
+ this.gainNode = this.context.createGain();
+ this.gainNode.connect(this.context.destination);
+ }, 200);
+ }
+
+ async resume() {
+ if (this.context.state === "suspended") {
+ await this.context.resume();
+ }
+ this.isStreamComplete = false;
+ this.scheduledTime = this.context.currentTime + this.initialBufferTime;
+ this.gainNode.gain.setValueAtTime(1, this.context.currentTime);
+ }
+
+ complete() {
+ this.isStreamComplete = true;
+ if (this.processingBuffer.length > 0) {
+ this.audioQueue.push(this.processingBuffer);
+ this.processingBuffer = new Float32Array(0);
+ if (this.isPlaying) {
+ this.scheduleNextBuffer();
+ }
+ } else {
+ this.onComplete();
+ }
+ }
+}
+
+// // Usage example:
+// const audioStreamer = new AudioStreamer();
+//
+// // In your streaming code:
+// function handleChunk(chunk: Uint8Array) {
+// audioStreamer.handleChunk(chunk);
+// }
+//
+// // To start playing (call this in response to a user interaction)
+// await audioStreamer.resume();
+//
+// // To stop playing
+// // audioStreamer.stop();
diff --git a/src/lib/audioworklet-registry.ts b/src/lib/audioworklet-registry.ts
new file mode 100644
index 0000000000000000000000000000000000000000..49471d30c0d74dea774e049c3864ade0c57d7bd1
--- /dev/null
+++ b/src/lib/audioworklet-registry.ts
@@ -0,0 +1,43 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A registry to map attached worklets by their audio-context
+ * any module using `audioContext.audioWorklet.addModule(` should register the worklet here
+ */
+export type WorkletGraph = {
+ node?: AudioWorkletNode;
+ handlers: Array<(this: MessagePort, ev: MessageEvent) => any>;
+};
+
+export const registeredWorklets: Map<
+ AudioContext,
+ Record
+> = new Map();
+
+export const createWorketFromSrc = (
+ workletName: string,
+ workletSrc: string,
+) => {
+ const script = new Blob(
+ [`registerProcessor("${workletName}", ${workletSrc})`],
+ {
+ type: "application/javascript",
+ },
+ );
+
+ return URL.createObjectURL(script);
+};
diff --git a/src/lib/multimodal-live-client.ts b/src/lib/multimodal-live-client.ts
new file mode 100644
index 0000000000000000000000000000000000000000..3651ce8afa6761c755cf9f3ba9ffea8b5858e609
--- /dev/null
+++ b/src/lib/multimodal-live-client.ts
@@ -0,0 +1,313 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Content, GenerativeContentBlob, Part } from "@google/generative-ai";
+import { EventEmitter } from "eventemitter3";
+import { difference } from "lodash";
+import {
+ ClientContentMessage,
+ isInterrupted,
+ isModelTurn,
+ isServerContentMessage,
+ isSetupCompleteMessage,
+ isToolCallCancellationMessage,
+ isToolCallMessage,
+ isTurnComplete,
+ LiveIncomingMessage,
+ ModelTurn,
+ RealtimeInputMessage,
+ ServerContent,
+ SetupMessage,
+ StreamingLog,
+ ToolCall,
+ ToolCallCancellation,
+ ToolResponseMessage,
+ type LiveConfig,
+} from "../multimodal-live-types";
+import { blobToJSON, base64ToArrayBuffer } from "./utils";
+
+/**
+ * the events that this client will emit
+ */
+interface MultimodalLiveClientEventTypes {
+ open: () => void;
+ log: (log: StreamingLog) => void;
+ close: (event: CloseEvent) => void;
+ audio: (data: ArrayBuffer) => void;
+ content: (data: ServerContent) => void;
+ interrupted: () => void;
+ setupcomplete: () => void;
+ turncomplete: () => void;
+ toolcall: (toolCall: ToolCall) => void;
+ toolcallcancellation: (toolcallCancellation: ToolCallCancellation) => void;
+}
+
+export type MultimodalLiveAPIClientConnection = {
+ url?: string;
+ apiKey?: string;
+};
+
+/**
+ * A event-emitting class that manages the connection to the websocket and emits
+ * events to the rest of the application.
+ * If you dont want to use react you can still use this.
+ */
+export class MultimodalLiveClient extends EventEmitter {
+ public ws: WebSocket | null = null;
+ protected config: LiveConfig | null = null;
+ public url: string;
+
+ constructor({ url, apiKey }: MultimodalLiveAPIClientConnection = {}) {
+ super();
+ this.url = url || `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`;
+ this.send = this.send.bind(this);
+ }
+
+ log(type: string, message: StreamingLog["message"]) {
+ const log: StreamingLog = {
+ date: new Date(),
+ type,
+ message,
+ };
+ this.emit("log", log);
+ }
+
+ connect(config: LiveConfig): Promise {
+ this.config = config;
+
+ const ws = new WebSocket(this.url);
+
+ ws.addEventListener("message", async (evt: MessageEvent) => {
+ if (evt.data instanceof Blob) {
+ this.receive(evt.data);
+ } else {
+ console.log("non blob message", evt);
+ }
+ });
+ return new Promise((resolve, reject) => {
+ const onError = (ev: Event) => {
+ this.disconnect(ws);
+ const message = `Could not connect to "${this.url}"`;
+ this.log(`server.${ev.type}`, message);
+ reject(new Error(message));
+ };
+ ws.addEventListener("error", onError);
+ ws.addEventListener("open", (ev: Event) => {
+ if (!this.config) {
+ reject("Invalid config sent to `connect(config)`");
+ return;
+ }
+ this.log(`client.${ev.type}`, `connected to socket`);
+ this.emit("open");
+
+ this.ws = ws;
+
+ const setupMessage: SetupMessage = {
+ setup: this.config,
+ };
+ this._sendDirect(setupMessage);
+ this.log("client.send", "setup");
+
+ ws.removeEventListener("error", onError);
+ ws.addEventListener("close", (ev: CloseEvent) => {
+ console.log(ev);
+ this.disconnect(ws);
+ let reason = ev.reason || "";
+ if (reason.toLowerCase().includes("error")) {
+ const prelude = "ERROR]";
+ const preludeIndex = reason.indexOf(prelude);
+ if (preludeIndex > 0) {
+ reason = reason.slice(
+ preludeIndex + prelude.length + 1,
+ Infinity,
+ );
+ }
+ }
+ this.log(
+ `server.${ev.type}`,
+ `disconnected ${reason ? `with reason: ${reason}` : ``}`,
+ );
+ this.emit("close", ev);
+ });
+ resolve(true);
+ });
+ });
+ }
+
+ disconnect(ws?: WebSocket) {
+ // could be that this is an old websocket and theres already a new instance
+ // only close it if its still the correct reference
+ if ((!ws || this.ws === ws) && this.ws) {
+ this.ws.close();
+ this.ws = null;
+ this.log("client.close", `Disconnected`);
+ return true;
+ }
+ return false;
+ }
+
+ protected async receive(blob: Blob) {
+ const response: LiveIncomingMessage = (await blobToJSON(
+ blob,
+ )) as LiveIncomingMessage;
+ if (isToolCallMessage(response)) {
+ this.log("server.toolCall", response);
+ this.emit("toolcall", response.toolCall);
+ return;
+ }
+ if (isToolCallCancellationMessage(response)) {
+ this.log("receive.toolCallCancellation", response);
+ this.emit("toolcallcancellation", response.toolCallCancellation);
+ return;
+ }
+
+ if (isSetupCompleteMessage(response)) {
+ this.log("server.send", "setupComplete");
+ this.emit("setupcomplete");
+ return;
+ }
+
+ // this json also might be `contentUpdate { interrupted: true }`
+ // or contentUpdate { end_of_turn: true }
+ if (isServerContentMessage(response)) {
+ const { serverContent } = response;
+ if (isInterrupted(serverContent)) {
+ this.log("receive.serverContent", "interrupted");
+ this.emit("interrupted");
+ return;
+ }
+ if (isTurnComplete(serverContent)) {
+ this.log("server.send", "turnComplete");
+ this.emit("turncomplete");
+ //plausible theres more to the message, continue
+ }
+
+ if (isModelTurn(serverContent)) {
+ let parts: Part[] = serverContent.modelTurn.parts;
+
+ // when its audio that is returned for modelTurn
+ const audioParts = parts.filter(
+ (p) => p.inlineData && p.inlineData.mimeType.startsWith("audio/pcm"),
+ );
+ const base64s = audioParts.map((p) => p.inlineData?.data);
+
+ // strip the audio parts out of the modelTurn
+ const otherParts = difference(parts, audioParts);
+ // console.log("otherParts", otherParts);
+
+ base64s.forEach((b64) => {
+ if (b64) {
+ const data = base64ToArrayBuffer(b64);
+ this.emit("audio", data);
+ this.log(`server.audio`, `buffer (${data.byteLength})`);
+ }
+ });
+ if (!otherParts.length) {
+ return;
+ }
+
+ parts = otherParts;
+
+ const content: ModelTurn = { modelTurn: { parts } };
+ this.emit("content", content);
+ this.log(`server.content`, response);
+ }
+ } else {
+ console.log("received unmatched message", response);
+ }
+ }
+
+ /**
+ * send realtimeInput, this is base64 chunks of "audio/pcm" and/or "image/jpg"
+ */
+ sendRealtimeInput(chunks: GenerativeContentBlob[]) {
+ let hasAudio = false;
+ let hasVideo = false;
+ for (let i = 0; i < chunks.length; i++) {
+ const ch = chunks[i];
+ if (ch.mimeType.includes("audio")) {
+ hasAudio = true;
+ }
+ if (ch.mimeType.includes("image")) {
+ hasVideo = true;
+ }
+ if (hasAudio && hasVideo) {
+ break;
+ }
+ }
+ const message =
+ hasAudio && hasVideo
+ ? "audio + video"
+ : hasAudio
+ ? "audio"
+ : hasVideo
+ ? "video"
+ : "unknown";
+
+ const data: RealtimeInputMessage = {
+ realtimeInput: {
+ mediaChunks: chunks,
+ },
+ };
+ this._sendDirect(data);
+ this.log(`client.realtimeInput`, message);
+ }
+
+ /**
+ * send a response to a function call and provide the id of the functions you are responding to
+ */
+ sendToolResponse(toolResponse: ToolResponseMessage["toolResponse"]) {
+ const message: ToolResponseMessage = {
+ toolResponse,
+ };
+
+ this._sendDirect(message);
+ this.log(`client.toolResponse`, message);
+ }
+
+ /**
+ * send normal content parts such as { text }
+ */
+ send(parts: Part | Part[], turnComplete: boolean = true) {
+ parts = Array.isArray(parts) ? parts : [parts];
+ const content: Content = {
+ role: "user",
+ parts,
+ };
+
+ const clientContentRequest: ClientContentMessage = {
+ clientContent: {
+ turns: [content],
+ turnComplete,
+ },
+ };
+
+ this._sendDirect(clientContentRequest);
+ this.log(`client.send`, clientContentRequest);
+ }
+
+ /**
+ * used internally to send all messages
+ * don't use directly unless trying to send an unsupported message type
+ */
+ _sendDirect(request: object) {
+ if (!this.ws) {
+ throw new Error("WebSocket is not connected");
+ }
+ const str = JSON.stringify(request);
+ this.ws.send(str);
+ }
+}
diff --git a/src/lib/platform.ts b/src/lib/platform.ts
new file mode 100644
index 0000000000000000000000000000000000000000..a637ce00160adb15f57490f5766d33446529b264
--- /dev/null
+++ b/src/lib/platform.ts
@@ -0,0 +1,6 @@
+export const isIOS = (): boolean => {
+ const userAgent = window.navigator.userAgent.toLowerCase();
+ return /iphone|ipad|ipod/.test(userAgent) ||
+ // Detect iPad on iOS 13+ (which reports as Mac)
+ (userAgent.includes('mac') && 'ontouchend' in document);
+};
\ No newline at end of file
diff --git a/src/lib/store-logger.ts b/src/lib/store-logger.ts
new file mode 100644
index 0000000000000000000000000000000000000000..9e52fbb52628f6fee3931909c00cfb21efeec88f
--- /dev/null
+++ b/src/lib/store-logger.ts
@@ -0,0 +1,65 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { create } from "zustand";
+import { StreamingLog } from "../multimodal-live-types";
+import { mockLogs } from "../components/logger/mock-logs";
+
+interface StoreLoggerState {
+ maxLogs: number;
+ logs: StreamingLog[];
+ log: (streamingLog: StreamingLog) => void;
+ clearLogs: () => void;
+}
+
+export const useLoggerStore = create((set, get) => ({
+ maxLogs: 500,
+ logs: [], //mockLogs,
+ log: ({ date, type, message }: StreamingLog) => {
+ set((state) => {
+ const prevLog = state.logs.at(-1);
+ if (prevLog && prevLog.type === type && prevLog.message === message) {
+ return {
+ logs: [
+ ...state.logs.slice(0, -1),
+ {
+ date,
+ type,
+ message,
+ count: prevLog.count ? prevLog.count + 1 : 1,
+ } as StreamingLog,
+ ],
+ };
+ }
+ return {
+ logs: [
+ ...state.logs.slice(-(get().maxLogs - 1)),
+ {
+ date,
+ type,
+ message,
+ } as StreamingLog,
+ ],
+ };
+ });
+ },
+
+ clearLogs: () => {
+ console.log("clear log");
+ set({ logs: [] });
+ },
+ setMaxLogs: (n: number) => set({ maxLogs: n }),
+}));
diff --git a/src/lib/utils.ts b/src/lib/utils.ts
new file mode 100644
index 0000000000000000000000000000000000000000..da140b5fda1b7dd36e25605f03b56248c08e6896
--- /dev/null
+++ b/src/lib/utils.ts
@@ -0,0 +1,86 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type GetAudioContextOptions = AudioContextOptions & {
+ id?: string;
+};
+
+const map: Map = new Map();
+
+export const audioContext: (
+ options?: GetAudioContextOptions,
+) => Promise = (() => {
+ const didInteract = new Promise((res) => {
+ window.addEventListener("pointerdown", res, { once: true });
+ window.addEventListener("keydown", res, { once: true });
+ });
+
+ return async (options?: GetAudioContextOptions) => {
+ try {
+ const a = new Audio();
+ a.src =
+ "data:audio/wav;base64,UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA";
+ await a.play();
+ if (options?.id && map.has(options.id)) {
+ const ctx = map.get(options.id);
+ if (ctx) {
+ return ctx;
+ }
+ }
+ const ctx = new AudioContext(options);
+ if (options?.id) {
+ map.set(options.id, ctx);
+ }
+ return ctx;
+ } catch (e) {
+ await didInteract;
+ if (options?.id && map.has(options.id)) {
+ const ctx = map.get(options.id);
+ if (ctx) {
+ return ctx;
+ }
+ }
+ const ctx = new AudioContext(options);
+ if (options?.id) {
+ map.set(options.id, ctx);
+ }
+ return ctx;
+ }
+ };
+})();
+
+export const blobToJSON = (blob: Blob) =>
+ new Promise((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onload = () => {
+ if (reader.result) {
+ const json = JSON.parse(reader.result as string);
+ resolve(json);
+ } else {
+ reject("oops");
+ }
+ };
+ reader.readAsText(blob);
+ });
+
+export function base64ToArrayBuffer(base64: string) {
+ var binaryString = atob(base64);
+ var bytes = new Uint8Array(binaryString.length);
+ for (let i = 0; i < binaryString.length; i++) {
+ bytes[i] = binaryString.charCodeAt(i);
+ }
+ return bytes.buffer;
+}
diff --git a/src/lib/worklets/audio-processing.ts b/src/lib/worklets/audio-processing.ts
new file mode 100644
index 0000000000000000000000000000000000000000..8f6798818ceb7a650e068655f0a6afc629258539
--- /dev/null
+++ b/src/lib/worklets/audio-processing.ts
@@ -0,0 +1,73 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const AudioRecordingWorklet = `
+class AudioProcessingWorklet extends AudioWorkletProcessor {
+
+ // send and clear buffer every 2048 samples,
+ // which at 16khz is about 8 times a second
+ buffer = new Int16Array(2048);
+
+ // current write index
+ bufferWriteIndex = 0;
+
+ constructor() {
+ super();
+ this.hasAudio = false;
+ }
+
+ /**
+ * @param inputs Float32Array[][] [input#][channel#][sample#] so to access first inputs 1st channel inputs[0][0]
+ * @param outputs Float32Array[][]
+ */
+ process(inputs) {
+ if (inputs[0].length) {
+ const channel0 = inputs[0][0];
+ this.processChunk(channel0);
+ }
+ return true;
+ }
+
+ sendAndClearBuffer(){
+ this.port.postMessage({
+ event: "chunk",
+ data: {
+ int16arrayBuffer: this.buffer.slice(0, this.bufferWriteIndex).buffer,
+ },
+ });
+ this.bufferWriteIndex = 0;
+ }
+
+ processChunk(float32Array) {
+ const l = float32Array.length;
+
+ for (let i = 0; i < l; i++) {
+ // convert float32 -1 to 1 to int16 -32768 to 32767
+ const int16Value = float32Array[i] * 32768;
+ this.buffer[this.bufferWriteIndex++] = int16Value;
+ if(this.bufferWriteIndex >= this.buffer.length) {
+ this.sendAndClearBuffer();
+ }
+ }
+
+ if(this.bufferWriteIndex >= this.buffer.length) {
+ this.sendAndClearBuffer();
+ }
+ }
+}
+`;
+
+export default AudioRecordingWorklet;
diff --git a/src/lib/worklets/safari-audio-processing.ts b/src/lib/worklets/safari-audio-processing.ts
new file mode 100644
index 0000000000000000000000000000000000000000..1a82d748d6472211446ae0940c57502f83d55609
--- /dev/null
+++ b/src/lib/worklets/safari-audio-processing.ts
@@ -0,0 +1,99 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const SafariAudioRecordingWorklet = `
+class AudioProcessingWorklet extends AudioWorkletProcessor {
+ // Safari seems to work better with smaller buffer sizes
+ // and more frequent updates
+ buffer = new Int16Array(1024);
+ bufferWriteIndex = 0;
+ lastProcessTime = 0;
+ sampleRate = 0;
+
+ constructor(options) {
+ super();
+ console.log('Safari AudioProcessingWorklet constructed with options:', options);
+ this.sampleRate = options.processorOptions?.sampleRate || sampleRate;
+ console.log('Using sample rate:', this.sampleRate);
+ }
+
+ process(inputs) {
+ // Log processing details periodically
+ const now = currentTime;
+ if (now - this.lastProcessTime > 1) {
+ console.log('Safari AudioProcessingWorklet processing:', {
+ inputChannels: inputs[0]?.length,
+ inputSamples: inputs[0]?.[0]?.length,
+ bufferWriteIndex: this.bufferWriteIndex,
+ time: now
+ });
+ this.lastProcessTime = now;
+ }
+
+ if (!inputs[0]?.length) {
+ console.warn('No input channels available');
+ return true;
+ }
+
+ const channel0 = inputs[0][0];
+ if (!channel0?.length) {
+ console.warn('Empty input channel');
+ return true;
+ }
+
+ this.processChunk(channel0);
+ return true;
+ }
+
+ sendAndClearBuffer() {
+ if (this.bufferWriteIndex > 0) {
+ this.port.postMessage({
+ event: "chunk",
+ data: {
+ int16arrayBuffer: this.buffer.slice(0, this.bufferWriteIndex).buffer,
+ },
+ });
+ this.bufferWriteIndex = 0;
+ }
+ }
+
+ processChunk(float32Array) {
+ // Safari can sometimes send empty arrays or undefined
+ if (!float32Array?.length) {
+ return;
+ }
+
+ const l = float32Array.length;
+ for (let i = 0; i < l; i++) {
+ // Convert float32 -1 to 1 to int16 -32768 to 32767
+ // Add some additional gain for Safari which tends to be quieter
+ const int16Value = Math.max(-32768, Math.min(32767, float32Array[i] * 32768 * 1.5));
+ this.buffer[this.bufferWriteIndex++] = int16Value;
+
+ if (this.bufferWriteIndex >= this.buffer.length) {
+ this.sendAndClearBuffer();
+ }
+ }
+
+ // Make sure to send any remaining data
+ if (this.bufferWriteIndex > 0) {
+ this.sendAndClearBuffer();
+ }
+ }
+}
+`;
+
+export default SafariAudioRecordingWorklet;
\ No newline at end of file
diff --git a/src/lib/worklets/vol-meter.ts b/src/lib/worklets/vol-meter.ts
new file mode 100644
index 0000000000000000000000000000000000000000..bc7d0a46b973f3e6b0a379ffd64f37305b363b5b
--- /dev/null
+++ b/src/lib/worklets/vol-meter.ts
@@ -0,0 +1,65 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const VolMeterWorket = `
+ class VolMeter extends AudioWorkletProcessor {
+ volume
+ updateIntervalInMS
+ nextUpdateFrame
+
+ constructor() {
+ super()
+ this.volume = 0
+ this.updateIntervalInMS = 25
+ this.nextUpdateFrame = this.updateIntervalInMS
+ this.port.onmessage = event => {
+ if (event.data.updateIntervalInMS) {
+ this.updateIntervalInMS = event.data.updateIntervalInMS
+ }
+ }
+ }
+
+ get intervalInFrames() {
+ return (this.updateIntervalInMS / 1000) * sampleRate
+ }
+
+ process(inputs) {
+ const input = inputs[0]
+
+ if (input.length > 0) {
+ const samples = input[0]
+ let sum = 0
+ let rms = 0
+
+ for (let i = 0; i < samples.length; ++i) {
+ sum += samples[i] * samples[i]
+ }
+
+ rms = Math.sqrt(sum / samples.length)
+ this.volume = Math.max(rms, this.volume * 0.7)
+
+ this.nextUpdateFrame -= samples.length
+ if (this.nextUpdateFrame < 0) {
+ this.nextUpdateFrame += this.intervalInFrames
+ this.port.postMessage({volume: this.volume})
+ }
+ }
+
+ return true
+ }
+ }`;
+
+export default VolMeterWorket;
diff --git a/src/multimodal-live-types.ts b/src/multimodal-live-types.ts
new file mode 100644
index 0000000000000000000000000000000000000000..455186ea92bd02cebef20d3c97d9f47dc1fda0aa
--- /dev/null
+++ b/src/multimodal-live-types.ts
@@ -0,0 +1,242 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type {
+ Content,
+ FunctionCall,
+ GenerationConfig,
+ GenerativeContentBlob,
+ Part,
+ Tool,
+} from "@google/generative-ai";
+
+/**
+ * this module contains type-definitions and Type-Guards
+ */
+
+// Type-definitions
+
+/* outgoing types */
+
+/**
+ * the config to initiate the session
+ */
+export type LiveConfig = {
+ model: string;
+ systemInstruction?: { parts: Part[] };
+ generationConfig?: Partial;
+ tools?: Array;
+};
+
+export type LiveGenerationConfig = GenerationConfig & {
+ responseModalities: "text" | "audio" | "image";
+ speechConfig?: {
+ voiceConfig?: {
+ prebuiltVoiceConfig?: {
+ voiceName: "Puck" | "Charon" | "Kore" | "Fenrir" | "Aoede" | string;
+ };
+ };
+ };
+};
+
+export type LiveOutgoingMessage =
+ | SetupMessage
+ | ClientContentMessage
+ | RealtimeInputMessage
+ | ToolResponseMessage;
+
+export type SetupMessage = {
+ setup: LiveConfig;
+};
+
+export type ClientContentMessage = {
+ clientContent: {
+ turns: Content[];
+ turnComplete: boolean;
+ };
+};
+
+export type RealtimeInputMessage = {
+ realtimeInput: {
+ mediaChunks: GenerativeContentBlob[];
+ };
+};
+
+export type ToolResponseMessage = {
+ toolResponse: {
+ functionResponses: LiveFunctionResponse[];
+ };
+};
+
+export type ToolResponse = ToolResponseMessage["toolResponse"];
+
+export type LiveFunctionResponse = {
+ response: object;
+ id: string;
+};
+
+/** Incoming types */
+
+export type LiveIncomingMessage =
+ | ToolCallCancellationMessage
+ | ToolCallMessage
+ | ServerContentMessage
+ | SetupCompleteMessage;
+
+export type SetupCompleteMessage = { setupComplete: {} };
+
+export type ServerContentMessage = {
+ serverContent: ServerContent;
+};
+
+export type ServerContent = ModelTurn | TurnComplete | Interrupted;
+
+export type ModelTurn = {
+ modelTurn: {
+ parts: Part[];
+ };
+};
+
+export type TurnComplete = { turnComplete: boolean };
+
+export type Interrupted = { interrupted: true };
+
+export type ToolCallCancellationMessage = {
+ toolCallCancellation: {
+ ids: string[];
+ };
+};
+
+export type ToolCallCancellation =
+ ToolCallCancellationMessage["toolCallCancellation"];
+
+export type ToolCallMessage = {
+ toolCall: ToolCall;
+};
+
+export type LiveFunctionCall = FunctionCall & {
+ id: string;
+};
+
+/**
+ * A `toolCall` message
+ */
+export type ToolCall = {
+ functionCalls: LiveFunctionCall[];
+};
+
+/** log types */
+export type StreamingLog = {
+ date: Date;
+ type: string;
+ count?: number;
+ message: string | LiveOutgoingMessage | LiveIncomingMessage;
+};
+
+// Type-Guards
+
+const prop = (a: any, prop: string, kind: string = "object") =>
+ typeof a === "object" && typeof a[prop] === "object";
+
+// outgoing messages
+export const isSetupMessage = (a: unknown): a is SetupMessage =>
+ prop(a, "setup");
+
+export const isClientContentMessage = (a: unknown): a is ClientContentMessage =>
+ prop(a, "clientContent");
+
+export const isRealtimeInputMessage = (a: unknown): a is RealtimeInputMessage =>
+ prop(a, "realtimeInput");
+
+export const isToolResponseMessage = (a: unknown): a is ToolResponseMessage =>
+ prop(a, "toolResponse");
+
+// incoming messages
+export const isSetupCompleteMessage = (a: unknown): a is SetupCompleteMessage =>
+ prop(a, "setupComplete");
+
+export const isServerContentMessage = (a: any): a is ServerContentMessage =>
+ prop(a, "serverContent");
+
+export const isToolCallMessage = (a: any): a is ToolCallMessage =>
+ prop(a, "toolCall");
+
+export const isToolCallCancellationMessage = (
+ a: unknown,
+): a is ToolCallCancellationMessage =>
+ prop(a, "toolCallCancellation") &&
+ isToolCallCancellation((a as any).toolCallCancellation);
+
+export const isModelTurn = (a: any): a is ModelTurn =>
+ typeof (a as ModelTurn).modelTurn === "object";
+
+export const isTurnComplete = (a: any): a is TurnComplete =>
+ typeof (a as TurnComplete).turnComplete === "boolean";
+
+export const isInterrupted = (a: any): a is Interrupted =>
+ (a as Interrupted).interrupted;
+
+export function isToolCall(value: unknown): value is ToolCall {
+ if (!value || typeof value !== "object") return false;
+
+ const candidate = value as Record;
+
+ return (
+ Array.isArray(candidate.functionCalls) &&
+ candidate.functionCalls.every((call) => isLiveFunctionCall(call))
+ );
+}
+
+export function isToolResponse(value: unknown): value is ToolResponse {
+ if (!value || typeof value !== "object") return false;
+
+ const candidate = value as Record;
+
+ return (
+ Array.isArray(candidate.functionResponses) &&
+ candidate.functionResponses.every((resp) => isLiveFunctionResponse(resp))
+ );
+}
+
+export function isLiveFunctionCall(value: unknown): value is LiveFunctionCall {
+ if (!value || typeof value !== "object") return false;
+
+ const candidate = value as Record;
+
+ return (
+ typeof candidate.name === "string" &&
+ typeof candidate.id === "string" &&
+ typeof candidate.args === "object" &&
+ candidate.args !== null
+ );
+}
+
+export function isLiveFunctionResponse(
+ value: unknown,
+): value is LiveFunctionResponse {
+ if (!value || typeof value !== "object") return false;
+
+ const candidate = value as Record;
+
+ return (
+ typeof candidate.response === "object" && typeof candidate.id === "string"
+ );
+}
+
+export const isToolCallCancellation = (
+ a: unknown,
+): a is ToolCallCancellationMessage["toolCallCancellation"] =>
+ typeof a === "object" && Array.isArray((a as any).ids);
diff --git a/src/react-app-env.d.ts b/src/react-app-env.d.ts
new file mode 100644
index 0000000000000000000000000000000000000000..40734dbffac498ce9cc51d4d9c9e7612ec42385b
--- /dev/null
+++ b/src/react-app-env.d.ts
@@ -0,0 +1,17 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+///
diff --git a/src/reportWebVitals.ts b/src/reportWebVitals.ts
new file mode 100644
index 0000000000000000000000000000000000000000..f9261925dbae1867f74a181ae144180948407ee8
--- /dev/null
+++ b/src/reportWebVitals.ts
@@ -0,0 +1,31 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ReportHandler } from 'web-vitals';
+
+const reportWebVitals = (onPerfEntry?: ReportHandler) => {
+ if (onPerfEntry && onPerfEntry instanceof Function) {
+ import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => {
+ getCLS(onPerfEntry);
+ getFID(onPerfEntry);
+ getFCP(onPerfEntry);
+ getLCP(onPerfEntry);
+ getTTFB(onPerfEntry);
+ });
+ }
+};
+
+export default reportWebVitals;
diff --git a/src/setupTests.ts b/src/setupTests.ts
new file mode 100644
index 0000000000000000000000000000000000000000..474cc75d10bc5ee2fd3291c34fa1b0f8d29b3217
--- /dev/null
+++ b/src/setupTests.ts
@@ -0,0 +1,21 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// jest-dom adds custom jest matchers for asserting on DOM nodes.
+// allows you to do things like:
+// expect(element).toHaveTextContent(/react/i)
+// learn more: https://github.com/testing-library/jest-dom
+import '@testing-library/jest-dom';
diff --git a/tsconfig.json b/tsconfig.json
new file mode 100644
index 0000000000000000000000000000000000000000..0715e2b442f3c9006424d3c3b5f554047729087b
--- /dev/null
+++ b/tsconfig.json
@@ -0,0 +1,25 @@
+{
+ "compilerOptions": {
+ "target": "es2022",
+ "lib": ["dom", "dom.iterable", "esnext"],
+ "allowJs": true,
+ "skipLibCheck": true,
+ "esModuleInterop": true,
+ "allowSyntheticDefaultImports": true,
+ "strict": true,
+ "forceConsistentCasingInFileNames": true,
+ "noFallthroughCasesInSwitch": true,
+ "module": "esnext",
+ "moduleResolution": "node",
+ "resolveJsonModule": true,
+ "isolatedModules": true,
+ "noEmit": true,
+ "jsx": "react-jsx"
+ },
+ "include": ["src", "src/**/*"],
+ "ts-node": {
+ "compilerOptions": {
+ "module": "commonjs"
+ }
+ }
+}