publish

Dockerfile

@@ -0,0 +1,93 @@
+# build frontend with node
+FROM node:20-alpine AS frontend
+RUN apk add --no-cache libc6-compat
+WORKDIR /app
+
+COPY streaming-test-app .
+RUN \
+    if [ -f yarn.lock ]; then yarn --frozen-lockfile; \
+    elif [ -f package-lock.json ]; then npm ci; \
+    elif [ -f pnpm-lock.yaml ]; then yarn global add pnpm && pnpm i --frozen-lockfile; \
+    else echo "Lockfile not found." && exit 1; \
+    fi
+
+RUN npm run build
+
+# build backend on CUDA 
+FROM nvidia/cuda:11.8.0-cudnn8-devel-ubuntu22.04 AS backend
+WORKDIR /app
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV NODE_MAJOR=20
+
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install -y --no-install-recommends \
+    git \
+    git-lfs \
+    wget \
+    curl \
+    # python build dependencies \
+    build-essential \
+    libssl-dev \
+    zlib1g-dev \
+    libbz2-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    libncursesw5-dev \
+    xz-utils \
+    tk-dev \
+    libxml2-dev \
+    libxmlsec1-dev \
+    libffi-dev \
+    liblzma-dev \
+    sox libsox-fmt-all \
+    # gradio dependencies \
+    ffmpeg \
+    # fairseq2 dependencies \
+    libjpeg8-dev \
+    libpng-dev \
+    libsndfile-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+USER root
+RUN ln -s /usr/lib/x86_64-linux-gnu/libsox.so.3 /usr/lib/x86_64-linux-gnu/libsox.so
+# install older versions libjpeg62-turbo and libpng15
+RUN wget http://ftp.us.debian.org/debian/pool/main/libj/libjpeg-turbo/libjpeg62-turbo_2.1.5-2_amd64.deb && \
+    dpkg -i libjpeg62-turbo_2.1.5-2_amd64.deb && \
+    rm libjpeg62-turbo_2.1.5-2_amd64.deb
+RUN wget https://master.dl.sourceforge.net/project/libpng/libpng15/1.5.30/libpng-1.5.30.tar.gz && \
+    tar -xvf libpng-1.5.30.tar.gz && cd libpng-1.5.30 && ./configure && make && make install && cd .. && rm -rf libpng-1.5.30.tar.gz libpng-1.5.30
+    
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
+WORKDIR $HOME/app
+
+RUN curl https://pyenv.run | bash
+ENV PATH=$HOME/.pyenv/shims:$HOME/.pyenv/bin:$PATH
+ARG PYTHON_VERSION=3.10.12
+RUN pyenv install $PYTHON_VERSION && \
+    pyenv global $PYTHON_VERSION && \
+    pyenv rehash && \
+    pip install --no-cache-dir -U pip setuptools wheel
+
+COPY --chown=user:user ./seamless-server ./seamless-server
+# change dir since pip needs to seed whl folder
+RUN cd seamless-server && \
+    pip install fairseq2 --pre --extra-index-url https://fair.pkg.atmeta.com/fairseq2/whl/nightly/pt2.1.1/cu118 && \
+    pip install --no-cache-dir --upgrade -r requirements.txt
+COPY --from=frontend /app/dist ./streaming-test-app/dist
+
+WORKDIR $HOME/app/seamless-server
+RUN --mount=type=secret,id=HF_TOKEN,mode=0444,required=false \ 
+    huggingface-cli login --token $(cat /run/secrets/HF_TOKEN) || echo "HF_TOKEN error" && \
+    huggingface-cli download meta-private/SeamlessExpressive pretssel_melhifigan_wm-final.pt  --local-dir ./models/Seamless/ || echo "HF_TOKEN error" && \
+    ln -s $(readlink -f models/Seamless/pretssel_melhifigan_wm-final.pt) models/Seamless/pretssel_melhifigan_wm.pt || true;
+
+USER user
+RUN ["chmod", "+x", "./run_docker.sh"]
+CMD ./run_docker.sh
+

docker-compose.yml

@@ -0,0 +1,17 @@
+version: '3'
+services:
+  seamless:
+    build: .
+    volumes:
+      - ./seamless-server:/home/user/app/seamless-server # for hot reload in DEV
+    ports:
+      - "80:7860"
+    environment:
+      - NODE_ENV=development
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]

node-server/.gitignore

@@ -0,0 +1,2 @@
+/node_modules
+.env

node-server/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "server",
+  "version": "1.0.0",
+  "description": "",
+  "main": "server.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "start": "node server.js",
+    "dev": "nodemon index.js"
+  },
+  "engines": {
+    "node": "v20.5.0"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@deepgram/sdk": "^3.0.1",
+    "cors": "^2.8.5",
+    "crypto": "^1.0.1",
+    "dotenv": "^16.4.1",
+    "express": "^4.18.2",
+    "nodemon": "^3.0.3",
+    "socket.io": "^4.7.4"
+  }
+}

node-server/server.js

@@ -0,0 +1,33 @@
+require("dotenv").config();
+const express = require("express");
+const cors = require("cors");
+const app = express();
+const http = require("http").Server(app);
+const initializeWebSocket = require("./websocket");
+
+// TODO redis store
+const io = require("socket.io")(http, {
+  cors: {
+    origin: "http://localhost:5173",
+    methods: ["GET", "POST"],
+  },
+});
+
+initializeWebSocket(io);
+
+app.use(cors({ credentials: false, origin: "http://localhost:5173" }));
+
+app.use(express.json());
+
+app.use((req, _, next) => {
+  req.io = io;
+  next();
+});
+
+
+app.get("/", (req, res) => res.send("worked"))
+
+const PORT = process.env.PORT || 3002;
+http.listen(PORT, () => {
+  console.log(`Server listening at http://localhost:${PORT}`);
+});

node-server/transcription-client.js

@@ -0,0 +1,172 @@
+const { createClient, LiveTranscriptionEvents } = require("@deepgram/sdk");
+const EventEmitter = require("events");
+const crypto = require("crypto");
+
+class TranscriptionClient extends EventEmitter {
+  constructor() {
+    super();
+    this.deepgramStream = null;
+    this.deepgramSessionId = null; 
+    this.currentTranscript = "";
+    this.currentDiarization = {};
+    this.releaseTimeout = null;
+    this.killTimeout = null;
+    this.releaseThresholdMS = 4000;
+    this.killThresholdMS = 1000 * 60 * 2;
+    this.diarize = false;
+    this.speakerLabels = {};
+  }
+
+  startTranscriptionStream(language) {
+    console.log("started deepgram");
+    const localSessionId = crypto.randomUUID();
+    this.deepgramSessionId = localSessionId;
+    const deepgram = createClient(process.env.DEEPGRAM_API_KEY);
+    this.deepgramStream = deepgram.listen.live({
+      model: "nova-2",
+      punctuate: true,
+      language,
+      interim_results: true,
+      diarize: this.diarize,
+      smart_format: true,
+      endpointing: "2",
+    });
+
+    this.deepgramStream.on(LiveTranscriptionEvents.Error, (err) => {
+      console.log("Deepgram error: ", err);
+    });
+
+    this.deepgramStream.on(LiveTranscriptionEvents.Warning, (err) => {
+      console.log("Deepgram error: ", err);
+    });
+
+    this.deepgramStream.on(LiveTranscriptionEvents.Open, () => {
+      this.resetKillTimeout();
+
+      this.deepgramStream.on(
+        LiveTranscriptionEvents.Transcript,
+        async (data) => {
+          try {
+            const response = data.channel.alternatives[0];
+            const text = response?.transcript || "";
+            if (text.length > 1) {
+              clearTimeout(this.releaseTimeout);
+              this.releaseTimeout = setTimeout(() => {
+                this.releaseTranslations(true);
+              }, this.releaseThresholdMS);
+              this.resetKillTimeout();
+            }
+
+            // important not to translate interim results
+            if (response.transcript && data.is_final) {
+              // console.log(response.transcript);
+              const words = data.channel?.alternatives[0]?.words || [];
+              words.forEach(({ punctuated_word, speaker, start, end }) => {
+                if (!this.currentDiarization[speaker])
+                  this.currentDiarization[speaker] = "";
+                this.currentDiarization[speaker] += " " + punctuated_word;
+              });
+              this.emit("transcript", text)
+              this.currentTranscript += " " + text;
+              this.releaseTranslations();
+              // this.fullTranscript += " " + this.currentTranscript;
+            }
+          } catch (err) {
+            console.log(
+              "TranscribeTranslate.LiveTranscriptionEvents.Transcript:",
+              err
+            );
+          }
+        }
+      );
+    });
+    return this.deepgramSessionId;
+  }
+
+  resetKillTimeout = () => {
+    clearTimeout(this.killTimeout);
+    this.killTimeout = setTimeout(
+      () => this.endTranscriptionStream(),
+      this.killThresholdMS
+    );
+  };
+
+  releaseTranslations = async (triggeredByPause = false) => {
+    try {
+      let segment = "";
+      let speaker = null;
+      if (this.diarize) {
+        const processedSpeakers = Object.entries(this.currentDiarization).map(
+          ([speaker, transcript]) => ({
+            ...this.checkShouldSegment(transcript, triggeredByPause ? 5 : 50),
+            speaker,
+          })
+        );
+        const chosen = processedSpeakers.find((s) => s.canRelease);
+        if (!chosen) return;
+        this.currentDiarization = { [chosen.speaker]: chosen.secondPart };
+        segment = chosen.firstPart;
+        speaker = this.getSpeakerLabel(chosen.speaker);
+      } else {
+        const { canRelease, firstPart, secondPart } = this.checkShouldSegment(
+          this.currentTranscript,
+          triggeredByPause ? 5 : 50
+        );
+        if (!canRelease) return;
+        this.currentTranscript = secondPart;
+        segment = firstPart;
+      }
+
+      // translate segment
+      this.emit("translation", segment)
+
+   
+      this.lastEmittedSpeaker = speaker;
+    } catch (err) {
+      console.log("TranscribeTranslate.releaseTranslations:", err);
+    }
+  };
+
+  endTranscriptionStream() {
+    try {
+      clearTimeout(this.releaseTimeout);
+      clearTimeout(this.killTimeout);
+      if (!this.deepgramStream) return;
+      this.deepgramStream.finish();
+      this.deepgramStream = null;
+      this.currentTranscript = "";
+    } catch (err) {
+      console.log("Failed to end deepgram stream", err);
+    }
+  }
+
+  checkShouldSegment = (str, minCharLimit = 25) => {
+    let firstPart = "";
+    let secondPart = "";
+    const punct = new Set([".", "!", "?", "。", "۔"]);
+    for (let i = 0; i < str.length; i += 1) {
+      const char = str[i];
+      if (i > minCharLimit) {
+        if (punct.has(char)) {
+          firstPart = str.slice(0, i + 1);
+          secondPart = str.slice(i + 1);
+        }
+      }
+    }
+  
+    return { canRelease: !!firstPart.length, firstPart, secondPart };
+  };
+
+  send(payload) {
+    try {
+        if (!this.deepgramStream) return;
+        if (this.deepgramStream.getReadyState() === 1) {
+          this.deepgramStream.send(payload);
+        }
+      } catch (err) {
+        console.log("Failed to start deepgram stream", err);
+      }
+  }
+}
+
+module.exports = TranscriptionClient;

node-server/websocket.js

@@ -0,0 +1,30 @@
+const TranscriptClient = require("./transcription-client");
+
+// TODO remove x seconds after host left (incase reconnect)
+const initializeWebSocket = (io) => {
+  io.on("connection", (socket) => {
+    console.log(`connection made (${socket.id})`);
+    const transcriptClient = new TranscriptClient();
+
+    transcriptClient.on("translation", (result) => {
+      console.log(result)
+      io.to(socket.id).emit("translation", result)
+    })
+
+    socket.on('configure_stream', ({language}) => {
+      transcriptClient.startTranscriptionStream("en-US")
+    })
+
+    socket.on('incoming_audio', (data) => {
+      transcriptClient.send(data)
+    })
+
+    socket.on("disconnect", () => {
+      transcriptClient.endTranscriptionStream()
+    });
+  });
+
+  return io;
+};
+
+module.exports = initializeWebSocket;

seamless-server/.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+src/__pycache__/
+debug/
+.vscode/
+.env

seamless-server/models/Seamless/vad_s2st_sc_24khz_main.yaml

@@ -0,0 +1,25 @@
+agent_class: seamless_communication.streaming.agents.seamless_s2st.SeamlessS2STDualVocoderVADAgent
+monotonic_decoder_model_name: seamless_streaming_monotonic_decoder
+unity_model_name: seamless_streaming_unity
+sentencepiece_model: spm_256k_nllb100.model
+
+task: s2st
+tgt_lang: "eng"
+min_unit_chunk_size: 50
+decision_threshold: 0.7
+no_early_stop: True
+block_ngrams: True
+vocoder_name: vocoder_v2
+expr_vocoder_name: vocoder_pretssel
+gated_model_dir: .
+expr_vocoder_gain: 3.0
+upstream_idx: 1
+wav2vec_yaml: wav2vec.yaml
+min_starting_wait_w2vbert: 192
+
+config_yaml: cfg_fbank_u2t.yaml
+upstream_idx: 1
+detokenize_only: True
+device: cuda:0
+max_len_a: 0
+max_len_b: 1000

seamless-server/models/SeamlessStreaming/vad_s2st_sc_main.yaml

@@ -0,0 +1,21 @@
+agent_class: seamless_communication.streaming.agents.seamless_streaming_s2st.SeamlessStreamingS2STJointVADAgent
+monotonic_decoder_model_name: seamless_streaming_monotonic_decoder
+unity_model_name: seamless_streaming_unity
+sentencepiece_model: spm_256k_nllb100.model
+
+task: s2st
+tgt_lang: "eng"
+min_unit_chunk_size: 50
+decision_threshold: 0.7
+no_early_stop: True
+block_ngrams: True
+vocoder_name: vocoder_v2
+wav2vec_yaml: wav2vec.yaml
+min_starting_wait_w2vbert: 192
+
+config_yaml: cfg_fbank_u2t.yaml
+upstream_idx: 1
+detokenize_only: True
+device: cuda:0
+max_len_a: 0
+max_len_b: 1000

seamless-server/old_server.py

@@ -0,0 +1,874 @@
+from operator import itemgetter
+import os
+from typing import Any, Optional, Tuple, Dict, TypedDict
+from urllib import parse
+from uuid import uuid4
+from pprint import pformat
+import socketio
+import time
+import random
+import string
+import logging
+from starlette.applications import Starlette
+from starlette.routing import Mount, Route
+from starlette.staticfiles import StaticFiles
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from src.auth import google_auth_check
+from src.room import Room, Member
+from src.context import ContextManager
+from src.transcriber import Transcriber
+
+from src.simuleval_agent_directory import NoAvailableAgentException
+from src.simuleval_agent_directory import SimulevalAgentDirectory
+from src.simuleval_transcoder import SimulevalTranscoder
+from src.transcoder_helpers import get_transcoder_output_events
+from src.logging import initialize_logger
+
+DEBUG = True
+ALL_ROOM_ID = "ALL"
+ROOM_ID_USABLE_CHARACTERS = string.ascii_uppercase
+ROOM_ID_LENGTH = 4
+ROOM_LISTENERS_SUFFIX = "_listeners"
+ROOM_SPEAKERS_SUFFIX = "_speakers"
+ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME = "remove_server_lock"
+
+logger = initialize_logger("socketio_server_pubsub", level=logging.WARNING)
+
+print("=" * 20 + " ⭐️ Starting Server... ⭐️ " + "=" * 20)
+
+CLIENT_BUILD_PATH = "../streaming-test-app/dist/"
+static_files = {
+    "/": CLIENT_BUILD_PATH,
+    "/assets/seamless-db6a2555.svg": {
+        "filename": CLIENT_BUILD_PATH + "assets/seamless-db6a2555.svg",
+        "content_type": "image/svg+xml",
+    },
+}
+
+# sio is the main socket.io entrypoint
+sio = socketio.AsyncServer(
+    async_mode="asgi",
+    cors_allowed_origins="*",
+    logger=logger,
+    # engineio_logger=logger,
+)
+# sio.logger.setLevel(logging.DEBUG)
+socketio_app = socketio.ASGIApp(sio)
+
+app_routes = [
+    Mount("/ws", app=socketio_app),  # Mount Socket.IO server under /app
+    Mount(
+        "/", app=StaticFiles(directory=CLIENT_BUILD_PATH, html=True)
+    ),  # Serve static files from root
+]
+app = Starlette(debug=True, routes=app_routes)
+
+# rooms is indexed by room_id
+rooms: Dict[str, Room] = {}
+
+
+class MemberDirectoryObject(TypedDict):
+    room: Room
+    member_object: Member
+
+
+# member_directory is indexed by client_id
+# NOTE: client_id is really "client session id", meaning that it is unique to a single browser session.
+# If a user opens a new tab, they will have a different client_id and can join another room, join
+# the same room with different roles, etc.
+# NOTE: For a long-running production server we would want to clean up members after a certain timeout
+# but for this limited application we can just keep them around
+member_directory: Dict[str, MemberDirectoryObject] = {}
+
+
+class ServerLock(TypedDict):
+    name: str
+    client_id: str
+    member_object: Member
+
+
+SINGLE_USER = os.environ.get("SINGLE_USER")
+
+if os.environ.get("LOCK_SERVER_COMPLETELY", "0") == "1":
+    logger.info("LOCK_SERVER_COMPLETELY is set. Server will be locked on startup.")
+if SINGLE_USER == "1":
+    logger.info(
+        f"SINGLE_USER mode is set. Server will only allow one speaker or listener at a time."
+    )
+dummy_server_lock_member_object = Member(
+    client_id="seamless_user", session_id="dummy", name="Seamless User"
+)
+# Normally this would be an actual transcoder, but it's fine putting True here since currently we only check for the presence of the transcoder
+dummy_server_lock_member_object.transcoder = True
+server_lock: Optional[ServerLock] = (
+    {
+        "name": "Seamless User",
+        "client_id": "seamless_user",
+        "member_object": dummy_server_lock_member_object,
+    }
+    if os.environ.get("LOCK_SERVER_COMPLETELY", "0") == "1"
+    else None
+)
+
+server_id = str(uuid4())
+
+# Specify specific models to load (some environments have issues loading multiple models)
+# See AgentWithInfo with JSON format details.
+models_override = os.environ.get("MODELS_OVERRIDE")
+
+available_agents = SimulevalAgentDirectory()
+logger.info("Building and adding agents...")
+if models_override is not None:
+    logger.info(f"MODELS_OVERRIDE supplied from env vars: {models_override}")
+available_agents.build_and_add_agents(models_override)
+
+agents_capabilities_for_json = available_agents.get_agents_capabilities_list_for_json()
+
+
+def catch_and_log_exceptions_for_sio_event_handlers(func):
+    # wrapper should have the same signature as the original function
+    async def catch_exception_wrapper(*args, **kwargs):
+        try:
+            return await func(*args, **kwargs)
+        except Exception as e:
+            message = f"[app_pubsub] Caught exception in '{func.__name__}' event handler:\n\n{e}"
+            logger.exception(message, stack_info=True)
+
+            try:
+                exception_data = {
+                    "message": message,
+                    "timeEpochMs": int(time.time() * 1000),
+                }
+
+                try:
+                    # Let's try to add as much useful metadata as possible to the server_exception event
+                    sid = args[0]
+                    if isinstance(sid, str) and len(sid) > 0:
+                        session_data = await get_session_data(sid)
+                        if session_data:
+                            client_id = session_data.get("client_id")
+                            member = session_data.get("member_object")
+                            room = session_data.get("room_object")
+
+                            exception_data["room"] = str(room)
+                            exception_data["member"] = str(member)
+                            exception_data["clientID"] = str(client_id)
+                except Exception as inner_e:
+                    # We expect there will be times when clientID or other values aren't present, so just log this as a warning
+                    logger.warn(
+                        f"[app_pubsub] Caught exception while trying add additional_data to server_exception:\n\n{inner_e}"
+                    )
+
+                # For now let's emit this to all clients. We ultimatley may want to emit it just to the room it's happening in.
+                await sio.emit("server_exception", exception_data)
+            except Exception as inner_e:
+                logger.exception(
+                    f"[app_pubsub] Caught exception while trying to emit server_exception event:\n{inner_e}"
+                )
+
+            # Re-raise the exception so it's handled normally by the server
+            raise e
+
+    # Set the name of the wrapper to the name of the original function so that the socketio server can associate it with the right event
+    catch_exception_wrapper.__name__ = func.__name__
+    return catch_exception_wrapper
+
+
+async def emit_room_state_update(room):
+    await sio.emit(
+        "room_state_update",
+        room.to_json(),
+        room=room.room_id,
+    )
+
+
+async def emit_server_state_update():
+    room_statuses = {
+        room_id: room.get_room_status_dict() for room_id, room in rooms.items()
+    }
+    total_active_connections = sum(
+        [room_status["activeConnections"] for room_status in room_statuses.values()]
+    )
+    total_active_transcoders = sum(
+        [room_status["activeTranscoders"] for room_status in room_statuses.values()]
+    )
+    logger.info(
+        f"[Server Status]: {total_active_connections} active connections (in rooms); {total_active_transcoders} active transcoders"
+    )
+    logger.info(f"[Server Status]: server_lock={server_lock}")
+    server_lock_object_for_js = (
+        {
+            "name": server_lock.get("name"),
+            "clientID": server_lock.get("client_id"),
+            "isActive": server_lock.get("member_object")
+            and server_lock.get("member_object").transcoder is not None,
+        }
+        if server_lock
+        else None
+    )
+    await sio.emit(
+        "server_state_update",
+        {
+            "statusByRoom": room_statuses,
+            "totalActiveConnections": total_active_connections,
+            "totalActiveTranscoders": total_active_transcoders,
+            "agentsCapabilities": agents_capabilities_for_json,
+            "serverLock": server_lock_object_for_js,
+        },
+        room=ALL_ROOM_ID,
+    )
+
+
+async def get_session_data(sid):
+    session = await sio.get_session(sid)
+    # It seems like if the session has not been set that get_session may return None, so let's provide a fallback empty dictionary here
+    return session or {}
+
+
+async def set_session_data(
+    sid, client_id, room_id, room_object, member_object, context_obj, transcriber
+):
+    await sio.save_session(
+        sid,
+        {
+            "client_id": client_id,
+            "room_id": room_id,
+            "room_object": room_object,
+            "member_object": member_object,
+            "context_obj": context_obj,
+            "transcriber": transcriber,
+        },
+    )
+
+
+def get_random_room_id():
+    return "".join(random.choices(ROOM_ID_USABLE_CHARACTERS, k=ROOM_ID_LENGTH))
+
+
+def get_random_unused_room_id():
+    room_id = get_random_room_id()
+    while room_id in rooms:
+        room_id = get_random_room_id()
+    return room_id
+
+
+###############################################
+# Socket.io Basic Event Handlers
+###############################################
+
+
+@sio.on("connect")
+@catch_and_log_exceptions_for_sio_event_handlers
+async def connect(sid, environ):
+    logger.info(f"📥 [event: connected] sid={sid}")
+
+    # TODO: Sanitize/validate query param input
+    query_params = dict(parse.parse_qsl(environ["QUERY_STRING"]))
+    client_id = query_params.get("clientID")
+    token = query_params.get("token")
+
+    if google_auth_check(token) is None:
+        await sio.emit("auth_error", "Not authenticated", to=sid)
+        logger.info("Invalid auth token, Disconnecting...")
+        await sio.disconnect(sid)
+        return
+
+    logger.debug(f"query_params:\n{pformat(query_params)}")
+
+    if client_id is None:
+        logger.info("No clientID provided. Disconnecting...")
+        await sio.disconnect(sid)
+        return
+
+    # On reconnect we need to rejoin rooms and reset session data
+    if member_directory.get(client_id):
+        room = member_directory[client_id].get("room")
+        room_id = room.room_id
+        # Note: We could also get this from room.members[client_id]
+        member = member_directory[client_id].get("member_object")
+        context = member_directory[client_id].get("context_obj")
+        transcriber = member_directory[client_id].get("transcriber")
+        member.connection_status = "connected"
+        member.session_id = sid
+
+        logger.info(
+            f"[event: connect] {member} reconnected. Attempting to re-add them to socketio rooms and reset session data."
+        )
+
+        if room is None or member is None:
+            logger.error(
+                f"[event: connect] {client_id} is reconnecting, but room or member is None. This should not happen."
+            )
+            await sio.disconnect(sid)
+            return
+
+        sio.enter_room(sid, room_id)
+        sio.enter_room(sid, ALL_ROOM_ID)
+
+        if client_id in room.listeners:
+            sio.enter_room(sid, f"{room_id}{ROOM_LISTENERS_SUFFIX}")
+        if client_id in room.speakers:
+            sio.enter_room(sid, f"{room_id}{ROOM_SPEAKERS_SUFFIX}")
+
+        # Save the room_id to the socketio client session
+        await set_session_data(
+            sid,
+            client_id=client_id,
+            room_id=room.room_id,
+            room_object=room,
+            member_object=member,
+            context_obj=context,
+            transcriber=transcriber,
+        )
+        await emit_room_state_update(room)
+    else:
+        # Save the client id to the socketio client session
+        await set_session_data(
+            sid,
+            client_id=client_id,
+            room_id=None,
+            room_object=None,
+            member_object=None,
+            context_obj=None,
+            transcriber=None,
+        )
+
+    await sio.emit("server_id", server_id, to=sid)
+    await emit_server_state_update()
+
+
+@sio.event
+@catch_and_log_exceptions_for_sio_event_handlers
+async def disconnect(sid):
+    global server_lock
+    session_data = await get_session_data(sid)
+
+    client_id = None
+    member = None
+    room = None
+
+    if session_data:
+        client_id = session_data.get("client_id")
+        member = session_data.get("member_object")
+        room = session_data.get("room_object")
+
+    logger.info(
+        f"[event: disconnect][{room or 'NOT_IN_ROOM'}] member: {member or 'NO_MEMBER_OBJECT'} disconnected"
+    )
+
+    # Release the lock if this is the client that holds the current server lock
+    if server_lock and server_lock.get("client_id") == client_id:
+        server_lock = None
+
+    if member:
+        member.connection_status = "disconnected"
+
+        if member.transcoder:
+            member.transcoder.close = True
+            member.transcoder = None
+            member.requested_output_type = None
+
+        if room:
+            logger.info(
+                f"[event: disconnect] {member} disconnected from room {room.room_id}"
+            )
+            await emit_room_state_update(room)
+        else:
+            logger.info(
+                f"[event: disconnect] {member} disconnected, but no room object present. This should not happen."
+            )
+    else:
+        logger.info(
+            f"[event: disconnect] client_id {client_id or 'NO_CLIENT_ID'} with sid {sid} in rooms {str(sio.rooms(sid))} disconnected"
+        )
+
+    await emit_server_state_update()
+
+
+@sio.on("*")
+async def catch_all(event, sid, data):
+    logger.info(f"[unhandled event: {event}] sid={sid} data={data}")
+
+
+###############################################
+# Socket.io Streaming Event handlers
+###############################################
+
+
+@sio.on("join_room")
+@catch_and_log_exceptions_for_sio_event_handlers
+async def join_room(sid, client_id, room_id_from_client, config_dict):
+    global server_lock
+
+    args = {
+        "sid": sid,
+        "client_id": client_id,
+        "room_id": room_id_from_client,
+        "config_dict": config_dict,
+    }
+    logger.info(f"[event: join_room] {args}")
+    session_data = await get_session_data(sid)
+
+    logger.info(f"session_data: {session_data}")
+
+    room_id = room_id_from_client
+    if room_id is None:
+        room_id = get_random_unused_room_id()
+        logger.info(
+            f"No room_id provided. Generating a random, unused room_id: {room_id}"
+        )
+
+    # Create the room if it doesn't already exist
+    if room_id not in rooms:
+        rooms[room_id] = Room(room_id)
+
+    room = rooms[room_id]
+
+    member = None
+
+    name = "[NO_NAME]"
+
+    context = ContextManager()
+
+    transcriber = Transcriber()
+
+    # If the client is reconnecting use their existing member object. Otherwise create a new one.
+    if client_id in room.members:
+        member = room.members[client_id]
+        logger.info(f"{member} is rejoining room {room_id}.")
+    else:
+        member_number = len(room.members) + 1
+        name = f"Member {member_number}"
+        member = Member(
+            client_id=client_id,
+            session_id=sid,
+            name=name,
+        )
+        allow_user = check_and_lock_single_user(client_id, member)
+        if not allow_user:
+            logger.error(
+                f"In SINGLE_USER mode we only allow one user at a time. Ignoring request to configure stream from client {client_id}."
+            )
+            return {"status": "error", "message": "max_users"}
+
+        logger.info(f"Created a new Member object: {member}")
+        logger.info(f"Adding {member} to room {room_id}")
+        room.members[client_id] = member
+
+    # Also add them to the member directory
+    member_directory[client_id] = {"room": room, "member_object": member}
+
+    # Join the socketio room, which enables broadcasting to all members of the room
+    sio.enter_room(sid, room_id)
+    # Join the room for all clients
+    sio.enter_room(sid, ALL_ROOM_ID)
+
+    if "listener" in config_dict["roles"]:
+        sio.enter_room(sid, f"{room_id}{ROOM_LISTENERS_SUFFIX}")
+        if client_id not in room.listeners:
+            room.listeners.append(client_id)
+    else:
+        sio.leave_room(sid, f"{room_id}{ROOM_LISTENERS_SUFFIX}")
+        room.listeners = [
+            listener_id for listener_id in room.listeners if listener_id != client_id
+        ]
+
+    if "speaker" in config_dict["roles"]:
+        sio.enter_room(sid, f"{room_id}{ROOM_SPEAKERS_SUFFIX}")
+        if client_id not in room.speakers:
+            room.speakers.append(client_id)
+    else:
+        sio.leave_room(sid, f"{room_id}{ROOM_SPEAKERS_SUFFIX}")
+        # If the person is no longer a speaker they should no longer be able to lock the server
+        if server_lock and server_lock.get("client_id") == client_id:
+            logger.info(
+                f"🔓 Server is now unlocked from client {server_lock.get('client_id')} with name/info: {server_lock.get('name')}"
+            )
+            server_lock = None
+        if member.transcoder:
+            member.transcoder.close = True
+            member.transcoder = None
+        room.speakers = [
+            speaker_id for speaker_id in room.speakers if speaker_id != client_id
+        ]
+
+    # Only speakers should be able to lock the server
+    if config_dict.get("lockServerName") is not None and "speaker" in config_dict.get(
+        "roles", {}
+    ):
+        # If something goes wrong and the server gets stuck in a locked state the client can
+        # force the server to remove the lock by passing the special name ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME
+        if (
+            server_lock is not None
+            and config_dict.get("lockServerName")
+            == ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME
+            # If we are locking the server completely we don't want someone to be able to unlock it
+            and not os.environ.get("LOCK_SERVER_COMPLETELY", "0") == "1"
+        ):
+            server_lock = None
+            logger.info(
+                f"🔓 Server lock has been reset by {client_id} using the escape hatch name {ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME}"
+            )
+
+        # If the server is not locked, set a lock. If it's already locked to this client, update the lock object
+        if server_lock is None or server_lock.get("client_id") == client_id:
+            # TODO: Add some sort of timeout as a backstop in case someone leaves the browser tab open after locking the server
+            server_lock = {
+                "name": config_dict.get("lockServerName"),
+                "client_id": client_id,
+                "member_object": member,
+            }
+            logger.info(
+                f"🔒 Server is now locked to client {server_lock.get('client_id')} with name/info: {server_lock.get('name')}\nThis client will have priority over all others until they disconnect."
+            )
+        # If the server is already locked to someone else, don't allow this client to lock it
+        elif server_lock is not None and server_lock.get("client_id") != client_id:
+            logger.warn(
+                f"⚠️  Server is already locked to client {server_lock.get('client_id')}. Ignoring request to lock to client {client_id}."
+            )
+            # TODO: Maybe throw an error here?
+
+    # Save the room_id to the socketio client session
+    await set_session_data(
+        sid,
+        client_id=client_id,
+        room_id=room_id,
+        room_object=room,
+        member_object=member,
+        context_obj=context,
+        transcriber=transcriber,
+    )
+
+    await emit_room_state_update(room)
+    await emit_server_state_update()
+
+    return {"roomsJoined": sio.rooms(sid), "roomID": room_id}
+
+
+def check_and_lock_single_user(client_id, member):
+    global server_lock
+
+    if SINGLE_USER is None:
+        return True
+
+    if server_lock is None:
+        server_lock = {
+            "name": "single_user",
+            "client_id": client_id,
+            "member_object": member,
+        }
+        return True
+
+    return server_lock["client_id"] == client_id
+
+
+# @sio.on("disconnect")
+# @catch_and_log_exceptions_for_sio_event_handlers
+# async def disconnect(sid):
+#     logger.info(f"📤 [event: disconnected] sid={sid}")
+#     # Additional code to handle the disconnect event
+
+
+# TODO: Add code to prevent more than one speaker from connecting/streaming at a time
+@sio.event
+@catch_and_log_exceptions_for_sio_event_handlers
+async def configure_stream(sid, config):
+    session_data = await get_session_data(sid)
+    client_id, member, room, transcriber = itemgetter(
+        "client_id", "member_object", "room_object", "transcriber"
+    )(session_data)
+
+    logger.debug(
+        f"[event: configure_stream][{room}] Received stream config from {member}\n{pformat(config)}"
+    )
+
+    if member is None or room is None:
+        logger.error(
+            f"Received stream config from {member}, but member or room is None. This should not happen."
+        )
+        return {"status": "error", "message": "member_or_room_is_none"}
+
+    # if not allow_speaker(room, client_id):
+    #     logger.error(
+    #         f"In MAX_SPEAKERS mode we only allow one speaker at a time. Ignoring request to configure stream from client {client_id}."
+    #     )
+    #     return {"status": "error", "message": "max_speakers"}
+
+    # If there is a server lock WITH an active transcoder session, prevent other users from configuring and starting a stream
+    # If the server lock client does NOT have an active transcoder session allow this to proceed, knowing that
+    # this stream will be interrupted if the server lock client starts streaming
+    if (
+        server_lock is not None
+        and server_lock.get("client_id") != client_id
+        and server_lock.get("member_object")
+        and server_lock.get("member_object").transcoder is not None
+    ):
+        logger.warn(
+            f"Server is locked to client {server_lock.get('client_id')}. Ignoring request to configure stream from client {client_id}."
+        )
+        return {"status": "error", "message": "server_locked"}
+
+    debug = config.get("debug")
+    async_processing = config.get("async_processing")
+    manual_transcribe = config.get("manual_transcribe")
+    member.manual_transcribe = manual_transcribe
+
+    if manual_transcribe:
+        await transcriber.start()
+    else:
+        # Currently s2s, s2t or s2s&t
+        model_type = config.get("model_type")
+        member.requested_output_type = model_type
+
+        model_name = config.get("model_name")
+
+        try:
+            agent = available_agents.get_agent_or_throw(model_name)
+        except NoAvailableAgentException as e:
+            logger.warn(f"Error while getting agent: {e}")
+            # await sio.emit("error", str(e), to=sid)
+            await sio.disconnect(sid)
+            return {"status": "error", "message": str(e)}
+
+        if member.transcoder:
+            logger.warn(
+                "Member already has a transcoder configured. Closing it, and overwriting with a new transcoder..."
+            )
+            member.transcoder.close = True
+
+        t0 = time.time()
+        try:
+            member.transcoder = SimulevalTranscoder(
+                agent,
+                config["rate"],
+                debug=debug,
+                buffer_limit=int(config["buffer_limit"]),
+            )
+        except Exception as e:
+            logger.warn(f"Got exception while initializing agents: {e}")
+            # await sio.emit("error", str(e), to=sid)
+            await sio.disconnect(sid)
+            return {"status": "error", "message": str(e)}
+
+        t1 = time.time()
+        logger.debug(f"Booting up VAD and transcoder took {t1-t0} sec")
+
+        # TODO: if async_processing is false, then we need to run transcoder.process_pipeline_once() whenever we receive audio, or at some other sensible interval
+        if async_processing:
+            member.transcoder.start()
+
+    # We need to emit a room state update here since room state now includes # of active transcoders
+    await emit_room_state_update(room)
+    await emit_server_state_update()
+
+    return {"status": "ok", "message": "server_ready"}
+
+
+# The config here is a partial config, meaning it may not contain all the config values -- only the ones the user
+# wants to change
+@sio.on("set_dynamic_config")
+@catch_and_log_exceptions_for_sio_event_handlers
+async def set_dynamic_config(
+    sid,
+    # partial_config's type is defined in StreamingTypes.ts
+    partial_config,
+):
+    session_data = await get_session_data(sid)
+
+    member = None
+    context = None
+    if session_data:
+        member = session_data.get("member_object")
+        context = session_data.get("context_obj")
+
+    if member:
+        new_dynamic_config = {
+            **(member.transcoder_dynamic_config or {}),
+            **partial_config,
+        }
+        logger.info(
+            f"[set_dynamic_config] Setting new dynamic config:\n\n{pformat(new_dynamic_config)}\n"
+        )
+        member.transcoder_dynamic_config = new_dynamic_config
+
+    if context:
+        context.set_language(partial_config["targetLanguage"])
+
+    # TODO set transcriber language
+
+    return {"status": "ok", "message": "dynamic_config_set"}
+
+
+@sio.event
+@catch_and_log_exceptions_for_sio_event_handlers
+async def incoming_audio(sid, blob):
+    session_data = await get_session_data(sid)
+
+    client_id = None
+    member = None
+    room = None
+    context = None
+    transcriber = None
+
+    if session_data:
+        client_id = session_data.get("client_id")
+        member = session_data.get("member_object")
+        room = session_data.get("room_object")
+        context = session_data.get("context_obj")
+        transcriber = session_data.get("transcriber")
+
+    logger.debug(f"[event: incoming_audio] from member {member}")
+
+    # If the server is locked by someone else, kill our transcoder and ignore incoming audio
+    # If the server lock client does NOT have an active transcoder session allow this incoming audio pipeline to proceed,
+    # knowing that this stream will be interrupted if the server lock client starts streaming
+    if member.manual_transcribe:
+        print(blob)
+        await transcriber.sendAudio(blob)
+        return
+
+    if (
+        server_lock is not None
+        and server_lock.get("client_id") != client_id
+        and server_lock.get("member_object")
+        and server_lock.get("member_object").transcoder is not None
+    ):
+        # TODO: Send an event to the client to let them know their streaming session has been killed
+        if member.transcoder:
+            member.transcoder.close = True
+            member.transcoder = None
+            # Update both room state and server state given that the number of active transcoders has changed
+            if room:
+                await emit_room_state_update(room)
+            await emit_server_state_update()
+        logger.warn(
+            f"[incoming_audio] Server is locked to client {server_lock.get('client_id')}. Ignoring incoming audio from client {client_id}."
+        )
+        return
+
+    if member is None or room is None:
+        logger.error(
+            f"[incoming_audio] Received incoming_audio from {member}, but member or room is None. This should not happen."
+        )
+        return
+
+    if member.manual_transcribe:
+        transcriber.sendAudio(blob)
+    else:
+        # NOTE: bytes and bytearray are very similar, but bytes is immutable, and is what is returned by socketio
+        if not isinstance(blob, bytes):
+            logger.error(
+                f"[incoming_audio] Received audio from {member}, but it was not of type `bytes`. type(blob) = {type(blob)}"
+            )
+            return
+
+        if member.transcoder is None:
+            logger.error(
+                f"[incoming_audio] Received audio from {member}, but no transcoder configured to process it (member.transcoder is None). This should not happen."
+            )
+            return
+
+        member.transcoder.process_incoming_bytes(
+            blob, dynamic_config=member.transcoder_dynamic_config
+        )
+
+        # Send back any available model output
+        # NOTE: In theory it would make sense remove this from the incoming_audio handler and
+        # handle this in a dedicated thread that checks for output and sends it right away,
+        # but in practice for our limited demo use cases this approach didn't add noticeable
+        # latency, so we're keeping it simple for now.
+        events = get_transcoder_output_events(member.transcoder)
+        logger.debug(f"[incoming_audio] transcoder output events: {len(events)}")
+
+        if len(events) == 0:
+            logger.debug("[incoming_audio] No transcoder output to send")
+        else:
+            for e in events:
+                if e[
+                    "event"
+                ] == "translation_speech" and member.requested_output_type in [
+                    "s2s",
+                    "s2s&t",
+                ]:
+                    logger.debug("[incoming_audio] Sending translation_speech event")
+                    await sio.emit(
+                        "translation_speech", e, room=f"{room.room_id}_listeners"
+                    )
+                elif e[
+                    "event"
+                ] == "translation_text" and member.requested_output_type in [
+                    "s2t",
+                    "s2s&t",
+                ]:
+                    logger.debug("[incoming_audio] Sending translation_text event")
+                    await sio.emit(
+                        "translation_text", e, room=f"{room.room_id}_listeners"
+                    )
+                    context.add_text_chunk(e["payload"])
+                else:
+                    logger.error(
+                        f"[incoming_audio] Unexpected event type: {e['event']}"
+                    )
+
+    new_context = context.get_current_context()
+    if new_context:
+        await sio.emit(
+            "context",
+            {"event": "context", "payload": new_context},
+            room=f"{room.room_id}_listeners",
+        )
+    return
+
+
+@sio.event
+@catch_and_log_exceptions_for_sio_event_handlers
+async def stop_stream(sid):
+    session_data = await get_session_data(sid)
+    client_id, member, room = itemgetter("client_id", "member_object", "room_object")(
+        session_data
+    )
+
+    logger.debug(f"[event: stop_stream][{room}] Attempting to stop stream for {member}")
+
+    if member is None or room is None:
+        message = f"Received stop_stream from {member}, but member or room is None. This should not happen."
+        logger.error(message)
+        return {"status": "error", "message": message}
+
+    # In order to stop the stream and end the transcoder thread, set close to True and unset it for the member
+    if member.transcoder:
+        member.transcoder.close = True
+        member.transcoder = None
+    else:
+        message = f"Received stop_stream from {member}, but member.transcoder is None. This should not happen."
+        logger.warn(message)
+
+    # We need to emit a room state update here since room state now includes # of active transcoders
+    await emit_room_state_update(room)
+    # Emit a server state update now that we've changed the number of active transcoders
+    await emit_server_state_update()
+
+    return {"status": "ok", "message": "Stream stopped"}
+
+
+@sio.on("clear_transcript_for_all")
+@catch_and_log_exceptions_for_sio_event_handlers
+async def clear_transcript_for_all(sid):
+    session_data = await get_session_data(sid)
+
+    room = session_data.get("room_object")
+
+    if room:
+        await sio.emit("clear_transcript", room=f"{room.room_id}")
+    else:
+        logger.error("[clear_transcript] room is None. This should not happen.")
+
+
+@sio.event
+@catch_and_log_exceptions_for_sio_event_handlers
+async def set_name(sid, name):
+    logger.info(f"[Event: set_name] name={name}")
+    await sio.save_session(sid, {"name": name})

seamless-server/requirements.txt

@@ -0,0 +1,34 @@
+# seamless_communication
+git+https://github.com/facebookresearch/seamless_communication.git
+# ./whl/seamless_communication-1.0.0-py3-none-any.whl
+Flask==2.1.3
+Flask_Sockets==0.2.1
+g2p_en==2.1.0
+gevent==22.10.2
+gevent_websocket==0.10.1
+librosa==0.9.2
+numpy==1.24.4
+openai_whisper==20230124
+protobuf==4.24.2
+psola==0.0.1
+pydub==0.25.1
+silero==0.4.1
+soundfile==0.11.0
+stable_ts==1.4.0
+# torch  # to be installed by user for desired PyTorch version
+# simuleval  # to be installed by seamless_communication
+Werkzeug==2.0.3
+whisper==1.1.10
+colorlog==6.7.0
+python-socketio==5.9.0
+uvicorn[standard]==0.23.2
+parallel-wavegan==0.5.5
+python-jose[cryptography]==3.3.0
+starlette==0.32.0.post1 
+hf_transfer==0.1.4
+huggingface_hub==0.19.
+google-auth
+python-dotenv
+deepgram-sdk
+sentencepiece
+fairseq2

seamless-server/run_docker.sh

@@ -0,0 +1,5 @@
+# !/bin/bash
+if [ -f models/Seamless/pretssel_melhifigan_wm.pt ] ; then
+    export USE_EXPRESSIVE_MODEL=1;
+fi
+uvicorn new:app --host 0.0.0.0 --port 7860 --reload

seamless-server/server.py

@@ -0,0 +1,288 @@
+from operator import itemgetter
+import os
+from urllib import parse
+from pprint import pformat
+import socketio
+import time
+import logging
+from starlette.applications import Starlette
+from starlette.routing import Mount, Route
+from starlette.staticfiles import StaticFiles
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from src.auth import google_auth_check
+from src.client import Client
+from src.context import ContextManager
+from src.transcriber import Transcriber
+
+from src.simuleval_agent_directory import NoAvailableAgentException
+from src.simuleval_agent_directory import SimulevalAgentDirectory
+from src.simuleval_transcoder import SimulevalTranscoder
+from src.transcoder_helpers import get_transcoder_output_events
+from src.logging import (
+    initialize_logger,
+    catch_and_log_exceptions_for_sio_event_handlers,
+)
+
+logger = initialize_logger(__name__, level=logging.WARNING)
+print("=" * 20 + " ⭐️ Starting Server... ⭐️ " + "=" * 20)
+
+sio = socketio.AsyncServer(
+    async_mode="asgi",
+    cors_allowed_origins="*",
+    logger=logger,
+    # engineio_logger=logger,
+)
+socketio_app = socketio.ASGIApp(sio)
+
+app_routes = [
+    Mount("/ws", app=socketio_app),
+]
+app = Starlette(debug=True, routes=app_routes)
+
+# Specify specific models to load (some environments have issues loading multiple models)
+# See AgentWithInfo with JSON format details.
+models_override = os.environ.get("MODELS_OVERRIDE")
+
+available_agents = SimulevalAgentDirectory()
+logger.info("Building and adding agents...")
+if models_override is not None:
+    logger.info(f"MODELS_OVERRIDE supplied from env vars: {models_override}")
+available_agents.build_and_add_agents(models_override)
+
+agents_capabilities_for_json = available_agents.get_agents_capabilities_list_for_json()
+
+
+clients = {}
+
+
+@sio.on("connect")
+@catch_and_log_exceptions_for_sio_event_handlers(logger, sio)
+async def connect(sid, environ):
+    logger.info(f"📥 [event: connected] sid={sid}")
+
+    # TODO: Sanitize/validate query param input
+    query_params = dict(parse.parse_qsl(environ["QUERY_STRING"]))
+    client_id = query_params.get("clientID")
+    token = query_params.get("token")
+
+    if google_auth_check(token) is None:
+        await sio.emit("auth_error", "Not authenticated", to=sid)
+        logger.info("Invalid auth token, Disconnecting...")
+        await sio.disconnect(sid)
+        return
+
+    logger.debug(f"query_params:\n{pformat(query_params)}")
+
+    if client_id is None:
+        logger.info("No clientID provided. Disconnecting...")
+        await sio.disconnect(sid)
+        return
+
+    clients[sid] = Client(client_id)
+
+
+@sio.on("*")
+async def catch_all(event, sid, data):
+    logger.info(f"[unhandled event: {event}] sid={sid} data={data}")
+
+
+@sio.event
+@catch_and_log_exceptions_for_sio_event_handlers(logger, sio)
+async def configure_stream(sid, config):
+    client_obj = clients[sid]
+    logger.warning(sid)
+
+    if client_obj is None:
+        logger.error(f"No client object for {sid}")
+        await sio.disconnect(sid)
+        return {"status": "error", "message": "member_or_room_is_none"}
+
+    debug = config.get("debug")
+    async_processing = config.get("async_processing")
+    manual_transcribe = config.get("manual_transcribe")
+    client_obj.manual_transcribe = manual_transcribe
+
+    if manual_transcribe:
+        client_obj.transcriber = Transcriber()
+        client_obj.transcriber.start()
+    else:
+        # Currently s2s, s2t or s2s&t
+        model_type = config.get("model_type")
+        client_obj.requested_output_type = model_type
+
+        model_name = config.get("model_name")
+
+        try:
+            agent = available_agents.get_agent_or_throw(model_name)
+        except NoAvailableAgentException as e:
+            logger.warn(f"Error while getting agent: {e}")
+            await sio.disconnect(sid)
+            return {"status": "error", "message": str(e)}
+
+        if client_obj.transcoder:
+            logger.warn(
+                "Member already has a transcoder configured. Closing it, and overwriting with a new transcoder..."
+            )
+            client_obj.transcoder.close = True
+
+        t0 = time.time()
+        try:
+            client_obj.transcoder = SimulevalTranscoder(
+                agent,
+                config["rate"],
+                debug=debug,
+                buffer_limit=int(config["buffer_limit"]),
+            )
+        except Exception as e:
+            logger.warn(f"Got exception while initializing agents: {e}")
+            await sio.disconnect(sid)
+            return {"status": "error", "message": str(e)}
+
+        t1 = time.time()
+        logger.debug(f"Booting up VAD and transcoder took {t1-t0} sec")
+
+        # TODO: if async_processing is false, then we need to run transcoder.process_pipeline_once() whenever we receive audio, or at some other sensible interval
+        if async_processing:
+            client_obj.transcoder.start()
+
+    client_obj.context = ContextManager()
+    return {"status": "ok", "message": "server_ready"}
+
+
+@sio.on("set_dynamic_config")
+@catch_and_log_exceptions_for_sio_event_handlers(logger, sio)
+async def set_dynamic_config(
+    sid,
+    partial_config,
+):
+    client_obj = clients[sid]
+
+    if client_obj is None:
+        logger.error(f"No client object for {sid}")
+        await sio.disconnect(sid)
+        return {"status": "error", "message": "member_or_room_is_none"}
+
+    new_dynamic_config = {
+        **(client_obj.transcoder_dynamic_config or {}),
+        **partial_config,
+    }
+    logger.info(
+        f"[set_dynamic_config] Setting new dynamic config:\n\n{pformat(new_dynamic_config)}\n"
+    )
+
+    client_obj.transcoder_dynamic_config = new_dynamic_config
+
+    if client_obj.context:
+        client_obj.context.set_language(partial_config["targetLanguage"])
+
+    # TODO set transcriber language
+
+    return {"status": "ok", "message": "dynamic_config_set"}
+
+
+@sio.event
+async def incoming_audio(sid, blob):
+    client_obj = clients[sid]
+
+    if client_obj is None:
+        logger.error(f"No client object for {sid}")
+        await sio.disconnect(sid)
+        return {"status": "error", "message": "member_or_room_is_none"}
+
+    if client_obj.manual_transcribe:
+        client_obj.transcriber.send_audio(blob)
+    else:
+        # NOTE: bytes and bytearray are very similar, but bytes is immutable, and is what is returned by socketio
+        if not isinstance(blob, bytes):
+            logger.error(
+                f"[incoming_audio] Received audio from {sid}, but it was not of type `bytes`. type(blob) = {type(blob)}"
+            )
+            return
+
+        if client_obj.transcoder is None:
+            logger.error(
+                f"[incoming_audio] Received audio from {sid}, but no transcoder configured to process it (member.transcoder is None). This should not happen."
+            )
+            return
+
+        client_obj.transcoder.process_incoming_bytes(
+            blob, dynamic_config=client_obj.transcoder_dynamic_config
+        )
+
+        # Send back any available model output
+        # NOTE: In theory it would make sense remove this from the incoming_audio handler and
+        # handle this in a dedicated thread that checks for output and sends it right away,
+        # but in practice for our limited demo use cases this approach didn't add noticeable
+        # latency, so we're keeping it simple for now.
+        events = get_transcoder_output_events(client_obj.transcoder)
+        logger.debug(f"[incoming_audio] transcoder output events: {len(events)}")
+
+        if len(events) == 0:
+            logger.debug("[incoming_audio] No transcoder output to send")
+        else:
+            for e in events:
+                if e[
+                    "event"
+                ] == "translation_speech" and client_obj.requested_output_type in [
+                    "s2s",
+                    "s2s&t",
+                ]:
+                    logger.debug("[incoming_audio] Sending translation_speech event")
+                    await sio.emit("translation_speech", e, room=sid)
+                elif e[
+                    "event"
+                ] == "translation_text" and client_obj.requested_output_type in [
+                    "s2t",
+                    "s2s&t",
+                ]:
+                    logger.debug("[incoming_audio] Sending translation_text event")
+                    await sio.emit("translation_text", e, room=sid)
+                    client_obj.context.add_text_chunk(e["payload"])
+                else:
+                    logger.error(
+                        f"[incoming_audio] Unexpected event type: {e['event']}"
+                    )
+    new_context = client_obj.context.get_current_context()
+    if new_context:
+        await sio.emit(
+            "context",
+            {"event": "context", "payload": new_context},
+            room=sid,
+        )
+    return
+
+
+@sio.event
+async def stop_stream(sid):
+    client_obj = clients[sid]
+
+    if client_obj is None:
+        logger.error(f"No client object for {sid}")
+        await sio.disconnect(sid)
+        return {"status": "error", "message": "member_or_room_is_none"}
+
+    if client_obj.transcoder:
+        client_obj.transcoder.close = True
+        client_obj.transcoder = None
+
+    if client_obj.transcriber:
+        client_obj.transcriber.close_connection()
+
+
+@sio.event
+async def disconnect(sid):
+    client_obj = clients[sid]
+    if client_obj is None:
+        return
+
+    if client_obj.transcriber:
+        client_obj.transcriber.stop()
+
+    if client_obj.transcoder:
+        client_obj.transcoder.close = True
+        client_obj.transcoder = None
+
+    del clients[sid]

seamless-server/src/auth.py

@@ -0,0 +1,20 @@
+from src.logging import initialize_logger
+import requests
+
+logger = initialize_logger(__name__)
+
+
+def google_auth_check(token):
+    try:
+        response = requests.get(
+            "https://www.googleapis.com/oauth2/v3/tokeninfo",
+            params={"access_token": token},
+        )
+        if response.status_code == 200:
+            token_info = response.json()
+            return token_info
+        else:
+            return None
+    except Exception as e:
+        logger.info(e)
+        return None

seamless-server/src/client.py

@@ -0,0 +1,23 @@
+class Client:
+    def __init__(
+        self,
+        client_id,
+    ) -> None:
+        self.client_id = client_id
+        self.connection_status = "connected"
+        self.transcoder = None
+        self.transcriber = None
+        self.context = None
+        self.requested_output_type = None
+        self.transcoder_dynamic_config = None
+        self.manual_transcribe = None
+
+    def __str__(self) -> str:
+        return f"{self.name} (id: {self.client_id[:4]}...) ({self.connection_status})"
+
+    def to_json(self):
+        self_vars = vars(self)
+        return {
+            **self_vars,
+            "transcoder": self.transcoder is not None,
+        }

seamless-server/src/context.py

@@ -0,0 +1,83 @@
+import requests
+import json
+from threading import Thread
+from src.logging import initialize_logger
+import os
+
+# TODO get language key
+prompt = """
+Transcription: "[TRANSCRIPT]"
+Task: Give a concise, 1-sentence summary of what the speaker is talking about. 
+IMPORTANT: The summary must be in the language: [LANGUAGE].
+Return the response in JSON format with the following attribute: summary
+Response in JSON Format:
+"""
+
+
+logger = initialize_logger(__name__)
+
+
+class ContextManager:
+    def __init__(self):
+        self.text_buffer = ""
+        self.amt = 0
+        self.max_char_memory = 300
+        self.char_between_release = 200
+        self.language = None
+        self.current_context = {}
+
+    def get_current_context(self):
+        if self.current_context and self.current_context["read"] is False:
+            self.current_context["read"] = True
+            return self.current_context["text"]
+        return None
+
+    def summarize(self, text):
+        if self.language is None:
+            return
+        try:
+            url = "https://voice-llm.openai.azure.com/openai/deployments/voice-LLM/chat/completions?api-version=2023-12-01-preview"
+            headers = {
+                "Content-Type": "application/json",
+                "api-key": os.getenv("AZURE_API_KEY"),
+            }
+
+            body = {
+                "model": "gpt-35-turbo",
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": prompt.replace("[TRANSCRIPT]", text).replace(
+                            "[LANGUAGE]", self.language
+                        ),
+                    }
+                ],
+            }
+
+            response = requests.post(url, headers=headers, json=body)
+            response_data = response.json()
+            parsed = json.loads(response_data["choices"][0]["message"]["content"])[
+                "summary"
+            ]
+            self.current_context = {"text": parsed, "read": False}
+        except Exception as e:
+            logger.warning(e)
+
+    def add_text_chunk(self, text):
+        self.text_buffer += " " + text
+        cur_len = len(self.text_buffer)
+
+        # continously trim context to save memory
+        if len(self.text_buffer) > self.max_char_memory:
+            self.text_buffer = self.text_buffer[cur_len - self.max_char_memory :]
+
+        self.amt += len(text)
+        if self.amt > self.char_between_release:
+            self.amt = 0
+            thread = Thread(target=self.summarize, args=(self.text_buffer,))
+            thread.start()
+
+    def set_language(self, lang):
+        self.language = lang
+        self.text_buffer = ""
+        self.amt = 0

seamless-server/src/logging.py

@@ -0,0 +1,58 @@
+import logging
+import colorlog
+import sys
+import time
+
+
+def initialize_logger(name, level=logging.WARNING):
+    logger = logging.getLogger(name)
+    logger.propagate = False
+    handler = colorlog.StreamHandler(stream=sys.stdout)
+    formatter = colorlog.ColoredFormatter(
+        "%(log_color)s[%(asctime)s][%(levelname)s][%(module)s]:%(reset)s %(message)s",
+        reset=True,
+        log_colors={
+            "DEBUG": "cyan",
+            "INFO": "green",
+            "WARNING": "yellow",
+            "ERROR": "red",
+            "CRITICAL": "red,bg_white",
+        },
+    )
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    logger.setLevel(level)
+    return logger
+
+
+def catch_and_log_exceptions_for_sio_event_handlers(sio, logger):
+    # wrapper should have the same signature as the original function
+    def decorator(func):
+        async def catch_exception_wrapper(*args, **kwargs):
+            try:
+                return await func(*args, **kwargs)
+            except Exception as e:
+                message = f"[app_pubsub] Caught exception in '{func.__name__}' event handler:\n\n{e}"
+                logger.exception(message, stack_info=True)
+
+                try:
+                    exception_data = {
+                        "message": message,
+                        "timeEpochMs": int(time.time() * 1000),
+                    }
+
+                    # For now let's emit this to all clients. We ultimatley may want to emit it just to the room it's happening in.
+                    await sio.emit("server_exception", exception_data)
+                except Exception as inner_e:
+                    logger.exception(
+                        f"[app_pubsub] Caught exception while trying to emit server_exception event:\n{inner_e}"
+                    )
+
+                # Re-raise the exception so it's handled normally by the server
+                raise e
+
+        # Set the name of the wrapper to the name of the original function so that the socketio server can associate it with the right event
+        catch_exception_wrapper.__name__ = func.__name__
+        return catch_exception_wrapper
+
+    return decorator

seamless-server/src/room.py

@@ -0,0 +1,65 @@
+# import json
+import uuid
+
+
+class Room:
+    def __init__(self, room_id) -> None:
+        self.room_id = room_id
+        # members is a dict from client_id to Member
+        self.members = {}
+
+        # listeners and speakers are lists of client_id's
+        self.listeners = []
+        self.speakers = []
+
+    def __str__(self) -> str:
+        return f"Room {self.room_id} ({len(self.members)} member{'s' if len(self.members) == 1 else ''})"
+
+    def to_json(self):
+        varsResult = vars(self)
+        # Remember: result is just a shallow copy, so result.members === self.members
+        # Because of that, we need to jsonify self.members without writing over result.members,
+        # which we do here via dictionary unpacking (the ** operator)
+        result = {
+            **varsResult,
+            "members": {key: value.to_json() for (key, value) in self.members.items()},
+            "activeTranscoders": self.get_active_transcoders(),
+        }
+
+        return result
+
+    def get_active_connections(self):
+        return len(
+            [m for m in self.members.values() if m.connection_status == "connected"]
+        )
+
+    def get_active_transcoders(self):
+        return len([m for m in self.members.values() if m.transcoder is not None])
+
+    def get_room_status_dict(self):
+        return {
+            "activeConnections": self.get_active_connections(),
+            "activeTranscoders": self.get_active_transcoders(),
+        }
+
+
+class Member:
+    def __init__(self, client_id, session_id, name) -> None:
+        self.client_id = client_id
+        self.session_id = session_id
+        self.name = name
+        self.connection_status = "connected"
+        self.transcoder = None
+        self.requested_output_type = None
+        self.transcoder_dynamic_config = None
+        self.manual_transcribe = None
+
+    def __str__(self) -> str:
+        return f"{self.name} (id: {self.client_id[:4]}...) ({self.connection_status})"
+
+    def to_json(self):
+        self_vars = vars(self)
+        return {
+            **self_vars,
+            "transcoder": self.transcoder is not None,
+        }

seamless-server/src/simuleval_agent_directory.py

@@ -0,0 +1,171 @@
+# Creates a directory in which to look up available agents
+
+import os
+from typing import List, Optional
+from src.simuleval_transcoder import SimulevalTranscoder
+import json
+import logging
+
+logger = logging.getLogger("socketio_server_pubsub")
+
+# fmt: off
+M4T_P0_LANGS = [
+    "eng",
+    "arb", "ben", "cat", "ces", "cmn", "cym", "dan",
+    "deu", "est", "fin", "fra", "hin", "ind", "ita",
+    "jpn", "kor", "mlt", "nld", "pes", "pol", "por",
+    "ron", "rus", "slk", "spa", "swe", "swh", "tel",
+    "tgl", "tha", "tur", "ukr", "urd", "uzn", "vie",
+]
+# fmt: on
+
+
+class NoAvailableAgentException(Exception):
+    pass
+
+
+class AgentWithInfo:
+    def __init__(
+        self,
+        agent,
+        name: str,
+        modalities: List[str],
+        target_langs: List[str],
+        # Supported dynamic params are defined in StreamingTypes.ts
+        dynamic_params: List[str] = [],
+        description="",
+        has_expressive: Optional[bool] = None,
+    ):
+        self.agent = agent
+        self.has_expressive = has_expressive
+        self.name = name
+        self.description = description
+        self.modalities = modalities
+        self.target_langs = target_langs
+        self.dynamic_params = dynamic_params
+
+    def get_capabilities_for_json(self):
+        return {
+            "name": self.name,
+            "description": self.description,
+            "modalities": self.modalities,
+            "targetLangs": self.target_langs,
+            "dynamicParams": self.dynamic_params,
+        }
+
+    @classmethod
+    def load_from_json(cls, config: str):
+        """
+        Takes in JSON array of models to load in, e.g.
+        [{"name": "s2s_m4t_emma-unity2_multidomain_v0.1", "description": "M4T model that supports simultaneous S2S and S2T", "modalities": ["s2t", "s2s"], "targetLangs": ["en"]},
+        {"name": "s2s_m4t_expr-emma_v0.1", "description": "ES-EN expressive model that supports S2S and S2T", "modalities": ["s2t", "s2s"], "targetLangs": ["en"]}]
+        """
+        configs = json.loads(config)
+        agents = []
+        for config in configs:
+            agent = SimulevalTranscoder.build_agent(config["name"])
+            agents.append(
+                AgentWithInfo(
+                    agent=agent,
+                    name=config["name"],
+                    modalities=config["modalities"],
+                    target_langs=config["targetLangs"],
+                )
+            )
+        return agents
+
+
+class SimulevalAgentDirectory:
+    # Available models. These are the directories where the models can be found, and also serve as an ID for the model.
+    seamless_streaming_agent = "SeamlessStreaming"
+    seamless_agent = "Seamless"
+
+    def __init__(self):
+        self.agents = []
+        self.did_build_and_add_agents = False
+
+    def add_agent(self, agent: AgentWithInfo):
+        self.agents.append(agent)
+
+    def build_agent_if_available(self, model_id, config_name=None):
+        agent = None
+        try:
+            if config_name is not None:
+                agent = SimulevalTranscoder.build_agent(
+                    model_id,
+                    config_name=config_name,
+                )
+            else:
+                agent = SimulevalTranscoder.build_agent(
+                    model_id,
+                )
+        except Exception as e:
+            from fairseq2.assets.error import AssetError
+            logger.warning("Failed to build agent %s: %s" % (model_id, e))
+            if isinstance(e, AssetError):
+                logger.warning(
+                    "Please download gated assets and set `gated_model_dir` in the config"
+                )
+            raise e
+
+        return agent
+
+    def build_and_add_agents(self, models_override=None):
+        if self.did_build_and_add_agents:
+            return
+
+        if models_override is not None:
+            agent_infos = AgentWithInfo.load_from_json(models_override)
+            for agent_info in agent_infos:
+                self.add_agent(agent_info)
+        else:
+            s2s_agent = None
+            if os.environ.get("USE_EXPRESSIVE_MODEL", "0") == "1":
+                logger.info("Building expressive model...")
+                s2s_agent = self.build_agent_if_available(
+                    SimulevalAgentDirectory.seamless_agent,
+                    config_name="vad_s2st_sc_24khz_main.yaml",
+                )
+                has_expressive = True
+            else:
+                logger.info("Building non-expressive model...")
+                s2s_agent = self.build_agent_if_available(
+                    SimulevalAgentDirectory.seamless_streaming_agent,
+                    config_name="vad_s2st_sc_main.yaml",
+                )
+                has_expressive = False
+
+            if s2s_agent:
+                self.add_agent(
+                    AgentWithInfo(
+                        agent=s2s_agent,
+                        name=SimulevalAgentDirectory.seamless_streaming_agent,
+                        modalities=["s2t", "s2s"],
+                        target_langs=M4T_P0_LANGS,
+                        dynamic_params=["expressive"],
+                        description="multilingual expressive model that supports S2S and S2T",
+                        has_expressive=has_expressive,
+                    )
+                )
+
+        if len(self.agents) == 0:
+            logger.error(
+                "No agents were loaded. This likely means you are missing the actual model files specified in simuleval_agent_directory."
+            )
+
+        self.did_build_and_add_agents = True
+
+    def get_agent(self, name):
+        for agent in self.agents:
+            if agent.name == name:
+                return agent
+        return None
+
+    def get_agent_or_throw(self, name):
+        agent = self.get_agent(name)
+        if agent is None:
+            raise NoAvailableAgentException("No agent found with name= %s" % (name))
+        return agent
+
+    def get_agents_capabilities_list_for_json(self):
+        return [agent.get_capabilities_for_json() for agent in self.agents]

seamless-server/src/simuleval_transcoder.py

@@ -0,0 +1,409 @@
+from simuleval.utils.agent import build_system_from_dir
+from typing import Any, List, Optional, Tuple, Union
+import numpy as np
+import soundfile
+import io
+import asyncio
+from simuleval.agents.pipeline import TreeAgentPipeline
+from simuleval.agents.states import AgentStates
+from simuleval.data.segments import Segment, EmptySegment, SpeechSegment
+import threading
+from pathlib import Path
+import time
+from g2p_en import G2p
+import torch
+import traceback
+import time
+import random
+from src.logging import initialize_logger
+from .speech_and_text_output import SpeechAndTextOutput
+
+MODEL_SAMPLE_RATE = 16_000
+
+logger = initialize_logger(__name__)
+
+
+class OutputSegments:
+    def __init__(self, segments: Union[List[Segment], Segment]):
+        if isinstance(segments, Segment):
+            segments = [segments]
+        self.segments: List[Segment] = [s for s in segments]
+
+    @property
+    def is_empty(self):
+        return all(segment.is_empty for segment in self.segments)
+
+    @property
+    def finished(self):
+        return all(segment.finished for segment in self.segments)
+
+    def compute_length(self, g2p):
+        lengths = []
+        for segment in self.segments:
+            if segment.data_type == "text":
+                lengths.append(len([x for x in g2p(segment.content) if x != " "]))
+            elif segment.data_type == "speech":
+                lengths.append(len(segment.content) / MODEL_SAMPLE_RATE)
+            elif isinstance(segment, EmptySegment):
+                continue
+            else:
+                logger.warning(
+                    f"Unexpected data_type: {segment.data_type} not in 'speech', 'text'"
+                )
+        return max(lengths)
+
+    @classmethod
+    def join_output_buffer(
+        cls, buffer: List[List[Segment]], output: SpeechAndTextOutput
+    ):
+        num_segments = len(buffer[0])
+        for i in range(num_segments):
+            segment_list = [
+                buffer[j][i]
+                for j in range(len(buffer))
+                if buffer[j][i].data_type is not None
+            ]
+            if len(segment_list) == 0:
+                continue
+            if len(set(segment.data_type for segment in segment_list)) != 1:
+                logger.warning(
+                    f"Data type mismatch at {i}: {set(segment.data_type for segment in segment_list)}"
+                )
+                continue
+            data_type = segment_list[0].data_type
+            if data_type == "text":
+                if output.text is not None:
+                    logger.warning("Multiple text outputs, overwriting!")
+                output.text = " ".join([segment.content for segment in segment_list])
+            elif data_type == "speech":
+                if output.speech_samples is not None:
+                    logger.warning("Multiple speech outputs, overwriting!")
+                speech_out = []
+                for segment in segment_list:
+                    speech_out += segment.content
+                output.speech_samples = speech_out
+                output.speech_sample_rate = segment.sample_rate
+            elif isinstance(segment_list[0], EmptySegment):
+                continue
+            else:
+                logger.warning(
+                    f"Invalid output buffer data type: {data_type}, expected 'speech' or 'text"
+                )
+
+        return output
+
+    def __repr__(self) -> str:
+        repr_str = str(self.segments)
+        return f"{self.__class__.__name__}(\n\t{repr_str}\n)"
+
+
+class SimulevalTranscoder:
+    def __init__(self, agent, sample_rate, debug, buffer_limit):
+        self.agent = agent.agent
+        self.has_expressive = agent.has_expressive
+        self.input_queue = asyncio.Queue()
+        self.output_queue = asyncio.Queue()
+        self.states = self.agent.build_states()
+        if debug:
+            self.get_states_root().debug = True
+        self.incoming_sample_rate = sample_rate
+        self.close = False
+        self.g2p = G2p()
+
+        # buffer all outgoing translations within this amount of time
+        self.output_buffer_idle_ms = 5000
+        self.output_buffer_size_limit = (
+            buffer_limit  # phonemes for text, seconds for speech
+        )
+        self.output_buffer_cur_size = 0
+        self.output_buffer: List[List[Segment]] = []
+        self.speech_output_sample_rate = None
+
+        self.last_output_ts = time.time() * 1000
+        self.timeout_ms = (
+            30000  # close the transcoder thread after this amount of silence
+        )
+        self.first_input_ts = None
+        self.first_output_ts = None
+        self.debug = debug
+        self.debug_ts = f"{time.time()}_{random.randint(1000, 9999)}"
+        if self.debug:
+            debug_folder = Path(__file__).resolve().parent.parent / "debug"
+            self.test_incoming_wav = soundfile.SoundFile(
+                debug_folder / f"{self.debug_ts}_test_incoming.wav",
+                mode="w+",
+                format="WAV",
+                subtype="PCM_16",
+                samplerate=self.incoming_sample_rate,
+                channels=1,
+            )
+            self.get_states_root().test_input_segments_wav = soundfile.SoundFile(
+                debug_folder / f"{self.debug_ts}_test_input_segments.wav",
+                mode="w+",
+                format="WAV",
+                samplerate=MODEL_SAMPLE_RATE,
+                channels=1,
+            )
+
+    def get_states_root(self) -> AgentStates:
+        if isinstance(self.agent, TreeAgentPipeline):
+            # self.states is a dict
+            return self.states[self.agent.source_module]
+        else:
+            # self.states is a list
+            return self.states[0]
+
+    def reset_states(self):
+        if isinstance(self.agent, TreeAgentPipeline):
+            states_iter = self.states.values()
+        else:
+            states_iter = self.states
+        for state in states_iter:
+            state.reset()
+
+    def debug_log(self, *args):
+        if self.debug:
+            logger.info(*args)
+
+    @classmethod
+    def build_agent(cls, model_path, config_name):
+        logger.info(f"Building simuleval agent: {model_path}, {config_name}")
+        agent = build_system_from_dir(
+            Path(__file__).resolve().parent.parent / f"models/{model_path}",
+            config_name=config_name,
+        )
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        logger.warning(f"agent built on {device}")
+        agent.to(device, fp16=True)
+        logger.info(
+            f"Successfully built simuleval agent {model_path} on device {device}"
+        )
+
+        return agent
+
+    def process_incoming_bytes(self, incoming_bytes, dynamic_config):
+        # TODO: We probably want to do some validation on dynamic_config to ensure it has what we needs
+        segment, sr = self._preprocess_wav(incoming_bytes)
+        segment = SpeechSegment(
+            content=segment,
+            sample_rate=sr,
+            tgt_lang=dynamic_config.get("targetLanguage"),
+            config=dynamic_config,
+        )
+        if dynamic_config.get("expressive") is True and self.has_expressive is False:
+            logger.warning(
+                "Passing 'expressive' but the agent does not support expressive output!"
+            )
+        # # segment is array([0, 0, 0, ..., 0, 0, 0], dtype=int16)
+        self.input_queue.put_nowait(segment)
+
+    def get_input_segment(self):
+        if self.input_queue.empty():
+            return None
+        chunk = self.input_queue.get_nowait()
+        self.input_queue.task_done()
+        return chunk
+
+    def convert_waveform(
+        self,
+        waveform: Union[np.ndarray, torch.Tensor],
+        sample_rate: int,
+        normalize_volume: bool = False,
+        to_mono: bool = False,
+        to_sample_rate: Optional[int] = None,
+    ) -> Tuple[Union[np.ndarray, torch.Tensor], int]:
+        """convert a waveform:
+        - to a target sample rate
+        - from multi-channel to mono channel
+        - volume normalization
+
+        Args:
+            waveform (numpy.ndarray or torch.Tensor): 2D original waveform
+                (channels x length)
+            sample_rate (int): original sample rate
+            normalize_volume (bool): perform volume normalization
+            to_mono (bool): convert to mono channel if having multiple channels
+            to_sample_rate (Optional[int]): target sample rate
+        Returns:
+            waveform (numpy.ndarray): converted 2D waveform (channels x length)
+            sample_rate (float): target sample rate
+        """
+        try:
+            import torchaudio.sox_effects as ta_sox
+        except ImportError:
+            raise ImportError("Please install torchaudio: pip install torchaudio")
+
+        effects = []
+        if normalize_volume:
+            effects.append(["gain", "-n"])
+        if to_sample_rate is not None and to_sample_rate != sample_rate:
+            effects.append(["rate", f"{to_sample_rate}"])
+        if to_mono and waveform.shape[0] > 1:
+            effects.append(["channels", "1"])
+        if len(effects) > 0:
+            is_np_input = isinstance(waveform, np.ndarray)
+            _waveform = torch.from_numpy(waveform) if is_np_input else waveform
+            converted, converted_sample_rate = ta_sox.apply_effects_tensor(
+                _waveform, sample_rate, effects
+            )
+            if is_np_input:
+                converted = converted.numpy()
+            return converted, converted_sample_rate
+        return waveform, sample_rate
+
+    def _preprocess_wav(self, data: Any) -> Tuple[np.ndarray, int]:
+        segment, sample_rate = soundfile.read(
+            io.BytesIO(data),
+            dtype="float32",
+            always_2d=True,
+            frames=-1,
+            start=0,
+            format="RAW",
+            subtype="PCM_16",
+            samplerate=self.incoming_sample_rate,
+            channels=1,
+        )
+        if self.debug:
+            self.test_incoming_wav.seek(0, soundfile.SEEK_END)
+            self.test_incoming_wav.write(segment)
+
+        segment = segment.T
+        segment, new_sample_rate = self.convert_waveform(
+            segment,
+            sample_rate,
+            normalize_volume=False,
+            to_mono=True,
+            to_sample_rate=MODEL_SAMPLE_RATE,
+        )
+
+        assert MODEL_SAMPLE_RATE == new_sample_rate
+        segment = segment.squeeze(axis=0)
+        return segment, new_sample_rate
+
+    def process_pipeline_impl(self, input_segment):
+        try:
+            with torch.no_grad():
+                output_segment = OutputSegments(
+                    self.agent.pushpop(input_segment, self.states)
+                )
+            if (
+                self.get_states_root().first_input_ts is not None
+                and self.first_input_ts is None
+            ):
+                # TODO: this is hacky
+                self.first_input_ts = self.get_states_root().first_input_ts
+
+            if not output_segment.is_empty:
+                self.output_queue.put_nowait(output_segment)
+
+            if output_segment.finished:
+                self.debug_log("OUTPUT SEGMENT IS FINISHED. Resetting states.")
+
+                self.reset_states()
+
+                if self.debug:
+                    # when we rebuild states, this value is reset to whatever
+                    # is in the system dir config, which defaults debug=False.
+                    self.get_states_root().debug = True
+        except Exception as e:
+            logger.error(f"Got exception while processing pipeline: {e}")
+            traceback.print_exc()
+        return input_segment
+
+    def process_pipeline_loop(self):
+        if self.close:
+            return  # closes the thread
+
+        self.debug_log("processing_pipeline")
+        while not self.close:
+            input_segment = self.get_input_segment()
+            if input_segment is None:
+                if self.get_states_root().is_fresh_state:  # TODO: this is hacky
+                    time.sleep(0.3)
+                else:
+                    time.sleep(0.03)
+                continue
+            self.process_pipeline_impl(input_segment)
+        self.debug_log("finished processing_pipeline")
+
+    def process_pipeline_once(self):
+        if self.close:
+            return
+
+        self.debug_log("processing pipeline once")
+        input_segment = self.get_input_segment()
+        if input_segment is None:
+            return
+        self.process_pipeline_impl(input_segment)
+        self.debug_log("finished processing_pipeline_once")
+
+    def get_output_segment(self):
+        if self.output_queue.empty():
+            return None
+
+        output_chunk = self.output_queue.get_nowait()
+        self.output_queue.task_done()
+        return output_chunk
+
+    def start(self):
+        self.debug_log("starting transcoder in a thread")
+        threading.Thread(target=self.process_pipeline_loop).start()
+
+    def first_translation_time(self):
+        return round((self.first_output_ts - self.first_input_ts) / 1000, 2)
+
+    def get_buffered_output(self) -> SpeechAndTextOutput:
+        now = time.time() * 1000
+        self.debug_log(f"get_buffered_output queue size: {self.output_queue.qsize()}")
+        while not self.output_queue.empty():
+            tmp_out = self.get_output_segment()
+            if tmp_out and tmp_out.compute_length(self.g2p) > 0:
+                if len(self.output_buffer) == 0:
+                    self.last_output_ts = now
+                self._populate_output_buffer(tmp_out)
+                self._increment_output_buffer_size(tmp_out)
+
+                if tmp_out.finished:
+                    self.debug_log("tmp_out.finished")
+                    res = self._gather_output_buffer_data(final=True)
+                    self.debug_log(f"gathered output data: {res}")
+                    self.output_buffer = []
+                    self.increment_output_buffer_size = 0
+                    self.last_output_ts = now
+                    self.first_output_ts = now
+                    return res
+            else:
+                self.debug_log("tmp_out.compute_length is not > 0")
+
+        if len(self.output_buffer) > 0 and (
+            now - self.last_output_ts >= self.output_buffer_idle_ms
+            or self.output_buffer_cur_size >= self.output_buffer_size_limit
+        ):
+            self.debug_log(
+                "[get_buffered_output] output_buffer is not empty. getting res to return."
+            )
+            self.last_output_ts = now
+            res = self._gather_output_buffer_data(final=False)
+            self.debug_log(f"gathered output data: {res}")
+            self.output_buffer = []
+            self.output_buffer_phoneme_count = 0
+            self.first_output_ts = now
+            return res
+        else:
+            self.debug_log("[get_buffered_output] output_buffer is empty...")
+            return None
+
+    def _gather_output_buffer_data(self, final):
+        output = SpeechAndTextOutput()
+        output.final = final
+        output = OutputSegments.join_output_buffer(self.output_buffer, output)
+        return output
+
+    def _increment_output_buffer_size(self, segment: OutputSegments):
+        self.output_buffer_cur_size += segment.compute_length(self.g2p)
+
+    def _populate_output_buffer(self, segment: OutputSegments):
+        self.output_buffer.append(segment.segments)
+
+    def _compute_phoneme_count(self, string: str) -> int:
+        return len([x for x in self.g2p(string) if x != " "])

seamless-server/src/speech_and_text_output.py

@@ -0,0 +1,15 @@
+# Provides a container to return both speech and text output from our model at the same time
+
+
+class SpeechAndTextOutput:
+    def __init__(
+        self,
+        text: str = None,
+        speech_samples: list = None,
+        speech_sample_rate: float = None,
+        final: bool = False,
+    ):
+        self.text = text
+        self.speech_samples = speech_samples
+        self.speech_sample_rate = speech_sample_rate
+        self.final = final

seamless-server/src/transcoder_helpers.py

@@ -0,0 +1,44 @@
+import logging
+
+logger = logging.getLogger("socketio_server_pubsub")
+
+
+def get_transcoder_output_events(transcoder) -> list:
+    speech_and_text_output = transcoder.get_buffered_output()
+    if speech_and_text_output is None:
+        logger.debug("No output from transcoder.get_buffered_output()")
+        return []
+
+    logger.debug(f"We DID get output from the transcoder! {speech_and_text_output}")
+
+    lat = None
+
+    events = []
+
+
+    if speech_and_text_output.speech_samples:
+        events.append(
+            {
+                "event": "translation_speech",
+                "payload": speech_and_text_output.speech_samples,
+                "sample_rate": speech_and_text_output.speech_sample_rate,
+            }
+        )
+
+    if speech_and_text_output.text:
+        events.append(
+            {
+                "event": "translation_text",
+                "payload": speech_and_text_output.text,
+            }
+        )
+
+    for e in events:
+        e["eos"] = speech_and_text_output.final
+
+    # if not latency_sent:
+    #     lat = transcoder.first_translation_time()
+    #     latency_sent = True
+    #     to_send["latency"] = lat
+
+    return events

seamless-server/src/transcriber.py

@@ -0,0 +1,128 @@
+from deepgram import DeepgramClient, LiveTranscriptionEvents, LiveOptions
+import asyncio
+import os
+from src.logging import initialize_logger
+import logging
+import threading
+import time
+
+logger = initialize_logger("transcriber", level=logging.INFO)
+
+options = LiveOptions(
+    model="nova-2",
+    language="en-US",
+    smart_format=True,
+    punctuate=True,
+    # smart_format=True,
+    sample_rate=48000,
+    interim_results=True,
+)
+
+
+class Transcriber:
+    def __init__(
+        self,
+    ):
+        self.deepgram_api_key = os.getenv("DEEPGRAM_API_KEY")
+        self.deepgram = None
+        self.dg_connection = None
+        self.audio_queue = asyncio.Queue()
+        self.stop_event = threading.Event()
+
+    def process_audio(self):
+        while not self.stop_event.is_set():
+            try:
+                if self.dg_connection is None:
+                    logger.info("returned from process")
+                    return
+
+                if self.audio_queue.empty():
+                    time.sleep(0.1)
+                    continue
+
+                data = self.audio_queue.get_nowait()
+                self.dg_connection.send(data)
+                self.audio_queue.task_done()
+                logger.info("sent data to deepgram")
+            except Exception as e:
+                logger.warning(f"Error while sending data: {e}")
+                break
+
+        logger.info("Audio processing thread is stopping")
+
+    def on_transcript(self, result, *args, **kwargs):
+        try:
+            sentence = result.channel.alternatives[0].transcript
+            logger.info(f"Transcription: {sentence}")
+        except Exception as e:
+            logger.warning(e)
+
+    def close_connection(self):
+        if self.dg_connection:
+            self.dg_connection.finish()
+            self.dg_connection = None
+            logger.info("finished deepgram connection")
+
+    def stop(self):
+        self.stop_event.set()
+        self.close_connection()
+        logger.info("Requested to stop the audio processing thread")
+
+    def on_close(self, *args, **kwargs):
+        logger.info("Deepgram connection closed")
+        self.dg_connection = None
+
+    def on_utterance_end(self, utterance_end, *args, **kwargs):
+        logger.info(f"\n\n{utterance_end}\n\n")
+
+    def on_error(self, e, *args, **kwargs):
+        logger.warning(f"Deepgram error received {e}")
+        self.dg_connection = None
+
+    def start_deepgram(self):
+        try:
+            self.deepgram = DeepgramClient(self.deepgram_api_key)
+            dg_connection = self.deepgram.listen.live.v("1")
+        except Exception as e:
+            logger.warning(f"Could not open socket: {e}")
+            return
+
+        def on_message(self, result, **kwargs):
+            sentence = result.channel.alternatives[0].transcript
+            if len(sentence) == 0:
+                return
+            logger.info(f"speaker: {sentence}")
+
+        def on_metadata(self, metadata, **kwargs):
+            logger.info(f"\n\n{metadata}\n\n")
+
+        def on_utterance_end(self, utterance_end, **kwargs):
+            logger.info(f"\n\n{utterance_end}\n\n")
+
+        def on_error(self, error, **kwargs):
+            logger.info(f"\n\n{error}\n\n")
+
+        def on_close(self, **kwargs):
+            logger.info(f"\n\nclosed\n\n")
+
+        dg_connection.on(LiveTranscriptionEvents.Transcript, on_message)
+        dg_connection.on(LiveTranscriptionEvents.Metadata, on_metadata)
+        # dg_connection.on(LiveTranscriptionEvents.SpeechStarted, on_speech_started)
+        dg_connection.on(LiveTranscriptionEvents.UtteranceEnd, on_utterance_end)
+        dg_connection.on(LiveTranscriptionEvents.Error, on_error)
+        dg_connection.on(LiveTranscriptionEvents.Close, on_close)
+
+        dg_connection.start(options)
+        self.dg_connection = dg_connection
+
+        logger.info("deepgram connection opened")
+        self.process_audio()
+
+    def start(self):
+        threading.Thread(target=self.start_deepgram).start()
+
+    def send_audio(self, data):
+        try:
+            self.audio_queue.put_nowait(data)
+        except Exception as e:
+            logger.warning(e)

seamless-server/src/translate.py

@@ -0,0 +1,21 @@
+import torch
+from seamless_communication.inference import Translator
+
+
+# Initialize a Translator object with a multitask model, vocoder on the GPU.
+translator = Translator(
+    "seamlessM4T_v2_large", "vocoder_v2", torch.device("cuda:0"), torch.float16
+)
+
+
+def translate_text(text):
+    print("test")
+    # text_output, speech_output = translator.predict(
+    #     input=text,
+    #     task_str="T2ST",
+    #     tgt_lang="spa",
+    #     src_lang="eng",
+    #     text_generation_opts=None,
+    #     unit_generation_opts=None,
+    # )
+    # print(text_output)

streaming-test-app/.eslintrc.cjs

@@ -0,0 +1,18 @@
+module.exports = {
+  root: true,
+  env: {browser: true, es2020: true},
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'plugin:react-hooks/recommended',
+  ],
+  ignorePatterns: ['dist', '.eslintrc.cjs'],
+  parser: '@typescript-eslint/parser',
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': [
+      'warn',
+      {allowConstantExport: true},
+    ],
+  },
+};

streaming-test-app/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

streaming-test-app/index.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/src/assets/seamless.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Seamless Translation</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>

streaming-test-app/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "streaming-test-app",
+  "private": true,
+  "version": "0.0.14",
+  "type": "module",
+  "scripts": {
+    "dev": "vite --host --strictPort",
+    "build": "vite build",
+    "preview": "vite preview",
+    "clean:node-modules": "rm -rf node_modules/",
+    "ts-check": "tsc --noEmit",
+    "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
+    "prettier-check": "cd ../ && yarn run prettier-base --check streaming-test-app",
+    "signal": "concurrently --names \"TS,LINT,PRETTIER\" -c \"bgBlack.bold,bgRed.bold,bgCyan.bold\" \"yarn run ts-check\" \"yarn run lint\" \"yarn run prettier-check\""
+  },
+  "dependencies": {
+    "@emotion/react": "11.11.1",
+    "@emotion/styled": "11.11.0",
+    "@mui/icons-material": "5.14.3",
+    "@mui/material": "5.14.5",
+    "@react-three/drei": "^9.83.9",
+    "@react-three/fiber": "^8.14.1",
+    "@react-three/xr": "^5.7.1",
+    "amazon-cognito-identity-js": "^6.3.6",
+    "audiobuffer-to-wav": "^1.0.0",
+    "aws-sdk": "^2.1472.0",
+    "js-cookie": "^3.0.5",
+    "lodash": "4.17.21",
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0",
+    "react-google-charts": "^4.0.1",
+    "socket.io-client": "^4.7.2",
+    "three": "^0.156.1",
+    "three-mesh-ui": "^6.5.4",
+    "uuid": "^9.0.0",
+    "zustand": "^4.4.3"
+  },
+  "devDependencies": {
+    "@types/node": "^20.5.3",
+    "@types/react": "^18.2.15",
+    "@types/react-dom": "^18.2.7",
+    "@types/uuid": "^9.0.2",
+    "@typescript-eslint/eslint-plugin": "^6.0.0",
+    "@typescript-eslint/parser": "^6.0.0",
+    "@vitejs/plugin-react": "^4.0.3",
+    "concurrently": "8.2.1",
+    "eslint": "^8.45.0",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react-refresh": "^0.4.3",
+    "typescript": "5.1.6",
+    "vite": "^4.4.5"
+  }
+}

streaming-test-app/src/App.tsx

@@ -0,0 +1,57 @@
+import SocketWrapper from './SocketWrapper';
+import {ThemeProvider} from '@mui/material/styles';
+import theme from './theme';
+import StreamingInterface from './StreamingInterface';
+import CssBaseline from '@mui/material/CssBaseline';
+import {createContext, useCallback, useState} from 'react';
+import packageJson from '../package.json';
+
+console.log(`Streaming React App version: ${packageJson?.version}`);
+
+// Roboto font for mui ui library
+// import '@fontsource/roboto/300.css';
+// import '@fontsource/roboto/400.css';
+// import '@fontsource/roboto/500.css';
+// import '@fontsource/roboto/700.css';
+
+export const AppResetKeyContext = createContext<(newKey: string) => void>(
+  () => {
+    throw new Error('AppResetKeyContext not initialized');
+  },
+);
+
+function App() {
+  return (
+    <ThemeProvider theme={theme}>
+      <CssBaseline />
+      <SocketWrapper>
+        <StreamingInterface />
+      </SocketWrapper>
+    </ThemeProvider>
+  );
+}
+
+function AppWrapper() {
+  const [appResetKey, setAppResetKey] = useState<string>('[initial value]');
+  const setAppResetKeyHandler = useCallback((newKey: string) => {
+    setAppResetKey((prev) => {
+      console.warn(
+        `Resetting the app with appResetKey: ${newKey}; prevKey: ${prev}`,
+      );
+      if (prev === newKey) {
+        console.error(
+          `The appResetKey was the same as the previous key, so the app will not reset.`,
+        );
+      }
+      return newKey;
+    });
+  }, []);
+
+  return (
+    <AppResetKeyContext.Provider value={setAppResetKeyHandler}>
+      <App key={appResetKey} />
+    </AppResetKeyContext.Provider>
+  );
+}
+
+export default AppWrapper;

streaming-test-app/src/Blink.tsx

@@ -0,0 +1,41 @@
+import Box from '@mui/material/Box';
+import {useEffect, useState} from 'react';
+
+type Props = {
+  intervalMs: number;
+  children: React.ReactNode;
+  shouldBlink: boolean;
+  // display?: 'block' | 'inline' | 'inline-block';
+};
+
+export default function Blink({
+  // display = 'inline-block',
+  shouldBlink,
+  intervalMs,
+  children,
+}: Props): React.ReactElement {
+  const [cursorBlinkOn, setCursorBlinkOn] = useState(false);
+
+  useEffect(() => {
+    if (shouldBlink) {
+      const interval = setInterval(() => {
+        setCursorBlinkOn((prev) => !prev);
+      }, intervalMs);
+
+      return () => clearInterval(interval);
+    } else {
+      setCursorBlinkOn(false);
+    }
+  }, [intervalMs, shouldBlink]);
+
+  return (
+    <Box
+      component="span"
+      sx={{
+        display: 'inline-block',
+        visibility: cursorBlinkOn ? 'visible' : 'hidden',
+      }}>
+      {children}
+    </Box>
+  );
+}

streaming-test-app/src/DebugSection.tsx

@@ -0,0 +1,62 @@
+import {Chart} from 'react-google-charts';
+import debug from './debug';
+import {
+  Accordion,
+  AccordionDetails,
+  AccordionSummary,
+  Button,
+  Typography,
+} from '@mui/material';
+import {useState} from 'react';
+import ArrowDropDownIcon from '@mui/icons-material/ArrowDropDown';
+
+export default function DebugChart() {
+  const [showDebugTimings, setShowDebugTimings] = useState<boolean>(false);
+
+  const data = debug()?.getChartData();
+  const options = {
+    timeline: {
+      groupByRowLabel: true,
+    },
+  };
+
+  return (
+    <div className="horizontal-padding-sra text-chunk-sra">
+      <Accordion
+        expanded={showDebugTimings}
+        onChange={() => setShowDebugTimings(!showDebugTimings)}
+        elevation={0}
+        sx={{border: 1, borderColor: 'rgba(0, 0, 0, 0.3)'}}>
+        <AccordionSummary
+          expandIcon={<ArrowDropDownIcon />}
+          className="debug-section">
+          Debug Info
+        </AccordionSummary>
+        <AccordionDetails>
+          {data && data.length > 1 ? (
+            <>
+              <Chart
+                chartType="Timeline"
+                data={data}
+                width="100%"
+                height="400px"
+                options={options}
+              />
+              <Button
+                variant="contained"
+                sx={{marginBottom: 1}}
+                onClick={() => {
+                  debug()?.downloadInputAudio();
+                  debug()?.downloadOutputAudio();
+                }}>
+                Download Input / Ouput Audio
+              </Button>
+            </>
+          ) : (
+            <Typography>No input / output detected</Typography>
+          )}
+        </AccordionDetails>
+      </Accordion>
+    </div>
+  );
+}

streaming-test-app/src/RoomConfig.tsx

@@ -0,0 +1,271 @@
+import Stack from '@mui/material/Stack';
+import TextField from '@mui/material/TextField';
+import {isValidRoomID, isValidPartialRoomID} from './generateNewRoomID';
+import {useCallback, useEffect, useState} from 'react';
+import Button from '@mui/material/Button';
+import {useSocket} from './useSocket';
+import FormGroup from '@mui/material/FormGroup';
+import FormControlLabel from '@mui/material/FormControlLabel';
+import Checkbox from '@mui/material/Checkbox';
+import {RoomState} from './types/RoomState';
+import setURLParam from './setURLParam';
+import {getURLParams} from './URLParams';
+import {
+  JoinRoomConfig,
+  Roles,
+  ServerState,
+  StreamingStatus,
+} from './types/StreamingTypes';
+import Alert from '@mui/material/Alert';
+
+function capitalize(str: string): string {
+  return str.charAt(0).toUpperCase() + str.slice(1);
+}
+
+type Props = {
+  roomState: RoomState | null;
+  serverState: ServerState | null;
+  onJoinRoomOrUpdateRoles?: () => void;
+  streamingStatus: StreamingStatus;
+  setHasMaxUsers: (hasMaxUsers: boolean) => void;
+};
+
+export default function RoomConfig({
+  roomState,
+  serverState,
+  onJoinRoomOrUpdateRoles,
+  setHasMaxUsers,
+  streamingStatus,
+}: Props) {
+  const {socket, clientID} = useSocket();
+
+  const urlParams = getURLParams();
+  const roomIDParam = urlParams.roomID;
+  const autoJoinRoom = urlParams.autoJoin;
+
+  const [roomID, setRoomID] = useState<string>(
+    (roomIDParam ?? '').toUpperCase(),
+  );
+  const [roomIDError, setRoomIDError] = useState<boolean>(false);
+  const [roles, setRoles] = useState<{speaker: boolean; listener: boolean}>({
+    speaker: true,
+    listener: true,
+  });
+  const [lockServer, setLockServer] = useState<boolean>(false);
+  const [lockServerName, setLockServerName] = useState<string>('');
+
+  const [joinInProgress, setJoinInProgress] = useState<boolean>(false);
+  const [didAttemptAutoJoin, setDidAttemptAutoJoin] = useState<boolean>(false);
+
+  const isValidServerLock =
+    lockServer === false ||
+    (lockServerName != null && lockServerName.length > 0);
+  const isValidRoles = Object.values(roles).filter(Boolean).length > 0;
+  const isValidAllInputs =
+    isValidRoomID(roomID) && isValidRoles && isValidServerLock;
+  const roomIDFromServer = roomState?.room_id ?? null;
+
+  const onJoinRoom = useCallback(
+    (createNewRoom: boolean) => {
+      if (socket == null) {
+        console.error('Socket is null, cannot join room');
+        return;
+      }
+      console.debug(`Attempting to join roomID ${roomID}...`);
+
+      const lockServerValidated: string | null =
+        lockServer && roles['speaker'] ? lockServerName : null;
+
+      setJoinInProgress(true);
+
+      const configObject: JoinRoomConfig = {
+        roles: (Object.keys(roles) as Array<Roles>).filter(
+          (role) => roles[role] === true,
+        ),
+        lockServerName: lockServerValidated,
+      };
+
+      socket.emit(
+        'join_room',
+        clientID,
+        createNewRoom ? null : roomID,
+        configObject,
+        (result) => {
+          console.log('join_room result:', result);
+          if (result.message === 'max_users') {
+            setHasMaxUsers(true);
+            setJoinInProgress(false);
+            return;
+          } else {
+            setHasMaxUsers(false);
+          }
+          if (createNewRoom) {
+            setRoomID(result.roomID);
+          }
+          if (onJoinRoomOrUpdateRoles != null) {
+            onJoinRoomOrUpdateRoles();
+          }
+          setURLParam('roomID', result.roomID);
+          setJoinInProgress(false);
+        },
+      );
+    },
+    [
+      clientID,
+      lockServer,
+      lockServerName,
+      onJoinRoomOrUpdateRoles,
+      roles,
+      roomID,
+      socket,
+    ],
+  );
+
+  useEffect(() => {
+    if (
+      autoJoinRoom === true &&
+      didAttemptAutoJoin === false &&
+      socket != null
+    ) {
+      // We want to consider this an attempt whether or not we actually try to join, because
+      // we only want auto-join to happen on initial load
+      setDidAttemptAutoJoin(true);
+      if (
+        isValidAllInputs &&
+        joinInProgress === false &&
+        roomIDFromServer == null
+      ) {
+        console.debug('Attempting to auto-join room...');
+
+        onJoinRoom(false);
+      } else {
+        console.debug('Unable to auto-join room', {
+          isValidAllInputs,
+          joinInProgress,
+          roomIDFromServer,
+        });
+      }
+    }
+  }, [
+    autoJoinRoom,
+    didAttemptAutoJoin,
+    isValidAllInputs,
+    joinInProgress,
+    onJoinRoom,
+    roomIDFromServer,
+    socket,
+  ]);
+
+  return (
+    <Stack direction="column" spacing="12px">
+      <Stack direction="row" spacing="12px" sx={{alignItems: 'center'}}>
+        <TextField
+          size="small"
+          label="Room Code"
+          variant="outlined"
+          disabled={roomState?.room_id != null}
+          value={roomID}
+          error={roomIDError}
+          onChange={(e) => {
+            const id = e.target.value.toUpperCase();
+            if (isValidPartialRoomID(id)) {
+              setRoomIDError(false);
+              setRoomID(id);
+            } else {
+              setRoomIDError(true);
+            }
+          }}
+          sx={{width: '8em'}}
+        />
+
+        <div>
+          <Button
+            variant="contained"
+            disabled={
+              isValidAllInputs === false ||
+              joinInProgress ||
+              streamingStatus !== 'stopped'
+            }
+            onClick={() => onJoinRoom(false)}>
+            {roomState?.room_id != null ? 'Update Roles' : 'Join Room'}
+          </Button>
+        </div>
+
+        {roomState?.room_id == null && (
+          <div>
+            <Button
+              variant="contained"
+              disabled={
+                roomState?.room_id != null ||
+                joinInProgress ||
+                streamingStatus !== 'stopped'
+              }
+              onClick={() => onJoinRoom(true)}>
+              {'Create New Room'}
+            </Button>
+          </div>
+        )}
+      </Stack>
+
+      <FormGroup>
+        {Object.keys(roles).map((role) => {
+          return (
+            <FormControlLabel
+              disabled={streamingStatus !== 'stopped'}
+              key={role}
+              control={
+                <Checkbox
+                  checked={roles[role]}
+                  onChange={(event: React.ChangeEvent<HTMLInputElement>) => {
+                    setRoles((prevRoles) => ({
+                      ...prevRoles,
+                      [role]: event.target.checked,
+                    }));
+                  }}
+                />
+              }
+              label={capitalize(role)}
+            />
+          );
+        })}
+
+        {urlParams.enableServerLock && roles['speaker'] === true && (
+          <>
+            <FormControlLabel
+              disabled={streamingStatus !== 'stopped'}
+              control={
+                <Checkbox
+                  checked={lockServer}
+                  onChange={(event: React.ChangeEvent<HTMLInputElement>) => {
+                    setLockServer(event.target.checked);
+                  }}
+                />
+              }
+              label="Lock Server (prevent other users from streaming)"
+            />
+          </>
+        )}
+      </FormGroup>
+
+      {urlParams.enableServerLock &&
+        roles['speaker'] === true &&
+        lockServer && (
+          <TextField
+            disabled={streamingStatus !== 'stopped'}
+            label="Enter Your Name + Expected Lock End Time"
+            variant="outlined"
+            value={lockServerName}
+            onChange={(event: React.ChangeEvent<HTMLInputElement>) => {
+              setLockServerName(event.target.value);
+            }}
+            helperText="Locking the server will prevent anyone else from using it until you close the page, in order to maximize server performance. Please only use this for live demos."
+          />
+        )}
+
+      {serverState?.serverLock != null &&
+        serverState.serverLock.clientID === clientID && (
+          <Alert severity="success">{`The server is now locked for your use (${serverState?.serverLock?.name}). Close this window to release the lock so that others may use the server.`}</Alert>
+        )}
+    </Stack>
+  );
+}

streaming-test-app/src/SocketWrapper.tsx

@@ -0,0 +1,218 @@
+import {useContext, useEffect, useMemo, useRef, useState} from 'react';
+import socketIOClient, {Socket} from 'socket.io-client';
+import useStable from './useStable';
+import {v4 as uuidv4} from 'uuid';
+import {SocketContext} from './useSocket';
+import {AppResetKeyContext} from './App';
+import Backdrop from '@mui/material/Backdrop';
+import CircularProgress from '@mui/material/CircularProgress';
+import Typography from '@mui/material/Typography';
+import {getURLParams} from './URLParams';
+
+// The time to wait before showing a "disconnected" screen upon initial app load
+const INITIAL_DISCONNECT_SCREEN_DELAY = 2000;
+const SERVER_URL_DEFAULT = `${window.location.protocol === "https:" ? "wss" : "ws"
+                    }://${window.location.host}`;
+
+export default function SocketWrapper({children}) {
+  const [socket, setSocket] = useState<Socket | null>(null);
+  const [connected, setConnected] = useState<boolean | null>(null);
+  // Default to true:
+  const [willAttemptReconnect] = useState<boolean>(true);
+  const serverIDRef = useRef<string | null>(null);
+
+  const setAppResetKey = useContext(AppResetKeyContext);
+
+  /**
+   * Previously we had stored the clientID in local storage, but in that case
+   * if a user refreshes their page they'll still have the same clientID, and
+   * will be put back into the same room, which may be confusing if they're trying
+   * to join a new room or reset the app interface. So now clientIDs persist only as
+   * long as the react app full lifecycle
+   */
+  const clientID = useStable<string>(() => {
+    const newID = uuidv4();
+    // Set the clientID in session storage so if the page reloads the person
+    // still retains their member/room config
+    return newID;
+  });
+
+  const socketObject = useMemo(
+    () => ({socket, clientID, connected: connected ?? false}),
+    [socket, clientID, connected],
+  );
+
+  useEffect(() => {
+    const queryParams = {
+      clientID: clientID,
+    };
+
+    const serverURLFromParams = getURLParams().serverURL;
+    const serverURL = serverURLFromParams ?? SERVER_URL_DEFAULT;
+
+    console.log(
+      `Opening socket connection to ${
+        serverURL?.length === 0 ? 'window.location.host' : serverURL
+      } with query params:`,
+      queryParams,
+    );
+
+    const newSocket: Socket = socketIOClient(serverURL, {
+      query: queryParams,
+      // Normally socket.io will fallback to http polling, but we basically never
+      // want that because that'd mean awful performance. It'd be better for the app
+      // to simply break in that case and not connect.
+      transports: ['websocket'],
+      path: '/ws/socket.io'
+    });
+
+    const onServerID = (serverID: string) => {
+      console.debug('Received server ID:', serverID);
+      if (serverIDRef.current != null) {
+        if (serverIDRef.current !== serverID) {
+          console.error(
+            'Server ID changed. Resetting the app using the app key',
+          );
+          setAppResetKey(serverID);
+        }
+      }
+      serverIDRef.current = serverID;
+    };
+
+    newSocket.on('server_id', onServerID);
+
+    setSocket(newSocket);
+
+    return () => {
+      newSocket.off('server_id', onServerID);
+      console.log(
+        'Closing socket connection in the useEffect cleanup function...',
+      );
+      newSocket.disconnect();
+      setSocket(null);
+    };
+  }, [clientID, setAppResetKey]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onAny = (eventName: string, ...args) => {
+        console.debug(`[event: ${eventName}] args:`, ...args);
+      };
+
+      socket.onAny(onAny);
+
+      return () => {
+        socket.offAny(onAny);
+      };
+    }
+    return () => {};
+  }, [socket]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onConnect = (...args) => {
+        console.debug('Connected to server with args:', ...args);
+        setConnected(true);
+      };
+
+      const onConnectError = (err) => {
+        console.error(`Connection error due to ${err.message}`);
+      };
+
+      const onDisconnect = (reason) => {
+        setConnected(false);
+        console.log(`Disconnected due to ${reason}`);
+      };
+
+      socket.on('connect', onConnect);
+      socket.on('connect_error', onConnectError);
+      socket.on('disconnect', onDisconnect);
+
+      return () => {
+        socket.off('connect', onConnect);
+        socket.off('connect_error', onConnectError);
+        socket.off('disconnect', onDisconnect);
+      };
+    }
+  }, [socket]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onReconnectError = (err) => {
+        console.log(`Reconnect error due to ${err.message}`);
+      };
+
+      socket.io.on('reconnect_error', onReconnectError);
+
+      const onError = (err) => {
+        console.log(`General socket error with message ${err.message}`);
+      };
+      socket.io.on('error', onError);
+
+      const onReconnect = (attempt) => {
+        console.log(`Reconnected after ${attempt} attempt(s)`);
+      };
+      socket.io.on('reconnect', onReconnect);
+
+      const disconnectOnBeforeUnload = () => {
+        console.log('Disconnecting due to beforeunload event...');
+        socket.disconnect();
+        setSocket(null);
+      };
+      window.addEventListener('beforeunload', disconnectOnBeforeUnload);
+
+      return () => {
+        socket.io.off('reconnect_error', onReconnectError);
+        socket.io.off('error', onError);
+        socket.io.off('reconnect', onReconnect);
+        window.removeEventListener('beforeunload', disconnectOnBeforeUnload);
+      };
+    }
+  }, [clientID, setAppResetKey, socket]);
+
+  /**
+   * Wait to show the disconnected screen on initial app load
+   */
+  useEffect(() => {
+    window.setTimeout(() => {
+      setConnected((prev) => {
+        if (prev === null) {
+          return false;
+        }
+        return prev;
+      });
+    }, INITIAL_DISCONNECT_SCREEN_DELAY);
+  }, []);
+
+  return (
+    <SocketContext.Provider value={socketObject}>
+      {children}
+
+      <Backdrop
+        open={connected === false && willAttemptReconnect === true}
+        sx={{
+          color: '#fff',
+          zIndex: (theme) => theme.zIndex.drawer + 1,
+        }}>
+        <div
+          style={{
+            alignItems: 'center',
+            flexDirection: 'column',
+            textAlign: 'center',
+          }}>
+          <CircularProgress color="inherit" />
+          <Typography
+            align="center"
+            fontSize={{sm: 18, xs: 16}}
+            sx={{
+              fontFamily:
+                'ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace',
+              fontWeight: 'bold',
+            }}>
+            {'Disconnected. Attempting to reconnect...'}
+          </Typography>
+        </div>
+      </Backdrop>
+    </SocketContext.Provider>
+  );
+}

streaming-test-app/src/StreamingInterface.css

@@ -0,0 +1,56 @@
+.app-wrapper-sra {
+  display: flex;
+  flex-direction: column;
+  justify-content: center;
+  align-items: center;
+}
+
+.main-container-sra {
+  background-color: white;
+  display: flex;
+  flex-direction: column;
+  justify-content: flex-start;
+  text-align: left;
+  margin: 16px;
+  margin-bottom: 36px;
+  border-radius: 8px;
+  box-shadow: 0px 24px 30px rgba(0, 0, 0, 0.3);
+  border: 1px solid rgba(0, 0, 0, 0.05);
+  overflow: hidden;
+}
+
+.top-section-sra {
+  padding-top: 24px;
+  margin-bottom: 24px;
+  display: flex;
+  flex-direction: column;
+  justify-content: flex-start;
+}
+
+.horizontal-padding-sra {
+  padding-left: 20px;
+  padding-right: 20px;
+}
+
+.header-container-sra {
+  display: flex;
+  flex-direction: row;
+  justify-content: flex-start;
+  align-items: center;
+  margin-bottom: 24px;
+}
+
+.header-icon-sra {
+  display: block;
+  margin-right: 12px;
+}
+
+.translation-text-container-sra {
+  background-color: #f8f8f8;
+  padding-top: 12px;
+  padding-bottom: 4px;
+}
+
+.text-chunk-sra {
+  margin-bottom: 12px;
+}

streaming-test-app/src/StreamingInterface.tsx

@@ -0,0 +1,1219 @@
+import {useCallback, useEffect, useLayoutEffect, useRef, useState} from 'react';
+import Button from '@mui/material/Button';
+import Typography from '@mui/material/Typography';
+import InputLabel from '@mui/material/InputLabel';
+import FormControl from '@mui/material/FormControl';
+import Select, {SelectChangeEvent} from '@mui/material/Select';
+import MenuItem from '@mui/material/MenuItem';
+import Stack from '@mui/material/Stack';
+import seamlessLogoUrl from './assets/seamless.svg';
+import {
+  AgentCapabilities,
+  BaseResponse,
+  BrowserAudioStreamConfig,
+  DynamicConfig,
+  PartialDynamicConfig,
+  SUPPORTED_INPUT_SOURCES,
+  SUPPORTED_OUTPUT_MODES,
+  ServerExceptionData,
+  ServerSpeechData,
+  ServerState,
+  ServerTextData,
+  StartStreamEventConfig,
+  StreamingStatus,
+  SupportedInputSource,
+  SupportedOutputMode,
+  TranslationSentences,
+} from './types/StreamingTypes';
+import FormLabel from '@mui/material/FormLabel';
+import RadioGroup from '@mui/material/RadioGroup';
+import FormControlLabel from '@mui/material/FormControlLabel';
+import Radio from '@mui/material/Radio';
+import './StreamingInterface.css';
+import RoomConfig from './RoomConfig';
+import Divider from '@mui/material/Divider';
+import {useSocket} from './useSocket';
+import {RoomState} from './types/RoomState';
+import useStable from './useStable';
+import float32To16BitPCM from './float32To16BitPCM';
+import createBufferedSpeechPlayer from './createBufferedSpeechPlayer';
+import Checkbox from '@mui/material/Checkbox';
+import Alert from '@mui/material/Alert';
+import isScrolledToDocumentBottom from './isScrolledToDocumentBottom';
+import Box from '@mui/material/Box';
+import Slider from '@mui/material/Slider';
+import VolumeDown from '@mui/icons-material/VolumeDown';
+import VolumeUp from '@mui/icons-material/VolumeUp';
+import Mic from '@mui/icons-material/Mic';
+import MicOff from '@mui/icons-material/MicOff';
+import XRDialog from './react-xr/XRDialog';
+import getTranslationSentencesFromReceivedData from './getTranslationSentencesFromReceivedData';
+import {
+  sliceTranslationSentencesUpToIndex,
+  getTotalSentencesLength,
+} from './sliceTranslationSentencesUtils';
+import Blink from './Blink';
+import {CURSOR_BLINK_INTERVAL_MS} from './cursorBlinkInterval';
+import {getURLParams} from './URLParams';
+import debug from './debug';
+import DebugSection from './DebugSection';
+import Switch from '@mui/material/Switch';
+import Grid from '@mui/material/Grid';
+import {getLanguageFromThreeLetterCode} from './languageLookup';
+import HeadphonesIcon from '@mui/icons-material/Headphones';
+
+const AUDIO_STREAM_DEFAULTS = {
+  userMedia: {
+    echoCancellation: false,
+    noiseSuppression: true,
+  },
+  displayMedia: {
+    echoCancellation: false,
+    noiseSuppression: false,
+  },
+} as const;
+
+async function requestUserMediaAudioStream(
+  config: BrowserAudioStreamConfig = AUDIO_STREAM_DEFAULTS['userMedia'],
+) {
+  const stream = await navigator.mediaDevices.getUserMedia({
+    audio: {...config, channelCount: 1},
+  });
+  console.debug(
+    '[requestUserMediaAudioStream] stream created with settings:',
+    stream.getAudioTracks()?.[0]?.getSettings(),
+  );
+  return stream;
+}
+
+async function requestDisplayMediaAudioStream(
+  config: BrowserAudioStreamConfig = AUDIO_STREAM_DEFAULTS['displayMedia'],
+) {
+  const stream = await navigator.mediaDevices.getDisplayMedia({
+    audio: {...config, channelCount: 1},
+  });
+  console.debug(
+    '[requestDisplayMediaAudioStream] stream created with settings:',
+    stream.getAudioTracks()?.[0]?.getSettings(),
+  );
+  return stream;
+}
+
+const buttonLabelMap: {[key in StreamingStatus]: string} = {
+  stopped: 'Start Streaming',
+  running: 'Stop Streaming',
+  starting: 'Starting...',
+};
+
+const BUFFER_LIMIT = 1;
+
+const SCROLLED_TO_BOTTOM_THRESHOLD_PX = 36;
+
+const GAIN_MULTIPLIER_OVER_1 = 3;
+
+const getGainScaledValue = (value) =>
+  value > 1 ? (value - 1) * GAIN_MULTIPLIER_OVER_1 + 1 : value;
+
+const TOTAL_ACTIVE_TRANSCODER_WARNING_THRESHOLD = 2;
+
+const MAX_SERVER_EXCEPTIONS_TRACKED = 500;
+
+export const TYPING_ANIMATION_DELAY_MS = 6;
+
+export default function StreamingInterface() {
+  const urlParams = getURLParams();
+  const debugParam = urlParams.debug;
+  const [animateTextDisplay, setAnimateTextDisplay] = useState<boolean>(
+    urlParams.animateTextDisplay,
+  );
+
+  const socketObject = useSocket();
+  const {socket, clientID} = socketObject;
+
+  const [serverState, setServerState] = useState<ServerState | null>(null);
+  const [agent, setAgent] = useState<AgentCapabilities | null>(null);
+  const model = agent?.name ?? null;
+  const agentsCapabilities: Array<AgentCapabilities> =
+    serverState?.agentsCapabilities ?? [];
+  const currentAgent: AgentCapabilities | null =
+    agentsCapabilities.find((agent) => agent.name === model) ?? null;
+
+  const [serverExceptions, setServerExceptions] = useState<
+    Array<ServerExceptionData>
+  >([]);
+  const [roomState, setRoomState] = useState<RoomState | null>(null);
+  const roomID = roomState?.room_id ?? null;
+  const isSpeaker =
+    (clientID != null && roomState?.speakers.includes(clientID)) ?? false;
+  const isListener =
+    (clientID != null && roomState?.listeners.includes(clientID)) ?? false;
+
+  const [streamingStatus, setStreamingStatus] =
+    useState<StreamingStatus>('stopped');
+
+  const isStreamConfiguredRef = useRef<boolean>(false);
+  const [hasMaxUsers, setHasMaxUsers] = useState<boolean>(false);
+
+  const [outputMode, setOutputMode] = useState<SupportedOutputMode>('s2s&t');
+  const [inputSource, setInputSource] =
+    useState<SupportedInputSource>('userMedia');
+  const [enableNoiseSuppression, setEnableNoiseSuppression] = useState<
+    boolean | null
+  >(null);
+  const [enableEchoCancellation, setEnableEchoCancellation] = useState<
+    boolean | null
+  >(null);
+
+  // Dynamic Params:
+  const [targetLang, setTargetLang] = useState<string | null>(null);
+  const [enableExpressive, setEnableExpressive] = useState<boolean | null>(
+    null,
+  );
+
+  const [serverDebugFlag, setServerDebugFlag] = useState<boolean>(
+    debugParam ?? false,
+  );
+
+  const [receivedData, setReceivedData] = useState<Array<ServerTextData>>([]);
+  const [
+    translationSentencesAnimatedIndex,
+    setTranslationSentencesAnimatedIndex,
+  ] = useState<number>(0);
+
+  const lastTranslationResultRef = useRef<HTMLDivElement | null>(null);
+
+  const [inputStream, setInputStream] = useState<MediaStream | null>(null);
+  const [inputStreamSource, setInputStreamSource] =
+    useState<MediaStreamAudioSourceNode | null>(null);
+  const audioContext = useStable<AudioContext>(() => new AudioContext());
+  const [scriptNodeProcessor, setScriptNodeProcessor] =
+    useState<ScriptProcessorNode | null>(null);
+
+  const [muted, setMuted] = useState<boolean>(false);
+  // The onaudioprocess script needs an up-to-date reference to the muted state, so
+  // we use a ref here and keep it in sync via useEffect
+  const mutedRef = useRef<boolean>(muted);
+  useEffect(() => {
+    mutedRef.current = muted;
+  }, [muted]);
+
+  const [gain, setGain] = useState<number>(1);
+
+  const isScrolledToBottomRef = useRef<boolean>(isScrolledToDocumentBottom());
+
+  // Some config options must be set when starting streaming and cannot be chaned dynamically.
+  // This controls whether they are disabled or not
+  const streamFixedConfigOptionsDisabled =
+    streamingStatus !== 'stopped' || roomID == null;
+
+  const bufferedSpeechPlayer = useStable(() => {
+    const player = createBufferedSpeechPlayer({
+      onStarted: () => {
+        console.debug('📢 PLAYBACK STARTED 📢');
+      },
+      onEnded: () => {
+        console.debug('🛑 PLAYBACK ENDED 🛑');
+      },
+    });
+
+    // Start the player now so it eagerly plays audio when it arrives
+    player.start();
+    return player;
+  });
+
+  const translationSentencesBase: TranslationSentences =
+    getTranslationSentencesFromReceivedData(receivedData);
+
+  const translationSentencesBaseTotalLength = getTotalSentencesLength(
+    translationSentencesBase,
+  );
+
+  const translationSentences: TranslationSentences = animateTextDisplay
+    ? sliceTranslationSentencesUpToIndex(
+        translationSentencesBase,
+        translationSentencesAnimatedIndex,
+      )
+    : translationSentencesBase;
+
+  // We want the blinking cursor to show before any text has arrived, so let's add an empty string so that the cursor shows up
+  const translationSentencesWithEmptyStartingString =
+    streamingStatus === 'running' && translationSentences.length === 0
+      ? ['']
+      : translationSentences;
+
+  /******************************************
+   * Event Handlers
+   ******************************************/
+
+  const setAgentAndUpdateParams = useCallback(
+    (newAgent: AgentCapabilities | null) => {
+      setAgent((prevAgent) => {
+        if (prevAgent?.name !== newAgent?.name) {
+          setTargetLang(newAgent?.targetLangs[0] ?? null);
+          setEnableExpressive(null);
+        }
+        return newAgent;
+      });
+    },
+    [],
+  );
+
+  const onSetDynamicConfig = useCallback(
+    async (partialConfig: PartialDynamicConfig) => {
+      return new Promise<void>((resolve, reject) => {
+        if (socket == null) {
+          reject(new Error('[onSetDynamicConfig] socket is null '));
+          return;
+        }
+
+        socket.emit(
+          'set_dynamic_config',
+          partialConfig,
+          (result: BaseResponse) => {
+            console.log('[emit result: set_dynamic_config]', result);
+            if (result.status === 'ok') {
+              resolve();
+            } else {
+              reject();
+            }
+          },
+        );
+      });
+    },
+    [socket],
+  );
+
+  const configureStreamAsync = ({sampleRate}: {sampleRate: number}) => {
+    return new Promise<void>((resolve, reject) => {
+      if (socket == null) {
+        reject(new Error('[configureStreamAsync] socket is null '));
+        return;
+      }
+      const modelName = agent?.name ?? null;
+      if (modelName == null) {
+        reject(new Error('[configureStreamAsync] modelName is null '));
+        return;
+      }
+
+      const config: StartStreamEventConfig = {
+        event: 'config',
+        rate: sampleRate,
+        model_name: modelName,
+        debug: serverDebugFlag,
+        // synchronous processing isn't implemented on the v2 pubsub server, so hardcode this to true
+        async_processing: true,
+        buffer_limit: BUFFER_LIMIT,
+        model_type: outputMode,
+      };
+
+      console.log('[configureStreamAsync] sending config', config);
+
+      socket.emit('configure_stream', config, (statusObject) => {
+        if (statusObject.status === 'ok') {
+          isStreamConfiguredRef.current = true;
+          console.debug(
+            '[configureStreamAsync] stream configured!',
+            statusObject,
+          );
+          resolve();
+        } else {
+          isStreamConfiguredRef.current = false;
+          reject(
+            new Error(
+              `[configureStreamAsync] configure_stream returned status: ${statusObject.status}`,
+            ),
+          );
+          return;
+        }
+      });
+    });
+  };
+
+  const startStreaming = async () => {
+    if (streamingStatus !== 'stopped') {
+      console.warn(
+        `Attempting to start stream when status is ${streamingStatus}`,
+      );
+      return;
+    }
+
+    setStreamingStatus('starting');
+
+    if (audioContext.state === 'suspended') {
+      console.warn('audioContext was suspended! resuming...');
+      await audioContext.resume();
+    }
+
+    let stream: MediaStream | null = null;
+
+    try {
+      if (inputSource === 'userMedia') {
+        stream = await requestUserMediaAudioStream({
+          noiseSuppression:
+            enableNoiseSuppression ??
+            AUDIO_STREAM_DEFAULTS['userMedia'].noiseSuppression,
+          echoCancellation:
+            enableEchoCancellation ??
+            AUDIO_STREAM_DEFAULTS['userMedia'].echoCancellation,
+        });
+      } else if (inputSource === 'displayMedia') {
+        stream = await requestDisplayMediaAudioStream({
+          noiseSuppression:
+            enableNoiseSuppression ??
+            AUDIO_STREAM_DEFAULTS['displayMedia'].noiseSuppression,
+          echoCancellation:
+            enableEchoCancellation ??
+            AUDIO_STREAM_DEFAULTS['displayMedia'].echoCancellation,
+        });
+      } else {
+        throw new Error(`Unsupported input source requested: ${inputSource}`);
+      }
+      setInputStream(stream);
+    } catch (e) {
+      console.error('[startStreaming] media stream request failed:', e);
+      setStreamingStatus('stopped');
+      return;
+    }
+
+    const mediaStreamSource = audioContext.createMediaStreamSource(stream);
+    setInputStreamSource(mediaStreamSource);
+    /**
+     * NOTE: This currently uses a deprecated way of processing the audio (createScriptProcessor), but
+     * which is easy and convenient for our purposes.
+     *
+     * Documentation for the deprecated way of doing it is here: https://developer.mozilla.org/en-US/docs/Web/API/BaseAudioContext/createScriptProcessor
+     *
+     * In an ideal world this would be migrated to something like this SO answer: https://stackoverflow.com/a/65448287
+     */
+    const scriptProcessor = audioContext.createScriptProcessor(16384, 1, 1);
+    setScriptNodeProcessor(scriptProcessor);
+
+    scriptProcessor.onaudioprocess = (event) => {
+      if (isStreamConfiguredRef.current === false) {
+        console.debug('[onaudioprocess] stream is not configured yet!');
+        return;
+      }
+      if (socket == null) {
+        console.warn('[onaudioprocess] socket is null in onaudioprocess');
+        return;
+      }
+
+      if (mutedRef.current) {
+        // We still want to send audio to the server when we're muted to ensure we
+        // get any remaining audio back from the server, so let's pass an array length 1 with a value of 0
+        const mostlyEmptyInt16Array = new Int16Array(1);
+        socket.emit('incoming_audio', mostlyEmptyInt16Array);
+      } else {
+        const float32Audio = event.inputBuffer.getChannelData(0);
+        const pcm16Audio = float32To16BitPCM(float32Audio);
+        socket.emit('incoming_audio', pcm16Audio);
+      }
+
+      debug()?.sentAudio(event);
+    };
+
+    mediaStreamSource.connect(scriptProcessor);
+    scriptProcessor.connect(audioContext.destination);
+
+    bufferedSpeechPlayer.start();
+
+    try {
+      if (targetLang == null) {
+        throw new Error('[startStreaming] targetLang cannot be nullish');
+      }
+
+      // When we are starting the stream we want to pass all the dynamic config values
+      // available before actually configuring and starting the stream
+      const fullDynamicConfig: DynamicConfig = {
+        targetLanguage: targetLang,
+        expressive: enableExpressive,
+      };
+
+      await onSetDynamicConfig(fullDynamicConfig);
+
+      // NOTE: this needs to be the *audioContext* sample rate, not the sample rate of the input stream. Not entirely sure why.
+      await configureStreamAsync({
+        sampleRate: audioContext.sampleRate,
+      });
+    } catch (e) {
+      console.error('configureStreamAsync failed', e);
+      setStreamingStatus('stopped');
+      return;
+    }
+
+    setStreamingStatus('running');
+  };
+
+  const stopStreaming = useCallback(async () => {
+    if (streamingStatus === 'stopped') {
+      console.warn(
+        `Attempting to stop stream when status is ${streamingStatus}`,
+      );
+      return;
+    }
+
+    // Stop the speech playback right away
+    bufferedSpeechPlayer.stop();
+
+    if (inputStreamSource == null || scriptNodeProcessor == null) {
+      console.error(
+        'inputStreamSource || scriptNodeProcessor is null in stopStreaming',
+      );
+    } else {
+      inputStreamSource.disconnect(scriptNodeProcessor);
+      scriptNodeProcessor.disconnect(audioContext.destination);
+
+      // Release the mic input so we stop showing the red recording icon in the browser
+      inputStream?.getTracks().forEach((track) => track.stop());
+    }
+
+    if (socket == null) {
+      console.warn('Unable to emit stop_stream because socket is null');
+    } else {
+      socket.emit('stop_stream', (result) => {
+        console.debug('[emit result: stop_stream]', result);
+      });
+    }
+
+    setStreamingStatus('stopped');
+  }, [
+    audioContext.destination,
+    bufferedSpeechPlayer,
+    inputStream,
+    inputStreamSource,
+    scriptNodeProcessor,
+    socket,
+    streamingStatus,
+  ]);
+
+  const onClearTranscriptForAll = useCallback(() => {
+    if (socket != null) {
+      socket.emit('clear_transcript_for_all');
+    }
+  }, [socket]);
+
+  /******************************************
+   * Effects
+   ******************************************/
+
+  useEffect(() => {
+    if (socket == null) {
+      return;
+    }
+
+    const onRoomStateUpdate = (roomState: RoomState) => {
+      setRoomState(roomState);
+    };
+
+    socket.on('room_state_update', onRoomStateUpdate);
+
+    return () => {
+      socket.off('room_state_update', onRoomStateUpdate);
+    };
+  }, [socket]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onTranslationText = (data: ServerTextData) => {
+        setReceivedData((prev) => [...prev, data]);
+        debug()?.receivedText(data.payload);
+      };
+
+      const onTranslationSpeech = (data: ServerSpeechData) => {
+        bufferedSpeechPlayer.addAudioToBuffer(data.payload, data.sample_rate);
+      };
+
+      socket.on('translation_text', onTranslationText);
+      socket.on('translation_speech', onTranslationSpeech);
+
+      return () => {
+        socket.off('translation_text', onTranslationText);
+        socket.off('translation_speech', onTranslationSpeech);
+      };
+    }
+  }, [bufferedSpeechPlayer, socket]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onServerStateUpdate = (newServerState: ServerState) => {
+        setServerState(newServerState);
+
+        // If a client creates a server lock, we want to stop streaming if we're not them
+        if (
+          newServerState.serverLock?.isActive === true &&
+          newServerState.serverLock?.clientID !== clientID &&
+          streamingStatus === 'running'
+        ) {
+          stopStreaming();
+        }
+
+        const firstAgentNullable = newServerState.agentsCapabilities[0];
+        if (agent == null && firstAgentNullable != null) {
+          setAgentAndUpdateParams(firstAgentNullable);
+        }
+      };
+
+      socket.on('server_state_update', onServerStateUpdate);
+
+      return () => {
+        socket.off('server_state_update', onServerStateUpdate);
+      };
+    }
+  }, [
+    agent,
+    clientID,
+    setAgentAndUpdateParams,
+    socket,
+    stopStreaming,
+    streamingStatus,
+  ]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onServerException = (
+        exceptionDataWithoutClientTime: ServerExceptionData,
+      ) => {
+        const exceptionData = {
+          ...exceptionDataWithoutClientTime,
+          timeStringClient: new Date(
+            exceptionDataWithoutClientTime['timeEpochMs'],
+          ).toLocaleString(),
+        };
+
+        setServerExceptions((prev) =>
+          [exceptionData, ...prev].slice(0, MAX_SERVER_EXCEPTIONS_TRACKED),
+        );
+        console.error(
+          `[server_exception] The server encountered an exception: ${exceptionData['message']}`,
+          exceptionData,
+        );
+      };
+
+      socket.on('server_exception', onServerException);
+
+      return () => {
+        socket.off('server_exception', onServerException);
+      };
+    }
+  }, [socket]);
+
+  useEffect(() => {
+    if (socket != null) {
+      const onClearTranscript = () => {
+        setReceivedData([]);
+        setTranslationSentencesAnimatedIndex(0);
+      };
+
+      socket.on('clear_transcript', onClearTranscript);
+
+      return () => {
+        socket.off('clear_transcript', onClearTranscript);
+      };
+    }
+  }, [socket]);
+
+  useEffect(() => {
+    const onScroll = () => {
+      if (isScrolledToDocumentBottom(SCROLLED_TO_BOTTOM_THRESHOLD_PX)) {
+        isScrolledToBottomRef.current = true;
+        return;
+      }
+      isScrolledToBottomRef.current = false;
+      return;
+    };
+
+    document.addEventListener('scroll', onScroll);
+
+    return () => {
+      document.removeEventListener('scroll', onScroll);
+    };
+  }, []);
+
+  useLayoutEffect(() => {
+    if (
+      lastTranslationResultRef.current != null &&
+      isScrolledToBottomRef.current
+    ) {
+      // Scroll the div to the most recent entry
+      lastTranslationResultRef.current.scrollIntoView();
+    }
+    // Run the effect every time data is received, so that
+    // we scroll to the bottom even if we're just adding text to
+    // a pre-existing chunk
+  }, [receivedData]);
+
+  useEffect(() => {
+    if (!animateTextDisplay) {
+      return;
+    }
+
+    if (
+      translationSentencesAnimatedIndex < translationSentencesBaseTotalLength
+    ) {
+      const timeout = setTimeout(() => {
+        setTranslationSentencesAnimatedIndex((prev) => prev + 1);
+        debug()?.startRenderText();
+      }, TYPING_ANIMATION_DELAY_MS);
+
+      return () => clearTimeout(timeout);
+    } else {
+      debug()?.endRenderText();
+    }
+  }, [
+    animateTextDisplay,
+    translationSentencesAnimatedIndex,
+    translationSentencesBaseTotalLength,
+  ]);
+
+  /******************************************
+   * Sub-components
+   ******************************************/
+
+  const volumeSliderNode = (
+    <Stack
+      spacing={2}
+      direction="row"
+      sx={{mb: 1, width: '100%'}}
+      alignItems="center">
+      <VolumeDown color="primary" />
+      <Slider
+        aria-label="Volume"
+        defaultValue={1}
+        scale={getGainScaledValue}
+        min={0}
+        max={3}
+        step={0.1}
+        marks={[
+          {value: 0, label: '0%'},
+          {value: 1, label: '100%'},
+          {value: 2, label: '400%'},
+          {value: 3, label: '700%'},
+        ]}
+        valueLabelFormat={(value) => `${(value * 100).toFixed(0)}%`}
+        valueLabelDisplay="auto"
+        value={gain}
+        onChange={(_event: Event, newValue: number | number[]) => {
+          if (typeof newValue === 'number') {
+            const scaledGain = getGainScaledValue(newValue);
+            // We want the actual gain node to use the scaled value
+            bufferedSpeechPlayer.setGain(scaledGain);
+            // But we want react state to keep track of the non-scaled value
+            setGain(newValue);
+          } else {
+            console.error(
+              `[volume slider] Unexpected non-number value: ${newValue}`,
+            );
+          }
+        }}
+      />
+      <VolumeUp color="primary" />
+    </Stack>
+  );
+
+  const xrDialogComponent = (
+    <XRDialog
+      animateTextDisplay={
+        animateTextDisplay &&
+        translationSentencesAnimatedIndex == translationSentencesBaseTotalLength
+      }
+      bufferedSpeechPlayer={bufferedSpeechPlayer}
+      translationSentences={translationSentences}
+      roomState={roomState}
+      roomID={roomID}
+      startStreaming={startStreaming}
+      stopStreaming={stopStreaming}
+      debugParam={debugParam}
+      onARHidden={() => {
+        setAnimateTextDisplay(urlParams.animateTextDisplay);
+      }}
+      onARVisible={() => setAnimateTextDisplay(false)}
+    />
+  );
+
+  return (
+    <div className="app-wrapper-sra">
+      <Box
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+        // @ts-ignore Not sure why it's complaining about complexity here
+        sx={{width: '100%', maxWidth: '660px', minWidth: '320px'}}>
+        <div className="main-container-sra">
+          <div className="top-section-sra horizontal-padding-sra">
+            <div className="header-container-sra">
+              <img
+                src={seamlessLogoUrl}
+                className="header-icon-sra"
+                alt="Seamless Translation Logo"
+                height={24}
+                width={24}
+              />
+
+              <div>
+                <Typography variant="h1" sx={{color: '#65676B'}}>
+                  Seamless Translation
+                </Typography>
+              </div>
+            </div>
+            <div className="header-container-sra">
+              <div>
+                <Typography variant="body2" sx={{color: '#65676B'}}>
+                  Welcome! This space is limited to one user at a time.
+                  If using the live HF space, sharing room code to listeners on another
+                  IP address may not work because it's running on different replicas.
+                  Use headphones if you are both speaker and listener to prevent feedback.
+                  <br/>
+                  If max users reached, please duplicate the space <a target="_blank" rel="noopener noreferrer" href="https://huggingface.co/spaces/facebook/seamless-streaming?duplicate=true">here</a>.
+                  In your duplicated space, join a room as speaker or listener (or both),
+                  and share the room code to invite listeners.
+                  <br/>
+                  Check out the seamless_communication <a target="_blank" rel="noopener noreferrer" href="https://github.com/facebookresearch/seamless_communication/tree/main">README</a> for more information.
+                  <br/>
+                  SeamlessStreaming model is a research model and is not released
+                  for production deployment. It is important to use a microphone with
+                  noise cancellation (for e.g. a smartphone), otherwise you may see model hallucination on noises.
+                  It works best if you pause every couple of sentences, or you may wish adjust the VAD threshold
+                  in the model config. The real-time performance will degrade
+                  if you try streaming multiple speakers at the same time.
+                </Typography>
+              </div>
+            </div>
+            <Stack spacing="22px" direction="column">
+              <Box>
+                <RoomConfig
+                  roomState={roomState}
+                  serverState={serverState}
+                  streamingStatus={streamingStatus}
+                  setHasMaxUsers={setHasMaxUsers}
+                  onJoinRoomOrUpdateRoles={() => {
+                    // If the user has switched from speaker to listener we need to tell the
+                    // player to play eagerly, since currently the listener doesn't have any stop/start controls
+                    bufferedSpeechPlayer.start();
+                  }}
+                />
+
+                {isListener && !isSpeaker && (
+                  <Box
+                    sx={{
+                      paddingX: 6,
+                      paddingBottom: 2,
+                      marginY: 2,
+                      display: 'flex',
+                      flexDirection: 'column',
+                      alignItems: 'center',
+                    }}>
+                    {volumeSliderNode}
+                  </Box>
+                )}
+              </Box>
+
+              {isSpeaker && (
+                <>
+                  <Divider />
+
+                  <Stack spacing="12px" direction="column">
+                    <FormLabel id="output-modes-radio-group-label">
+                      Model
+                    </FormLabel>
+                    <FormControl
+                      disabled={
+                        streamFixedConfigOptionsDisabled ||
+                        agentsCapabilities.length === 0
+                      }
+                      fullWidth
+                      sx={{minWidth: '14em'}}>
+                      <InputLabel id="model-selector-input-label">
+                        Model
+                      </InputLabel>
+                      <Select
+                        labelId="model-selector-input-label"
+                        label="Model"
+                        onChange={(e: SelectChangeEvent) => {
+                          const newAgent =
+                            agentsCapabilities.find(
+                              (agent) => e.target.value === agent.name,
+                            ) ?? null;
+                          if (newAgent == null) {
+                            console.error(
+                              'Unable to find agent with name',
+                              e.target.value,
+                            );
+                          }
+                          setAgentAndUpdateParams(newAgent);
+                        }}
+                        value={model ?? ''}>
+                        {agentsCapabilities.map((agent) => (
+                          <MenuItem value={agent.name} key={agent.name}>
+                            {agent.name}
+                          </MenuItem>
+                        ))}
+                      </Select>
+                    </FormControl>
+
+                  </Stack>
+
+                  <Stack spacing={0.5}>
+                    <FormLabel id="output-modes-radio-group-label">
+                      Output
+                    </FormLabel>
+
+                    <Box sx={{paddingTop: 2, paddingBottom: 1}}>
+                      <FormControl fullWidth sx={{minWidth: '14em'}}>
+                        <InputLabel id="target-selector-input-label">
+                          Target Language
+                        </InputLabel>
+                        <Select
+                          labelId="target-selector-input-label"
+                          label="Target Language"
+                          onChange={(e: SelectChangeEvent) => {
+                            setTargetLang(e.target.value);
+                            onSetDynamicConfig({
+                              targetLanguage: e.target.value,
+                            });
+                          }}
+                          value={targetLang ?? ''}>
+                          {currentAgent?.targetLangs.map((langCode) => (
+                            <MenuItem value={langCode} key={langCode}>
+                              {getLanguageFromThreeLetterCode(langCode) != null
+                                ? `${getLanguageFromThreeLetterCode(
+                                    langCode,
+                                  )} (${langCode})`
+                                : langCode}
+                            </MenuItem>
+                          ))}
+                        </Select>
+                      </FormControl>
+                    </Box>
+
+                    <Grid container>
+                      <Grid item xs={12} sm={4}>
+                        <FormControl
+                          disabled={streamFixedConfigOptionsDisabled}>
+                          <RadioGroup
+                            aria-labelledby="output-modes-radio-group-label"
+                            value={outputMode}
+                            onChange={(e) =>
+                              setOutputMode(
+                                e.target.value as SupportedOutputMode,
+                              )
+                            }
+                            name="output-modes-radio-buttons-group">
+                            {
+                              // TODO: Use supported modalities from agentCapabilities
+                              SUPPORTED_OUTPUT_MODES.map(({value, label}) => (
+                                <FormControlLabel
+                                  key={value}
+                                  value={value}
+                                  control={<Radio />}
+                                  label={label}
+                                />
+                              ))
+                            }
+                          </RadioGroup>
+                        </FormControl>
+                      </Grid>
+
+                      <Grid item xs={12} sm={8}>
+                        <Stack
+                          direction="column"
+                          spacing={1}
+                          alignItems="flex-start"
+                          sx={{flexGrow: 1}}>
+                          {currentAgent?.dynamicParams?.includes(
+                            'expressive',
+                          ) && (
+                            <FormControlLabel
+                              control={
+                                <Switch
+                                  checked={enableExpressive ?? false}
+                                  onChange={(
+                                    event: React.ChangeEvent<HTMLInputElement>,
+                                  ) => {
+                                    const newValue = event.target.checked;
+                                    setEnableExpressive(newValue);
+                                    onSetDynamicConfig({
+                                      expressive: newValue,
+                                    });
+                                  }}
+                                />
+                              }
+                              label="Expressive"
+                            />
+                          )}
+
+                          {isListener && (
+                            <Box
+                              sx={{
+                                flexGrow: 1,
+                                paddingX: 1.5,
+                                paddingY: 1.5,
+                                width: '100%',
+                              }}>
+                              {volumeSliderNode}
+                            </Box>
+                          )}
+                        </Stack>
+                      </Grid>
+                    </Grid>
+                  </Stack>
+
+                  <Stack
+                    direction="row"
+                    spacing={2}
+                    justifyContent="space-between">
+                    <Box sx={{flex: 1}}>
+                      <FormControl disabled={streamFixedConfigOptionsDisabled}>
+                        <FormLabel id="input-source-radio-group-label">
+                          Input Source
+                        </FormLabel>
+                        <RadioGroup
+                          aria-labelledby="input-source-radio-group-label"
+                          value={inputSource}
+                          onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
+                            setInputSource(
+                              e.target.value as SupportedInputSource,
+                            )
+                          }
+                          name="input-source-radio-buttons-group">
+                          {SUPPORTED_INPUT_SOURCES.map(({label, value}) => (
+                            <FormControlLabel
+                              key={value}
+                              value={value}
+                              control={<Radio />}
+                              label={label}
+                            />
+                          ))}
+                        </RadioGroup>
+                      </FormControl>
+                    </Box>
+
+                    <Box sx={{flex: 1, flexGrow: 2}}>
+                    <FormControl disabled={streamFixedConfigOptionsDisabled}>
+                        <FormLabel>Options</FormLabel>
+                        <FormControlLabel
+                          control={
+                            <Checkbox
+                              checked={
+                                enableNoiseSuppression ??
+                                AUDIO_STREAM_DEFAULTS[inputSource]
+                                  .noiseSuppression
+                              }
+                              onChange={(
+                                event: React.ChangeEvent<HTMLInputElement>,
+                              ) =>
+                                setEnableNoiseSuppression(event.target.checked)
+                              }
+                            />
+                          }
+                          label="Noise Suppression"
+                        />
+                        <FormControlLabel
+                          control={
+                            <Checkbox
+                              checked={
+                                enableEchoCancellation ??
+                                AUDIO_STREAM_DEFAULTS[inputSource]
+                                  .echoCancellation
+                              }
+                              onChange={(
+                                event: React.ChangeEvent<HTMLInputElement>,
+                              ) =>
+                                setEnableEchoCancellation(event.target.checked)
+                              }
+                            />
+                          }
+                          label="Echo Cancellation (not recommended)"
+                        />
+                        <FormControlLabel
+                          control={
+                            <Checkbox
+                              checked={serverDebugFlag}
+                              onChange={(
+                                event: React.ChangeEvent<HTMLInputElement>,
+                              ) => setServerDebugFlag(event.target.checked)}
+                            />
+                          }
+                          label="Enable Server Debugging"
+                        />
+                      </FormControl>
+                    </Box>
+                  </Stack>
+
+                  {isSpeaker &&
+                    isListener &&
+                    inputSource === 'userMedia' &&
+                    !enableEchoCancellation &&
+                    gain !== 0 && (
+                      <div>
+                        <Alert severity="warning" icon={<HeadphonesIcon />}>
+                          Headphones required to prevent feedback.
+                        </Alert>
+                      </div>
+                    )}
+
+                  {isSpeaker && enableEchoCancellation && (
+                    <div>
+                      <Alert severity="warning">
+                        We don't recommend using echo cancellation as it may
+                        distort the input audio. If possible, use headphones and
+                        disable echo cancellation instead.
+                      </Alert>
+                    </div>
+                  )}
+
+                  <Stack direction="row" spacing={2}>
+                    {streamingStatus === 'stopped' ? (
+                      <Button
+                        variant="contained"
+                        onClick={startStreaming}
+                        disabled={
+                          roomID == null ||
+                          // Prevent users from starting streaming if there is a server lock with an active session
+                          (serverState?.serverLock?.isActive === true &&
+                            serverState.serverLock.clientID !== clientID)
+                        }>
+                        {buttonLabelMap[streamingStatus]}
+                      </Button>
+                    ) : (
+                      <Button
+                        variant="contained"
+                        color={
+                          streamingStatus === 'running' ? 'error' : 'primary'
+                        }
+                        disabled={
+                          streamingStatus === 'starting' || roomID == null
+                        }
+                        onClick={stopStreaming}>
+                        {buttonLabelMap[streamingStatus]}
+                      </Button>
+                    )}
+
+                    <Box>
+                      <Button
+                        variant="contained"
+                        aria-label={muted ? 'Unmute' : 'Mute'}
+                        color={muted ? 'info' : 'primary'}
+                        onClick={() => setMuted((prev) => !prev)}
+                        sx={{
+                          borderRadius: 100,
+                          paddingX: 0,
+                          minWidth: '36px',
+                        }}>
+                        {muted ? <MicOff /> : <Mic />}
+                      </Button>
+                    </Box>
+
+                    {roomID == null ? null : (
+                      <Box
+                        sx={{
+                          flexGrow: 1,
+                          display: 'flex',
+                          justifyContent: 'flex-end',
+                        }}>
+                        {xrDialogComponent}
+                      </Box>
+                    )}
+                  </Stack>
+
+                  {serverExceptions.length > 0 && (
+                    <div>
+                      <Alert severity="error">
+                        {`The server encountered an exception. See the browser console for details. You may need to refresh the page to continue using the app.`}
+                      </Alert>
+                    </div>
+                  )}
+                  {serverState != null &&
+                    serverState.totalActiveTranscoders >=
+                      TOTAL_ACTIVE_TRANSCODER_WARNING_THRESHOLD && (
+                      <div>
+                        <Alert severity="warning">
+                          {`The server currently has ${serverState?.totalActiveTranscoders} active streaming sessions. Performance may be degraded.`}
+                        </Alert>
+                      </div>
+                    )}
+
+                  {serverState?.serverLock != null &&
+                    serverState.serverLock.clientID !== clientID && (
+                      <div>
+                        <Alert severity="warning">
+                          {`The server is currently locked. Priority will be given to that client when they are streaming, and your streaming session may be halted abruptly.`}
+                        </Alert>
+                      </div>
+                    )}
+                </>
+              )}
+            </Stack>
+
+            {isListener && !isSpeaker && (
+              <Box sx={{marginBottom: 1, marginTop: 2}}>
+                {xrDialogComponent}
+              </Box>
+            )}
+          </div>
+
+          {hasMaxUsers && (
+            <div>
+              <Alert severity="error">
+                {`Maximum number of users reached. Please try again at a later time.`}
+              </Alert>
+            </div>
+          )}
+          {debugParam && roomID != null && <DebugSection />}
+
+          <div className="translation-text-container-sra horizontal-padding-sra">
+            <Stack
+              direction="row"
+              spacing={2}
+              sx={{mb: '16px', alignItems: 'center'}}>
+              <Typography variant="h1" sx={{fontWeight: 700, flexGrow: 1}}>
+                Transcript
+              </Typography>
+              {isSpeaker && (
+                <Button
+                  variant="text"
+                  size="small"
+                  onClick={onClearTranscriptForAll}>
+                  Clear Transcript for All
+                </Button>
+              )}
+            </Stack>
+            <Stack direction="row">
+              <div className="translation-text-sra">
+                {translationSentencesWithEmptyStartingString.map(
+                  (sentence, index, arr) => {
+                    const isLast = index === arr.length - 1;
+                    const maybeRef = isLast
+                      ? {ref: lastTranslationResultRef}
+                      : {};
+                    return (
+                      <div className="text-chunk-sra" key={index} {...maybeRef}>
+                        <Typography variant="body1">
+                          {sentence}
+                          {animateTextDisplay && isLast && (
+                            <Blink
+                              intervalMs={CURSOR_BLINK_INTERVAL_MS}
+                              shouldBlink={
+                                (roomState?.activeTranscoders ?? 0) > 0
+                              }>
+                              <Typography
+                                component="span"
+                                variant="body1"
+                                sx={{
+                                  display: 'inline-block',
+                                  transform: 'scaleY(1.25) translateY(-1px)',
+                                }}>
+                                {'|'}
+                              </Typography>
+                            </Blink>
+                          )}
+                        </Typography>
+                      </div>
+                    );
+                  },
+                )}
+              </div>
+            </Stack>
+          </div>
+        </div>
+      </Box>
+    </div>
+  );
+}

streaming-test-app/src/URLParams.ts

@@ -0,0 +1,50 @@
+import { getBooleanParamFlag, getStringParamFlag } from './getParamFlag';
+import { URLParamsObject } from './types/URLParamsTypes';
+
+/**
+ * These are the URL parameters you can provide to the app to change its behavior.
+ *
+ * Boolean flags can be set by just providing the flag name (`?autoJoin`), or by
+ * explicitly setting it to 1 (true) or 0 (false): `?autoJoin=1` or `?autoJoin=0`
+ *
+ * String flags require an explicit value: `?roomID=ABCD`
+ *
+ * Examples:
+ *
+ * - `http://localhost:5173/?roomID=BBCD&autoJoin&debug`
+ * - `http://localhost:5173/?serverURL=localhost:8000`
+
+ * @returns
+ */
+
+export function getURLParams(): URLParamsObject {
+  return {
+    // animate the translation text when it arrives, typing it out one letter at a time
+    animateTextDisplay: getBooleanParamFlag('animateTextDisplay', true), // default to true;
+
+    // automatically join the room when the app loads. requires roomID to be set via url param as well
+    autoJoin: getBooleanParamFlag('autoJoin', false),
+
+    // automatically check the server debug flag as true
+    debug: getBooleanParamFlag('debug', false),
+
+    // Enable UI on the client that allows locking out other users of the server when it's being used for high profile demos
+    // NOTE: There is an escape hatch for disabling a server lock by setting the name field to remove_server_lock
+    enableServerLock: getBooleanParamFlag('enableServerLock', false),
+
+    // Pre-populate the Room Code field with the provided roomID. Can be used in conjunction with autoJoin to jump straight into the room
+    roomID: getStringParamFlag('roomID'),
+
+    // Use an alternate server URL as the streaming server (useful for pointing to dev servers: http://localhost:5173/?serverURL=localhost:8000)
+    serverURL: getStringParamFlag('serverURL'),
+
+    // Skip the popup dialog that displays within VR, which is mostly redundant with the web based dialog
+    skipARIntro: getBooleanParamFlag('skipARIntro', true), // default to true
+
+    // Shows the translation text in AR in front of an opaque panel covering all the text area
+    // single_block = original single text block with background
+    // lines = each line is a separate block and animates
+    // lines_with_background = adds a panel behind lines
+    ARTranscriptionType: getStringParamFlag('ARTranscriptionType') || 'lines',
+  };
+}

streaming-test-app/src/createBufferedSpeechPlayer.ts

@@ -0,0 +1,173 @@
+import debug from './debug';
+
+type AddAudioToBufferFunction = (
+  samples: Array<number>,
+  sampleRate: number,
+) => void;
+
+export type BufferedSpeechPlayer = {
+  addAudioToBuffer: AddAudioToBufferFunction;
+  setGain: (gain: number) => void;
+  start: () => void;
+  stop: () => void;
+};
+
+type Options = {
+  onEnded?: () => void;
+  onStarted?: () => void;
+};
+
+export default function createBufferedSpeechPlayer({
+  onStarted,
+  onEnded,
+}: Options): BufferedSpeechPlayer {
+  const audioContext = new AudioContext();
+  const gainNode = audioContext.createGain();
+  gainNode.connect(audioContext.destination);
+
+  let unplayedAudioBuffers: Array<AudioBuffer> = [];
+
+  let currentPlayingBufferSource: AudioBufferSourceNode | null = null;
+
+  let isPlaying = false;
+
+  // This means that the player starts in the 'stopped' state, and you need to call player.start() for it to start playing
+  let shouldPlayWhenAudioAvailable = false;
+
+  const setGain = (gain: number) => {
+    gainNode.gain.setValueAtTime(gain, audioContext.currentTime);
+  };
+
+  const start = () => {
+    shouldPlayWhenAudioAvailable = true;
+    debug()?.start();
+    playNextBufferIfNotAlreadyPlaying();
+  };
+
+  // Stop will stop the audio and clear the buffers
+  const stop = () => {
+    shouldPlayWhenAudioAvailable = false;
+
+    // Stop the current buffers
+    currentPlayingBufferSource?.stop();
+    currentPlayingBufferSource = null;
+
+    unplayedAudioBuffers = [];
+
+    onEnded != null && onEnded();
+    isPlaying = false;
+    return;
+  };
+
+  const playNextBufferIfNotAlreadyPlaying = () => {
+    if (!isPlaying) {
+      playNextBuffer();
+    }
+  };
+
+  const playNextBuffer = () => {
+    if (shouldPlayWhenAudioAvailable === false) {
+      console.debug(
+        '[BufferedSpeechPlayer][playNextBuffer] Not playing any more audio because shouldPlayWhenAudioAvailable is false.',
+      );
+      // NOTE: we do not need to set isPlaying = false or call onEnded because that will be handled in the stop() function
+      return;
+    }
+    if (unplayedAudioBuffers.length === 0) {
+      console.debug(
+        '[BufferedSpeechPlayer][playNextBuffer] No buffers to play.',
+      );
+      if (isPlaying) {
+        isPlaying = false;
+        onEnded != null && onEnded();
+      }
+      return;
+    }
+
+    // If isPlaying is false, then we are starting playback fresh rather than continuing it, and should call onStarted
+    if (isPlaying === false) {
+      isPlaying = true;
+      onStarted != null && onStarted();
+    }
+
+    const source = audioContext.createBufferSource();
+
+    // Get the first unplayed buffer from the array, and remove it from the array
+    const buffer = unplayedAudioBuffers.shift() ?? null;
+    source.buffer = buffer;
+    console.debug(
+      `[BufferedSpeechPlayer] Playing buffer with ${source.buffer?.length} samples`,
+    );
+
+    source.connect(gainNode);
+
+    const startTime = new Date().getTime();
+    source.start();
+    currentPlayingBufferSource = source;
+    // This is probably not necessary, but it doesn't hurt
+    isPlaying = true;
+
+    // TODO: consider changing this to a while loop to avoid deep recursion
+    const onThisBufferPlaybackEnded = () => {
+      console.debug(
+        `[BufferedSpeechPlayer] Buffer with ${source.buffer?.length} samples ended.`,
+      );
+      source.removeEventListener('ended', onThisBufferPlaybackEnded);
+      const endTime = new Date().getTime();
+      debug()?.playedAudio(startTime, endTime, buffer);
+      currentPlayingBufferSource = null;
+
+      // We don't set isPlaying = false here because we are attempting to continue playing. It will get set to false if there are no more buffers to play
+      playNextBuffer();
+    };
+
+    source.addEventListener('ended', onThisBufferPlaybackEnded);
+  };
+
+  const addAudioToBuffer: AddAudioToBufferFunction = (samples, sampleRate) => {
+    const incomingArrayBufferChunk = audioContext.createBuffer(
+      // 1 channel
+      1,
+      samples.length,
+      sampleRate,
+    );
+
+    incomingArrayBufferChunk.copyToChannel(
+      new Float32Array(samples),
+      // first channel
+      0,
+    );
+
+    console.debug(
+      `[addAudioToBufferAndPlay] Adding buffer with ${incomingArrayBufferChunk.length} samples to queue.`,
+    );
+
+    unplayedAudioBuffers.push(incomingArrayBufferChunk);
+    debug()?.receivedAudio(
+      incomingArrayBufferChunk.length / incomingArrayBufferChunk.sampleRate,
+    );
+    const audioBuffersTableInfo = unplayedAudioBuffers.map((buffer, i) => {
+      return {
+        index: i,
+        duration: buffer.length / buffer.sampleRate,
+        samples: buffer.length,
+      };
+    });
+    const totalUnplayedDuration = unplayedAudioBuffers.reduce((acc, buffer) => {
+      return acc + buffer.length / buffer.sampleRate;
+    }, 0);
+
+    console.debug(
+      `[addAudioToBufferAndPlay] Current state of incoming audio buffers (${totalUnplayedDuration.toFixed(
+        1,
+      )}s unplayed):`,
+    );
+    console.table(audioBuffersTableInfo);
+
+    if (shouldPlayWhenAudioAvailable) {
+      playNextBufferIfNotAlreadyPlaying();
+    }
+  };
+
+  return {addAudioToBuffer, setGain, stop, start};
+}

streaming-test-app/src/cursorBlinkInterval.ts

@@ -0,0 +1 @@
+export const CURSOR_BLINK_INTERVAL_MS = 500;

streaming-test-app/src/debug.ts

@@ -0,0 +1,257 @@
+import {TYPING_ANIMATION_DELAY_MS} from './StreamingInterface';
+import {getURLParams} from './URLParams';
+import audioBuffertoWav from 'audiobuffer-to-wav';
+import './StreamingInterface.css';
+
+type StartEndTime = {
+  start: number;
+  end: number;
+};
+
+type StartEndTimeWithAudio = StartEndTime & {
+  float32Audio: Float32Array;
+};
+
+type Text = {
+  time: number;
+  chars: number;
+};
+
+type DebugTimings = {
+  receivedAudio: StartEndTime[];
+  playedAudio: StartEndTimeWithAudio[];
+  receivedText: Text[];
+  renderedText: StartEndTime[];
+  sentAudio: StartEndTimeWithAudio[];
+  startRenderTextTime: number | null;
+  startRecordingTime: number | null;
+  receivedAudioSampleRate: number | null;
+};
+
+function getInitialTimings(): DebugTimings {
+  return {
+    receivedAudio: [],
+    playedAudio: [],
+    receivedText: [],
+    renderedText: [],
+    sentAudio: [],
+    startRenderTextTime: null,
+    startRecordingTime: null,
+    receivedAudioSampleRate: null,
+  };
+}
+
+function downloadAudioBuffer(audioBuffer: AudioBuffer, fileName: string): void {
+  const wav = audioBuffertoWav(audioBuffer);
+  const wavBlob = new Blob([new DataView(wav)], {
+    type: 'audio/wav',
+  });
+  const url = URL.createObjectURL(wavBlob);
+  const anchor = document.createElement('a');
+  anchor.href = url;
+  anchor.target = '_blank';
+  anchor.download = fileName;
+  anchor.click();
+}
+
+// Uncomment for debugging without download
+// function playAudioBuffer(audioBuffer: AudioBuffer): void {
+//   const audioContext = new AudioContext();
+//   const source = audioContext.createBufferSource();
+
+//   source.buffer = audioBuffer;
+//   source.connect(audioContext.destination);
+//   source.start();
+// }
+
+// Accumulate timings and audio / text translation samples for debugging and exporting
+class DebugTimingsManager {
+  timings: DebugTimings = getInitialTimings();
+
+  start(): void {
+    this.timings = getInitialTimings();
+    this.timings.startRecordingTime = new Date().getTime();
+  }
+
+  sentAudio(event: AudioProcessingEvent): void {
+    const end = new Date().getTime();
+    const start = end - event.inputBuffer.duration * 1000;
+    // Copy or else buffer seems to be re-used
+    const float32Audio = new Float32Array(event.inputBuffer.getChannelData(0));
+    this.timings.sentAudio.push({
+      start,
+      end,
+      float32Audio,
+    });
+  }
+
+  receivedText(text: string): void {
+    this.timings.receivedText.push({
+      time: new Date().getTime(),
+      chars: text.length,
+    });
+  }
+
+  startRenderText(): void {
+    if (this.timings.startRenderTextTime == null) {
+      this.timings.startRenderTextTime = new Date().getTime();
+    }
+  }
+
+  endRenderText(): void {
+    if (this.timings.startRenderTextTime == null) {
+      console.warn(
+        'Wrong timings of start / end rendering text. startRenderText is null',
+      );
+      return;
+    }
+
+    this.timings.renderedText.push({
+      start: this.timings.startRenderTextTime as number,
+      end: new Date().getTime(),
+    });
+    this.timings.startRenderTextTime = null;
+  }
+
+  receivedAudio(duration: number): void {
+    const start = new Date().getTime();
+    this.timings.receivedAudio.push({
+      start,
+      end: start + duration * 1000,
+    });
+  }
+
+  playedAudio(start: number, end: number, buffer: AudioBuffer | null): void {
+    if (buffer != null) {
+      if (this.timings.receivedAudioSampleRate == null) {
+        this.timings.receivedAudioSampleRate = buffer.sampleRate;
+      }
+      if (this.timings.receivedAudioSampleRate != buffer.sampleRate) {
+        console.error(
+          'Sample rates of received audio are unequal, will fail to reconstruct debug audio',
+          this.timings.receivedAudioSampleRate,
+          buffer.sampleRate,
+        );
+      }
+    }
+    this.timings.playedAudio.push({
+      start,
+      end,
+      float32Audio:
+        buffer == null
+          ? new Float32Array()
+          : new Float32Array(buffer.getChannelData(0)),
+    });
+  }
+
+  getChartData() {
+    const columns = [
+      {type: 'string', id: 'Series'},
+      {type: 'date', id: 'Start'},
+      {type: 'date', id: 'End'},
+    ];
+    return [
+      columns,
+      ...this.timings.sentAudio.map((sentAudio) => [
+        'Sent Audio',
+        new Date(sentAudio.start),
+        new Date(sentAudio.end),
+      ]),
+      ...this.timings.receivedAudio.map((receivedAudio) => [
+        'Received Audio',
+        new Date(receivedAudio.start),
+        new Date(receivedAudio.end),
+      ]),
+      ...this.timings.playedAudio.map((playedAudio) => [
+        'Played Audio',
+        new Date(playedAudio.start),
+        new Date(playedAudio.end),
+      ]),
+      // Best estimate duration by multiplying length with animation duration for each letter
+      ...this.timings.receivedText.map((receivedText) => [
+        'Received Text',
+        new Date(receivedText.time),
+        new Date(
+          receivedText.time + receivedText.chars * TYPING_ANIMATION_DELAY_MS,
+        ),
+      ]),
+      ...this.timings.renderedText.map((renderedText) => [
+        'Rendered Text',
+        new Date(renderedText.start),
+        new Date(renderedText.end),
+      ]),
+    ];
+  }
+
+  downloadInputAudio() {
+    const audioContext = new AudioContext();
+    const totalLength = this.timings.sentAudio.reduce((acc, cur) => {
+      return acc + cur?.float32Audio?.length ?? 0;
+    }, 0);
+    if (totalLength === 0) {
+      return;
+    }
+
+    const incomingArrayBuffer = audioContext.createBuffer(
+      1, // 1 channel
+      totalLength,
+      audioContext.sampleRate,
+    );
+
+    const buffer = incomingArrayBuffer.getChannelData(0);
+    let i = 0;
+    this.timings.sentAudio.forEach((sentAudio) => {
+      sentAudio.float32Audio.forEach((bytes) => {
+        buffer[i++] = bytes;
+      });
+    });
+
+    // Play for debugging
+    // playAudioBuffer(incomingArrayBuffer);
+    downloadAudioBuffer(incomingArrayBuffer, `input_audio.wav`);
+  }
+
+  downloadOutputAudio() {
+    const playedAudio = this.timings.playedAudio;
+    const sampleRate = this.timings.receivedAudioSampleRate;
+    if (
+      playedAudio.length === 0 ||
+      this.timings.startRecordingTime == null ||
+      sampleRate == null
+    ) {
+      return null;
+    }
+
+    let previousEndTime = this.timings.startRecordingTime;
+    const audioArray: number[] = [];
+    playedAudio.forEach((audio) => {
+      const delta = (audio.start - previousEndTime) / 1000;
+      for (let i = 0; i < delta * sampleRate; i++) {
+        audioArray.push(0.0);
+      }
+      audio.float32Audio.forEach((bytes) => audioArray.push(bytes));
+      previousEndTime = audio.end;
+    });
+    const audioContext = new AudioContext();
+    const incomingArrayBuffer = audioContext.createBuffer(
+      1, // 1 channel
+      audioArray.length,
+      sampleRate,
+    );
+
+    incomingArrayBuffer.copyToChannel(
+      new Float32Array(audioArray),
+      0, // first channel
+    );
+
+    // Play for debugging
+    // playAudioBuffer(incomingArrayBuffer);
+    downloadAudioBuffer(incomingArrayBuffer, 'output_audio.wav');
+  }
+}
+
+const debugSingleton = new DebugTimingsManager();
+export default function debug(): DebugTimingsManager | null {
+  const debugParam = getURLParams().debug;
+  return debugParam ? debugSingleton : null;
+}