src/components/control-tray/ControlTray.tsx

/**
 * Copyright 2024 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import cn from "classnames";

import { memo, ReactNode, RefObject, useEffect, useRef, useState } from "react";
import { useLiveAPIContext } from "../../contexts/LiveAPIContext";
import { UseMediaStreamResult } from "../../hooks/use-media-stream-mux";
import { useScreenCapture } from "../../hooks/use-screen-capture";
import { useWebcam } from "../../hooks/use-webcam";
import { AudioRecorder } from "../../lib/audio-recorder";
import { audioContext } from "../../lib/utils";
import { isIOS } from "../../lib/platform";
import AudioPulse from "../audio-pulse/AudioPulse";
import "./control-tray.scss";

export type ControlTrayProps = {
  videoRef: RefObject<HTMLVideoElement>;
  children?: ReactNode;
  supportsVideo: boolean;
  onVideoStreamChange?: (stream: MediaStream | null) => void;
};

type MediaStreamButtonProps = {
  isStreaming: boolean;
  onIcon: string;
  offIcon: string;
  start: () => Promise<any>;
  stop: () => any;
};

/**
 * button used for triggering webcam or screen-capture
 */
const MediaStreamButton = memo(
  ({ isStreaming, onIcon, offIcon, start, stop }: MediaStreamButtonProps) =>
    isStreaming ? (
      <button className="action-button" onClick={stop}>
        <span className="material-symbols-outlined">{onIcon}</span>
      </button>
    ) : (
      <button className="action-button" onClick={start}>
        <span className="material-symbols-outlined">{offIcon}</span>
      </button>
    ),
);

function ControlTray({
  videoRef,
  children,
  onVideoStreamChange = () => {},
  supportsVideo,
}: ControlTrayProps) {
  const videoStreams = [useWebcam(), useScreenCapture()];
  const [activeVideoStream, setActiveVideoStream] =
    useState<MediaStream | null>(null);
  const [webcam, screenCapture] = videoStreams;
  const [inVolume, setInVolume] = useState(0);
  const [audioRecorder] = useState(() => new AudioRecorder());
  const [muted, setMuted] = useState(false);
  const [videoDevices, setVideoDevices] = useState<MediaDeviceInfo[]>([]);
  const [currentDeviceIndex, setCurrentDeviceIndex] = useState(0);
  const renderCanvasRef = useRef<HTMLCanvasElement>(null);
  const connectButtonRef = useRef<HTMLButtonElement>(null);
  const [simulatedVolume, setSimulatedVolume] = useState(0);
  const isIOSDevice = isIOS();

  const { client, connected, connect, disconnect, volume } =
    useLiveAPIContext();

  // Add iOS detection
  const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent);

  useEffect(() => {
    if (!connected && connectButtonRef.current) {
      console.log('🎯 Setting focus on connect button - connection status:', connected);
      connectButtonRef.current.focus();
    }
  }, [connected]);

  // Add iOS volume simulation effect
  useEffect(() => {
    if (isIOSDevice && connected && !muted) {
      console.log('📱 Starting iOS volume simulation');
      const interval = setInterval(() => {
        // Create a smooth pulsing effect
        const pulse = (Math.sin(Date.now() / 500) + 1) / 2; // Values between 0 and 1
        setSimulatedVolume(0.02 + pulse * 0.03); // Small range for subtle effect
      }, 50);
      
      return () => {
        console.log('🛑 Stopping iOS volume simulation');
        clearInterval(interval);
      };
    }
  }, [connected, muted, isIOSDevice]);

  useEffect(() => {
    document.documentElement.style.setProperty(
      "--volume",
      `${Math.max(5, Math.min((isIOSDevice ? simulatedVolume : inVolume) * 200, 8))}px`,
    );
  }, [inVolume, simulatedVolume, isIOSDevice]);

  useEffect(() => {
    const onData = (base64: string) => {
      console.log('🎤 Sending audio data chunk');
      client.sendRealtimeInput([
        {
          mimeType: "audio/pcm;rate=16000",
          data: base64,
        },
      ]);
    };

    if (connected && !muted && audioRecorder) {
      console.log('🎙️ Starting audio recorder');
      audioRecorder.on("data", onData).on("volume", setInVolume).start();
    } else {
      console.log('⏹️ Stopping audio recorder');
      audioRecorder.stop();
    }

    return () => {
      console.log('🧹 Cleaning up audio recorder listeners');
      audioRecorder.off("data", onData).off("volume", setInVolume);
    };
  }, [connected, client, muted, audioRecorder]);

  useEffect(() => {
    if (videoRef.current) {
      console.log('🎥 Setting video stream:', activeVideoStream ? 'active' : 'null');
      videoRef.current.srcObject = activeVideoStream;
    }

    let timeoutId = -1;

    function sendVideoFrame() {
      const video = videoRef.current;
      const canvas = renderCanvasRef.current;

      if (!video || !canvas) {
        console.log('⚠️ Missing video or canvas reference');
        return;
      }

      const ctx = canvas.getContext("2d")!;
      canvas.width = video.videoWidth * 0.25;
      canvas.height = video.videoHeight * 0.25;
      if (canvas.width + canvas.height > 0) {
        ctx.drawImage(videoRef.current, 0, 0, canvas.width, canvas.height);
        const base64 = canvas.toDataURL("image/jpeg", 1.0);
        const data = base64.slice(base64.indexOf(",") + 1, Infinity);
        console.log('📸 Sending video frame');
        client.sendRealtimeInput([{ mimeType: "image/jpeg", data }]);
      }
      if (connected) {
        timeoutId = window.setTimeout(sendVideoFrame, 1000 / 0.5);
      }
    }
    if (connected && activeVideoStream !== null) {
      console.log('🎬 Starting video frame capture');
      requestAnimationFrame(sendVideoFrame);
    }
    return () => {
      console.log('⏹️ Stopping video frame capture');
      clearTimeout(timeoutId);
    };
  }, [connected, activeVideoStream, client, videoRef]);

  // Add effect to track available video devices
  useEffect(() => {
    async function getVideoDevices() {
      console.log('📹 Enumerating video devices...');
      try {
        const devices = await navigator.mediaDevices.enumerateDevices();
        const videoInputs = devices.filter(device => device.kind === 'videoinput');
        console.log('📹 Available video devices:', videoInputs.length);
        videoInputs.forEach((device, index) => {
          console.log(`📹 Device ${index}:`, {
            deviceId: device.deviceId,
            label: device.label
          });
        });
        setVideoDevices(videoInputs);
      } catch (error) {
        console.error('❌ Error enumerating devices:', error);
      }
    }

    // Get initial device list
    getVideoDevices();

    // Listen for device changes
    navigator.mediaDevices.addEventListener('devicechange', getVideoDevices);
    return () => {
      navigator.mediaDevices.removeEventListener('devicechange', getVideoDevices);
    };
  }, []);

  const rotateWebcam = async () => {
    console.log('🔄 Rotating webcam...');
    if (videoDevices.length <= 1) {
      console.log('⚠️ Not enough video devices to rotate');
      return;
    }

    const nextIndex = (currentDeviceIndex + 1) % videoDevices.length;
    console.log(`🎯 Switching to device index ${nextIndex}`);
    
    try {
      const newStream = await navigator.mediaDevices.getUserMedia({
        video: { deviceId: { exact: videoDevices[nextIndex].deviceId } },
        audio: false
      });
      
      console.log('✅ Got new video stream');
      setActiveVideoStream(newStream);
      onVideoStreamChange(newStream);
      setCurrentDeviceIndex(nextIndex);
      
      // Clean up old stream
      if (activeVideoStream) {
        activeVideoStream.getTracks().forEach(track => track.stop());
      }
    } catch (error) {
      console.error('❌ Error rotating webcam:', error);
    }
  };

  //handler for swapping from one video-stream to the next
  const changeStreams = (next?: UseMediaStreamResult) => async () => {
    if (next) {
      console.log('🔄 Starting new video stream');
      const mediaStream = await next.start();
      setActiveVideoStream(mediaStream);
      onVideoStreamChange(mediaStream);
    } else {
      console.log('⏹️ Stopping video stream');
      setActiveVideoStream(null);
      onVideoStreamChange(null);
    }

    videoStreams.filter((msr) => msr !== next).forEach((msr) => {
      console.log('🛑 Stopping other video streams');
      msr.stop();
    });
  };

  return (
    <section className="control-tray">
      <canvas style={{ display: "none" }} ref={renderCanvasRef} />
      <nav className={cn("actions-nav", { disabled: !connected })}>
        <button
          className={cn("action-button mic-button")}
          onClick={() => setMuted(!muted)}
        >
          {!muted ? (
            <span className="material-symbols-outlined filled">mic</span>
          ) : (
            <span className="material-symbols-outlined filled">mic_off</span>
          )}
        </button>

        <div className="action-button no-action outlined">
          <AudioPulse volume={volume} active={connected} hover={false} />
        </div>

        {supportsVideo && (
          <>
            {!isIOSDevice && (
              <MediaStreamButton
                isStreaming={screenCapture.isStreaming}
                start={changeStreams(screenCapture)}
                stop={changeStreams()}
                onIcon="cancel_presentation"
                offIcon="present_to_all"
              />
            )}
            {webcam.isStreaming && videoDevices.length > 1 && (
              <button 
                className="action-button" 
                onClick={rotateWebcam}
                title="Switch camera"
              >
                <span className="material-symbols-outlined">switch_camera</span>
              </button>
            )}
            <MediaStreamButton
              isStreaming={webcam.isStreaming}
              start={changeStreams(webcam)}
              stop={changeStreams()}
              onIcon="videocam_off"
              offIcon="videocam"
            />
          </>
        )}
        {children}
      </nav>

      <div className={cn("connection-container", { connected })}>
        <div className="connection-button-container">
          <button
            ref={connectButtonRef}
            className={cn("action-button connect-toggle", { connected })}
            onClick={async () => {
              console.log('🎮 ControlTray: Connection button clicked');
              try {
                if (connected) {
                  console.log('📴 ControlTray: Initiating disconnect...');
                  await disconnect();
                  console.log('✅ ControlTray: Disconnected successfully');
                } else {
                  console.log('🔌 ControlTray: Starting connection process...');
                  console.log('📱 ControlTray: Device info:', { isIOSDevice, isSafari });
                  
                  console.log('📞 ControlTray: Calling LiveAPIContext.connect()...');
                  await connect();
                  console.log('✅ ControlTray: Connected successfully via LiveAPIContext');
                }
              } catch (err) {
                console.error('❌ ControlTray: Connection error:', err);
              }
            }}
          >
            <span className="material-symbols-outlined filled">
              {connected ? "pause" : "play_arrow"}
            </span>
          </button>
        </div>
        <span className="text-indicator">Streaming</span>
      </div>
    </section>
  );
}

export default memo(ControlTray);