add lip-sync

src/main.rs

@@ -9,13 +9,9 @@ use std::default::Default;
 use std::error::Error;
 use std::fmt::{Debug, Display, Formatter};
 use std::future::Future;
-use tokio::sync::mpsc::channel;
 use async_stream::stream;
 use aws_config::meta::region::RegionProviderChain;
-use aws_sdk_transcribestreaming::primitives::Blob;
-use aws_sdk_transcribestreaming::types::{AudioEvent, AudioStream, LanguageCode, MediaEncoding, TranscriptResultStream};
 use aws_sdk_transcribestreaming::{config::Region, meta::PKG_VERSION};
-use aws_sdk_transcribestreaming::operation::start_stream_transcription::StartStreamTranscriptionOutput;
 use clap::Parser;
 
 use poem::{Endpoint, EndpointExt, get, handler, IntoResponse, listener::TcpListener, Route, Server};
@@ -26,7 +22,6 @@ use futures_util::stream::StreamExt;
 use poem::web::{Data, Query};
 
 use tokio::select;
-use tokio::sync::mpsc::{Receiver, Sender};
 use tokio_stream::Stream;
 use serde::{Deserialize, Serialize};
 use lesson::{LessonsManager};
@@ -57,28 +52,6 @@ enum ReplyEvent {
     Synthesized(Vec<u8>),
 }
 
-
-async fn translate(client: &aws_sdk_translate::Client, transcript: Option<String>, source_lang_code: Option<String>) -> Option<String> {
-    let res = client.translate_text()
-        .set_text(transcript)
-        .set_source_language_code(Some("zh".to_string()))
-        .set_target_language_code(Some("en".to_string()))
-        .send().await;
-    res.expect("failed to translate").translated_text
-}
-
-async fn synthesize(client: &aws_sdk_polly::Client, transcript: String) -> Option<Vec<u8>> {
-    let res = client.synthesize_speech()
-        .set_text(Some(transcript))
-        .voice_id("Amy".into())
-        .output_format("pcm".into())
-        .language_code("en-US".into())
-        // .language_code("cmn-CN".into())
-        .send().await;
-    let bs = res.expect("failed to translate").audio_stream.collect().await.ok()?;
-    Some(bs.to_vec())
-}
-
 #[derive(Clone)]
 struct Context {
     lessons_manager: LessonsManager,
@@ -171,7 +144,9 @@ async fn stream_speaker(ctx: Data<&Context>, query: Query<LessonSpeakerQuery>, w
                 output = transcribe_rx.recv() => {
                     if let Ok(transcript) = output {
                         println!("Transcribed: {}", transcript);
-                        socket.send(Message::Text(transcript)).await.expect("failed to send");
+                        let evt = LiveLessonTextEvent::Transcription { text: transcript.clone() };
+                        let json = serde_json::to_string(&evt).expect("failed to serialize");
+                        let _ = socket.send(Message::Text(json)).await.expect("failed to send");
                     }
                 },
             }
@@ -281,26 +256,3 @@ impl Error for StreamTranscriptionError {
     }
 }
 
-fn process(translate_client: aws_sdk_translate::Client,
-               polly_client: aws_sdk_polly::Client,
-               res: Result<String, StreamTranscriptionError>) -> impl Stream<Item=Result<ReplyEvent, StreamTranscriptionError>> {
-    stream! {
-        match res {
-            Ok(transcription) => {
-                yield Ok(ReplyEvent::Transcribed(transcription.clone()));
-                let translated = translate(&translate_client, Some(transcription), Some("en".to_string())).await;
-                if let Some(has) = translated {
-                    yield Ok(ReplyEvent::Translated(has.clone()));
-                    println!("Translated: {}", has);
-                    if let Some(synthesized) = synthesize(&polly_client, has).await {
-                        yield Ok(ReplyEvent::Synthesized(synthesized));
-                    }
-                }
-            },
-            Err(e) => {
-                yield Err(e);
-            }
-        }
-
-    }
-}

static/client.js

@@ -1,303 +0,0 @@
-'use strict';
-
-//  Google Cloud Speech Playground with node.js and socket.io
-//  Created by Vinzenz Aubry for sansho 24.01.17
-//  Feel free to improve!
-//  Contact: [email protected]
-
-//connection to socket
-let websocket_uri
-if (location.protocol === "https:") {
-    websocket_uri = "wss:"
-} else {
-    websocket_uri = "ws:"
-}
-websocket_uri += "//" + location.host
-websocket_uri += "/ws" + location.pathname
-websocket_uri += location.search
-const socket = new WebSocket(websocket_uri);
-
-//================= CONFIG =================
-// Stream Audio
-let bufferSize = 2048,
-    AudioContext,
-    context,
-    processor,
-    input,
-    globalStream;
-
-//vars
-let audioElement = document.querySelector('audio'),
-    finalWord = false,
-    translationText = document.getElementById('Translation'),
-    transcriptionText = document.getElementById('Transcription'),
-    removeLastSentence = true,
-    streamStreaming = false;
-
-//audioStream constraints
-const constraints = {
-    audio: true,
-    video: false,
-};
-
-//================= RECORDING =================
-
-async function initRecording() {
-    // socket.emit('startGoogleCloudStream', ''); //init socket Google Speech Connection
-    streamStreaming = true;
-    AudioContext = window.AudioContext || window.webkitAudioContext;
-    context = new AudioContext({
-        // if Non-interactive, use 'playback' or 'balanced' // https://developer.mozilla.org/en-US/docs/Web/API/AudioContextLatencyCategory
-        latencyHint: 'interactive',
-    });
-
-    await context.audioWorklet.addModule('recorderWorkletProcessor.js')
-    context.resume();
-
-    globalStream = await navigator.mediaDevices.getUserMedia(constraints)
-    input = context.createMediaStreamSource(globalStream)
-    processor = new window.AudioWorkletNode(
-        context,
-        'recorder.worklet'
-    );
-    processor.connect(context.destination);
-    context.resume()
-    input.connect(processor)
-    processor.port.onmessage = (e) => {
-        const audioData = e.data;
-        microphoneProcess(audioData)
-    }
-}
-
-function microphoneProcess(buffer) {
-    socket.send(buffer);
-}
-
-//================= INTERFACE =================
-var startButton = document.getElementById('startRecButton');
-startButton.addEventListener('click', startRecording);
-
-var endButton = document.getElementById('stopRecButton');
-endButton.addEventListener('click', stopRecording);
-endButton.disabled = true;
-
-var recordingStatus = document.getElementById('recordingStatus');
-
-function startRecording() {
-    startButton.disabled = true;
-    endButton.disabled = false;
-    recordingStatus.style.visibility = 'visible';
-    initRecording();
-}
-
-function stopRecording() {
-    // waited for FinalWord
-    startButton.disabled = false;
-    endButton.disabled = true;
-    recordingStatus.style.visibility = 'hidden';
-    streamStreaming = false;
-    // socket.emit('endGoogleCloudStream', '');
-
-    let track = globalStream.getTracks()[0];
-    track.stop();
-
-    input.disconnect(processor);
-    processor.disconnect(context.destination);
-    context.close().then(function () {
-        input = null;
-        processor = null;
-        context = null;
-        AudioContext = null;
-        startButton.disabled = false;
-    });
-
-    // context.close();
-
-    // audiovideostream.stop();
-
-    // microphone_stream.disconnect(script_processor_node);
-    // script_processor_node.disconnect(audioContext.destination);
-    // microphone_stream = null;
-    // script_processor_node = null;
-
-    // audiovideostream.stop();
-    // videoElement.srcObject = null;
-}
-
-
-const audioQueue = new rxjs.Subject();
-audioQueue
-    .pipe(rxjs.concatMap(playAudio))
-    .subscribe(_ => console.log('played audio'));
-//================= SOCKET IO =================
-socket.onmessage = function (msg) {
-    if (msg.data instanceof Blob) {
-        audioQueue.next(msg.data)
-    } else {
-        // text
-        const evt = JSON.parse(msg.data)
-        if (evt.type === 'Translation') {
-            onSpeechData(transcriptionText, evt.text)
-        } else if (evt.type === 'Transcription') {
-            onSpeechData(translationText, evt.text)
-        } else {
-            console.log(evt.visemes)
-        }
-    }
-}
-socket.onclose = function () {
-    processor.stop()
-}
-
-function onSpeechData(resultText, data) {
-    var dataFinal = false;
-
-    if (dataFinal === false) {
-        // console.log(resultText.lastElementChild);
-        if (removeLastSentence) {
-            resultText.lastElementChild.remove();
-        }
-        removeLastSentence = true;
-
-        //add empty span
-        let empty = document.createElement('span');
-        resultText.appendChild(empty);
-
-        //add children to empty span
-        let edit = addTimeSettingsInterim(data);
-
-        for (var i = 0; i < edit.length; i++) {
-            resultText.lastElementChild.appendChild(edit[i]);
-            resultText.lastElementChild.appendChild(
-                document.createTextNode('\u00A0')
-            );
-        }
-    } else if (dataFinal === true) {
-        resultText.lastElementChild.remove();
-
-        //add empty span
-        let empty = document.createElement('span');
-        resultText.appendChild(empty);
-
-        //add children to empty span
-        let edit = addTimeSettingsFinal(data);
-        for (var i = 0; i < edit.length; i++) {
-            if (i === 0) {
-                edit[i].innerText = capitalize(edit[i].innerText);
-            }
-            resultText.lastElementChild.appendChild(edit[i]);
-
-            if (i !== edit.length - 1) {
-                resultText.lastElementChild.appendChild(
-                    document.createTextNode('\u00A0')
-                );
-            }
-        }
-        resultText.lastElementChild.appendChild(
-            document.createTextNode('\u002E\u00A0')
-        );
-
-        console.log("Google Speech sent 'final' Sentence.");
-        finalWord = true;
-        endButton.disabled = false;
-
-        removeLastSentence = false;
-    }
-}
-
-//================= Juggling Spans for nlp Coloring =================
-function addTimeSettingsInterim(wholeString) {
-    console.log(wholeString);
-
-    let nlpObject = nlp(wholeString).out('terms');
-
-    let words_without_time = [];
-
-    for (let i = 0; i < nlpObject.length; i++) {
-        //data
-        let word = nlpObject[i].text;
-        let tags = [];
-
-        //generate span
-        let newSpan = document.createElement('span');
-        newSpan.innerHTML = word;
-
-        //push all tags
-        for (let j = 0; j < nlpObject[i].tags.length; j++) {
-            tags.push(nlpObject[i].tags[j]);
-        }
-
-        //add all classes
-        for (let j = 0; j < nlpObject[i].tags.length; j++) {
-            let cleanClassName = tags[j];
-            // console.log(tags);
-            let className = `nl-${cleanClassName}`;
-            newSpan.classList.add(className);
-        }
-
-        words_without_time.push(newSpan);
-    }
-
-    finalWord = false;
-    endButton.disabled = true;
-
-    return words_without_time;
-}
-
-window.onbeforeunload = function () {
-    if (streamStreaming) {
-        // socket.emit('endGoogleCloudStream', '');
-    }
-};
-
-//================= SANTAS HELPERS =================
-
-// sampleRateHertz 16000 //saved sound is awefull
-function convertFloat32ToInt16(buffer) {
-    let l = buffer.length;
-    let buf = new Int16Array(l / 3);
-
-    while (l--) {
-        if (l % 3 == 0) {
-            buf[l / 3] = buffer[l] * 0xffff;
-        }
-    }
-    return buf.buffer;
-}
-
-function capitalize(s) {
-    if (s.length < 1) {
-        return s;
-    }
-    return s.charAt(0).toUpperCase() + s.slice(1);
-}
-
-const audioContext = new (window.AudioContext || window.webkitAudioContext)();
-
-let nextStartTime = audioContext.currentTime;
-
-async function playAudio(chunk) {
-    const totalLength = chunk.size;
-
-    // Create an AudioBuffer of enough size
-    const audioBuffer = audioContext.createBuffer(1, totalLength / Int16Array.BYTES_PER_ELEMENT, 16000); // Assuming mono audio at 44.1kHz
-    const output = audioBuffer.getChannelData(0);
-
-    // Copy the PCM samples into the AudioBuffer
-    const arrayBuf = await chunk.arrayBuffer();
-    const int16Array = new Int16Array(arrayBuf, 0, Math.floor(arrayBuf.byteLength / 2))
-    for(let i = 0; i < int16Array.length; i++) {
-        output[i] = int16Array[i] / 32768.0;  // Convert to [-1, 1] float32 range
-    }
-
-    // 3. Play the audio using Web Audio API
-
-    const source = audioContext.createBufferSource();
-    source.buffer = audioBuffer;
-    source.connect(audioContext.destination);
-    source.start(nextStartTime);
-    nextStartTime = Math.max(nextStartTime, audioContext.currentTime) + audioBuffer.duration;
-    source.onended = () => {
-        console.log('audio slice ended');
-    }
-}

static/index.html

@@ -1,55 +1,15 @@
-<!doctype html>
-<html class="no-js" lang="en">
-
-<head>
-  <meta charset="utf-8">
-  <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
-  <title>Polyhedron: Realtime Transcribe + Translate + Speech</title>
-  <meta name="description" content="Google Cloud Speech Recognition with Node and Socket.io">
-  <meta name="viewport" content="width=device-width, initial-scale=1">
-
-  <link rel="stylesheet" href="main.css">
-</head>
-
-<body>
-<div class="wrapper">
-  <h1>Polyhedron: Realtime Transcribe + Translate + Speech</h1>
-
-  <audio></audio>
-
-  <br>
-  <button id="startButton" type="button"> Start listening</button>
-  <button id="startRecButton" type="button"> Start recording</button>
-  <button id="stopRecButton" type="button"> Stop recording</button>
-  <div id="recordingStatus">&nbsp;</div>
-  <br>
-
-  <div>
-    <h1>Translation</h1>
-    <p id="Translation">
-                <span class="greyText">No Speech to Text yet
-                    <span>
-    </p>
-    <h1>Transcription</h1>
-    <p id="Transcription">
-                <span class="greyText">No Speech to Text yet
-                    <span>
-    </p>
-  </div>
-
-  <br>
-  <br>
-</div>
-
-<!-- Nlp -->
-<script src="https://unpkg.com/[email protected]/builds/compromise.min.js"></script>
-
-<!-- Socket -->
-<!--<script src="assets/js/socket.io.js"></script>-->
-
-<script src="https://unpkg.com/rxjs@%5E7/dist/bundles/rxjs.umd.min.js"></script>
-<!-- Client -->
-<script src="client.js"></script>
-</body>
-
-</html>
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite + React</title>
+    <script type="module" crossorigin src="/assets/index-835823f8.js"></script>
+    <link rel="stylesheet" href="/assets/index-983f9492.css">
+  </head>
+  <body>
+    <div id="root"></div>
+    
+  </body>
+</html>

static/main.css

@@ -1,232 +0,0 @@
-body {
-    font-family: sans-serif;
-    background-color: rgb(189, 189, 189);
-}
-
-p {
-    color: black;
-}
-
-a {
-    font-weight: bold;
-    text-decoration: none;
-    color: #2a2a2a;
-}
-
-a:visited {
-    font-weight: normal;
-}
-
-a:hover {
-    text-decoration: underline;
-}
-
-audio {
-    width: 300px;
-    height: auto;
-    backgorund-color: red;
-}
-
-.wrapper {
-    width: 90vw;
-    margin: 0 auto;
-}
-
-.greyText {
-    opacity: 0.4;
-}
-
-@-webkit-keyframes redGlow {
-    from {
-        background-color: #8c190a;
-        -webkit-box-shadow: 0 0 9px #9c291a;
-    }
-    50% {
-        background-color: #9c291a;
-        -webkit-box-shadow: 0 0 18px #bdb5b4;
-    }
-    to {
-        background-color: #8c190a;
-        -webkit-box-shadow: 0 0 9px #9c291a;
-    }
-}
-#recordingStatus {
-    display: inline-block;
-    width: 18px;
-    height: 18px;
-    border-radius: 20px;
-    visibility: hidden;
-    -webkit-animation-name: redGlow;
-    -webkit-animation-duration: 2s;
-    -webkit-animation-iteration-count: infinite;
-}
-
-#ResultText {
-    width: 80vw;
-}
-
-#ResultText span {
-    display: inline-block;
-    margin-top: 10px;
-}
-
-#sessionSpeechData {
-    width: 80vw;
-}
-
-#sessionSpeechData span {
-    display: inline-block;
-    margin-top: 10px;
-}
-
-.nl-Adjective {
-    background-color: #1ada47;
-    padding: 3px;
-    border-radius: 5px;
-}
-
-.nl-Noun {
-    background-color: #151ffa;
-    padding: 3px;
-    border-radius: 5px;
-    color: white;
-}
-
-.nl-Verb {
-    background-color: #ff1616;
-    padding: 3px;
-    border-radius: 5px;
-}
-
-
-.hiddenForms {
-    opacity: 0.2;
-}
-
-h1 {
-    color: black;
-}
-
-
-/* ==========================================================================
-   Media Queries
-   ========================================================================== */
-
-/*==========  Non-Mobile First Method  ==========*/
-
-/*Above */
-
-@media only screen and (min-width: 1201px) {}
-
-/* Large Devices, Wide Screens */
-
-@media only screen and (max-width: 1200px) {}
-
-/* Medium Devices, Desktops */
-
-@media only screen and (max-width: 992px) {}
-
-/* Small Devices, Tablets */
-
-@media only screen and (max-width: 768px) {}
-
-/* Extra Small Devices, Phones */
-
-@media only screen and (max-width: 480px) {}
-
-/* Custom, iPhone Retina */
-
-@media only screen and (max-width: 320px) {}
-
-@media print, (-o-min-device-pixel-ratio: 5/4), (-webkit-min-device-pixel-ratio: 1.25), (min-resolution: 120dpi) {}
-
-/* ==========================================================================
-   Helper classes
-   ========================================================================== */
-
-.hidden {
-    display: none !important;
-    visibility: hidden;
-}
-
-.visuallyhidden {
-    border: 0;
-    clip: rect(0 0 0 0);
-    height: 1px;
-    margin: -1px;
-    overflow: hidden;
-    padding: 0;
-    position: absolute;
-    width: 1px;
-}
-
-.visuallyhidden.focusable:active, .visuallyhidden.focusable:focus {
-    clip: auto;
-    height: auto;
-    margin: 0;
-    overflow: visible;
-    position: static;
-    width: auto;
-}
-
-.invisible {
-    visibility: hidden;
-}
-
-.clearfix:before, .clearfix:after {
-    content: " ";
-    display: table;
-}
-
-.clearfix:after {
-    clear: both;
-}
-
-.clearfix {
-    *zoom: 1;
-}
-
-/* ==========================================================================
-   Print styles
-   ========================================================================== */
-
-@media print {
-    *, *:before, *:after {
-        background: transparent !important;
-        color: #000 !important;
-        box-shadow: none !important;
-        text-shadow: none !important;
-    }
-    a, a:visited {
-        text-decoration: underline;
-    }
-    a[href]:after {
-        content: " (" attr(href) ")";
-    }
-    abbr[title]:after {
-        content: " (" attr(title) ")";
-    }
-    a[href^="#"]:after, a[href^="javascript:"]:after {
-        content: "";
-    }
-    pre, blockquote {
-        border: 1px solid #999;
-        page-break-inside: avoid;
-    }
-    thead {
-        display: table-header-group;
-    }
-    tr, img {
-        page-break-inside: avoid;
-    }
-    img {
-        max-width: 100% !important;
-    }
-    p, h2, h3 {
-        orphans: 3;
-        widows: 3;
-    }
-    h2, h3 {
-        page-break-after: avoid;
-    }
-}