paying with audio

app.py

@@ -3,6 +3,9 @@ from time import sleep
 from json import loads as json_loads
 import gradio as gr
 from openai import OpenAI
+# import assemblyai
+# import queue
+
 client = OpenAI()
 
 assistant_id = os_getenv("OPENAI_ASSISTANT_ID")
@@ -97,6 +100,8 @@ def new_state():
     return gr.State({
         "user": None,
         "thread": None,
+        "text_queue": None,
+        "transcriber": None,
     })
 
 def auth(token, state):
@@ -117,12 +122,74 @@ AUTH_JS = """function auth_js(token, state) {
 }
 """
 
+def audio_on_load(state):
+    q = state["text_queue"] = queue.Queue()
+
+    transcriber = assemblyai.RealtimeTranscriber(
+        on_data=q.put,
+        on_error=q.put,
+        sample_rate=48000,
+    )
+    transcriber.connect()
+
+    state["transcriber"] = transcriber
+    return state, gr.update(interactive=True)
+
+def audio_start(state, chati, audio):
+    if audio:
+        state["transcriber"].stream(audio[1].tobytes())
+        print(audio[0], len(audio[1]))
+
+    chati += [("", None)]
+    return state, chati
+
+def audio_msg(msg, chati):
+    if msg is None:
+        return False
+    if isinstance(msg, assemblyai.RealtimeFinalTranscript):
+        chati[-1][0]  = msg.text
+        chati += [("", None)]
+        return True
+    elif isinstance(msg, assemblyai.RealtimePartialTranscript):
+        chati[-1][0] = msg.text
+    elif isinstance(msg, assemblyai.RealtimeError):
+        print(f"-- {msg}")
+        gr.Info(str(msg))
+    else:
+        print(f"-- {msg}")
+
+    return False
+
+def audio_stop(state, chati, audio):
+    for i in range(20):
+        try:
+            msg = state["text_queue"].get(timeout=1)
+            if audio_msg(msg, chati):
+                break
+        except queue.Empty:
+            pass
+
+    return state, chati
+
+def audio_chunk(state, chati, audio):
+    state["transcriber"].stream(audio[1].tobytes())
+    print(audio[0], len(audio[1]))
+    try:
+        msg = state["text_queue"].get_nowait()
+        audio_msg(msg, chati)
+    except queue.Empty:
+        pass
+    
+    return state, chati
+
 
 with gr.Blocks(
     title="Je suis Le Petit Nicolas",
     fill_height=True, 
     theme=gr.themes.Base()) as demo:
         state = new_state()
+        # mic = gr.Microphone(streaming=True, interactive=False, render=False)
+        # demo.load(audio_on_load, inputs=state, outputs=[state, mic])
 
         gr.HTML("""
             <h1>Je suis Le Petit Nicolas</h1>
@@ -138,7 +205,7 @@ with gr.Blocks(
                 label = "Tout au long de la réponse",
                 scale=3)
 
-        gr.ChatInterface(
+        chati = gr.ChatInterface(
             chat,
             additional_inputs=[state, long_or_short],
             examples=[
@@ -147,6 +214,11 @@ with gr.Blocks(
             ],
         )
 
+        # mic.render()
+        # mic.start_recording(audio_start, inputs=[state, chati.chatbot, mic], outputs=[state, chati.chatbot])
+        # mic.stop_recording(audio_stop, inputs=[state, chati.chatbot, mic], outputs=[state, chati.chatbot])
+        # mic.stream(audio_chunk, inputs=[state, chati.chatbot, mic], outputs=[state, chati.chatbot])
+
         token = gr.Textbox(visible=False)
         demo.load(auth,
             [token,state],
@@ -155,4 +227,4 @@ with gr.Blocks(
 
 demo.launch(
     height=700,
-    allowed_paths=["."])
+    show_api=False)

requirements.txt

@@ -1,3 +1,3 @@
 gradio
 openai
-yagmail
+assemblyai