Update app.py

app.py

@@ -259,60 +259,123 @@ def transcribe_and_predict_video(video):
 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
 client = InferenceClient(MODEL_NAME)
 
-# Chatbot response function
-def respond(video, history, system_message, max_tokens, temperature, top_p):
-    video_path = video.name  # Get the uploaded video file path
-
-    # Process the video for emotions & text
-    result = transcribe_and_predict_video(video_path)
+# # Chatbot response function
+# def respond(video, history, system_message, max_tokens, temperature, top_p):
+#     video_path = video.name  # Get the uploaded video file path
+
+#     # Process the video for emotions & text
+#     result = transcribe_and_predict_video(video_path)
+
+#     # Construct a system prompt with extracted emotions & text
+#     system_prompt = (
+#         f"{system_message}\n\n"
+#         f"Detected Emotions:\n"
+#         f"- Text Emotion: {result['text_emotion']}\n"
+#         f"- Audio Emotion: {result['audio_emotion']}\n"
+#         f"- Image Emotion: {result['image_emotion']}\n\n"
+#         f"Extracted Speech: {result['extracted_text']}"
+#     )
+
+#     messages = [{"role": "system", "content": system_prompt}]
+
+#     for val in history:
+#         if val[0]:
+#             messages.append({"role": "user", "content": val[0]})
+#         if val[1]:
+#             messages.append({"role": "assistant", "content": val[1]})
+
+#     messages.append({"role": "user", "content": result['extracted_text']})
+
+#     response = ""
+
+#     try:
+#         for message in client.chat_completion(
+#             messages,
+#             max_tokens=max_tokens,
+#             stream=True,
+#             temperature=temperature,
+#             top_p=top_p,
+#         ):
+#             token = message.choices[0].delta.content if message.choices[0].delta else ""
+#             response += token
+#             yield response
+#     except Exception as e:
+#         yield f"Error: {str(e)}"
+
+# # Gradio UI for video chatbot
+# demo = gr.ChatInterface(
+#     respond,
+#     additional_inputs=[
+#         gr.Video(label="Upload a Video"),  # Video input
+#         gr.Textbox(value="You are a chatbot that analyzes emotions and responds accordingly.", label="System message"),
+#         gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max Tokens"),
+#         gr.Slider(minimum=0.1, maximum=2.0, value=0.7, step=0.1, label="Temperature"),
+#         gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p"),
+#     ],
+# )
+
+# if __name__ == "__main__":
+#     demo.launch()
+
+
+
+def respond(video, text_input, history):
+    """Processes user input (video, text, or both) and generates a chatbot response."""
+    messages = []
+    system_prompt = "You are a chatbot that can analyze emotions from videos and respond accordingly."
+
+    if video is not None:
+        video_path = video.name  # Get uploaded video path
+        result = transcribe_and_predict_video(video_path)
+        
+        system_prompt += f"\n\nDetected Emotions:\n"
+        system_prompt += f"- Text Emotion: {result['text_emotion']}\n"
+        system_prompt += f"- Audio Emotion: {result['audio_emotion']}\n"
+        system_prompt += f"- Image Emotion: {result['image_emotion']}\n\n"
+        system_prompt += f"Extracted Speech: {result['extracted_text']}\n"
 
-    # Construct a system prompt with extracted emotions & text
-    system_prompt = (
-        f"{system_message}\n\n"
-        f"Detected Emotions:\n"
-        f"- Text Emotion: {result['text_emotion']}\n"
-        f"- Audio Emotion: {result['audio_emotion']}\n"
-        f"- Image Emotion: {result['image_emotion']}\n\n"
-        f"Extracted Speech: {result['extracted_text']}"
-    )
+        messages.append({"role": "user", "content": result["extracted_text"]})  # Add extracted speech
 
-    messages = [{"role": "system", "content": system_prompt}]
+    if text_input:
+        messages.append({"role": "user", "content": text_input})  # Add text input if provided
 
+    # Append chat history
     for val in history:
         if val[0]:
             messages.append({"role": "user", "content": val[0]})
         if val[1]:
             messages.append({"role": "assistant", "content": val[1]})
 
-    messages.append({"role": "user", "content": result['extracted_text']})
+    # Include system prompt
+    messages.insert(0, {"role": "system", "content": system_prompt})
 
     response = ""
-
+    
     try:
-        for message in client.chat_completion(
-            messages,
-            max_tokens=max_tokens,
-            stream=True,
-            temperature=temperature,
-            top_p=top_p,
-        ):
+        for message in client.chat_completion(messages, max_tokens=512, stream=True, temperature=0.7, top_p=0.95):
             token = message.choices[0].delta.content if message.choices[0].delta else ""
             response += token
             yield response
     except Exception as e:
         yield f"Error: {str(e)}"
 
-# Gradio UI for video chatbot
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Video(label="Upload a Video"),  # Video input
-        gr.Textbox(value="You are a chatbot that analyzes emotions and responds accordingly.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max Tokens"),
-        gr.Slider(minimum=0.1, maximum=2.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p"),
-    ],
-)
+# Define ChatGPT-style UI
+with gr.Blocks(theme="soft") as demo:
+    gr.Markdown("<h2 align='center'>📹🎤💬 Multi-Modal Chatbot (Video + Text) </h2>")
+
+    chatbot = gr.Chatbot(label="ChatGPT-Like Chat")
+    video_input = gr.Video(label="Upload Video (Optional)")
+    text_input = gr.Textbox(label="Enter Text (Optional)", placeholder="Type your message here...")
+    
+    clear_button = gr.Button("Clear Chat")
+
+    def clear_chat():
+        return [], None, None
+
+    video_input.change(clear_chat, outputs=[chatbot, video_input, text_input])
+    text_input.submit(respond, inputs=[video_input, text_input, chatbot], outputs=[chatbot])
+    clear_button.click(clear_chat, outputs=[chatbot, video_input, text_input])
 
+# Launch chatbot
 if __name__ == "__main__":
     demo.launch()