Update app.py

app.py

@@ -2,10 +2,13 @@ import gradio as gr
 import time
 from video_processing import process_video
 from PIL import Image
-import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
 
-matplotlib.rcParams['figure.dpi'] = 300
-matplotlib.rcParams['savefig.dpi'] = 300
+# Ensure high DPI plots
+plt.rcParams['figure.dpi'] = 300
+plt.rcParams['savefig.dpi'] = 300
 
 def process_and_show_completion(video_input_path, anomaly_threshold_input, fps, progress=gr.Progress()):
     try:
@@ -33,14 +36,11 @@ def process_and_show_completion(video_input_path, anomaly_threshold_input, fps,
 
         output = [
             exec_time, results_summary,
-            df, mse_embeddings, mse_posture, mse_voice,
             mse_plot_embeddings, mse_plot_posture, mse_plot_voice,
             mse_histogram_embeddings, mse_histogram_posture, mse_histogram_voice,
             mse_heatmap_embeddings, mse_heatmap_posture, mse_heatmap_voice,
             anomaly_faces_embeddings_pil, anomaly_frames_posture_pil,
             face_samples_frequent,
-            aligned_faces_folder, frames_folder,
-            mse_embeddings, mse_posture, mse_voice,
             heatmap_video_path, combined_mse_plot, correlation_heatmap
         ]
 
@@ -51,10 +51,32 @@ def process_and_show_completion(video_input_path, anomaly_threshold_input, fps,
         print(error_message)
         import traceback
         traceback.print_exc()
-        return [error_message] + [None] * 27
+        return [error_message] + [None] * 16
 
-def show_results(outputs):
-    return gr.Group(visible=True)
+def on_button_click(video, threshold, fps):
+    results = process_and_show_completion(video, threshold, fps)
+    
+    return {
+        execution_time: gr.update(visible=True, value=results[0]),
+        results_tab: gr.update(visible=True),
+        description_tab: gr.update(visible=False),
+        results_text: results[1],
+        mse_features_plot: results[2],
+        mse_posture_plot: results[3],
+        mse_voice_plot: results[4],
+        mse_features_hist: results[5],
+        mse_posture_hist: results[6],
+        mse_voice_hist: results[7],
+        mse_features_heatmap: results[8],
+        mse_posture_heatmap: results[9],
+        mse_voice_heatmap: results[10],
+        anomaly_frames_features: results[11],
+        anomaly_frames_posture: results[12],
+        face_samples_most_frequent: results[13],
+        heatmap_video: results[14],
+        combined_mse_plot: results[15],
+        correlation_heatmap_plot: results[16]
+    }
 
 with gr.Blocks() as iface:
     gr.Markdown("""
@@ -71,10 +93,11 @@ with gr.Blocks() as iface:
     fps_slider = gr.Slider(minimum=5, maximum=20, step=1, value=10, label="Frames Per Second (FPS)")
     process_btn = gr.Button("Detect Anomalies")
     progress_bar = gr.Progress()
-    execution_time = gr.Number(label="Execution Time (seconds)")
+    
+    execution_time = gr.Number(label="Execution Time (seconds)", visible=False)
 
-    with gr.Tabs():
-        with gr.TabItem("Description"):
+    with gr.Tabs() as tabs:
+        with gr.TabItem("Description", id="description_tab") as description_tab:
             with gr.Column():
                 gr.Markdown("""
                 # Multimodal Behavioral Anomalies Detection
@@ -90,10 +113,29 @@ with gr.Blocks() as iface:
                 - Monitor and assess emotional states in communications.
                 - Evaluate changes in vocal tone and speech patterns.
 
-                ... (rest of the description content)
+                ## Features
+
+                - **Face Extraction**: Extracts faces from video frames using the MTCNN model.
+                - **Feature Embeddings**: Extracts facial feature embeddings using the InceptionResnetV1 model.
+                - **Body Posture Analysis**: Evaluates body postures using MediaPipe Pose.
+                - **Voice Analysis**: Extracts and segment speaker embeddings from audio using PyAnnote.
+                - **Anomaly Detection**: Uses Variational Autoencoder (VAE) to detect anomalies in facial expressions, body postures, and voice features over time.
+                - **Visualization**: Represents changes in facial expressions, body postures, and vocal tone over time, marking anomaly key points.
+
+                ## Limitations
+
+                - **Evaluation Challenges**: Since this is an unsupervised method, there is no labeled data to compare against.
+                - **Subjectivity**: The concept of what constitutes an "anomaly" can be subjective and context-dependent.
+                - **Lighting and Resolution**: Variability in lighting conditions and camera resolution can affect the quality of detected features.
+                - **Audio Quality**: Background noise and poor audio quality can affect the accuracy of voice analysis.
+                - **Generalization**: The model may not generalize well to all types of videos and contexts.
+                - **Computationally Intensive**: Processing high-resolution video frames can be computationally demanding.
+
+                ## Conclusion
+                This tool offers solutions for detecting behavioral anomalies in video content. However, users should be aware of its limitations and interpret results with caution.
                 """)
 
-        with gr.TabItem("Results", visible=False) as results_group:
+        with gr.TabItem("Results", id="results_tab", visible=False) as results_tab:
             with gr.Tabs():
                 with gr.TabItem("Facial Features"):
                     results_text = gr.TextArea(label="Faces Breakdown", lines=5)
@@ -119,37 +161,19 @@ with gr.Blocks() as iface:
                     combined_mse_plot = gr.Plot(label="Combined MSE Plot")
                     correlation_heatmap_plot = gr.Plot(label="Correlation Heatmap")
 
-                
-    df_store = gr.State()
-    mse_features_store = gr.State()
-    mse_posture_store = gr.State()
-    mse_voice_store = gr.State()
-    aligned_faces_folder_store = gr.State()
-    frames_folder_store = gr.State()
-    mse_heatmap_embeddings_store = gr.State()
-    mse_heatmap_posture_store = gr.State()
-    mse_heatmap_voice_store = gr.State()
-    
     process_btn.click(
-        process_and_show_completion,
+        fn=on_button_click,
         inputs=[video_input, anomaly_threshold, fps_slider],
         outputs=[
-            execution_time, results_text, df_store,
-            mse_features_store, mse_posture_store, mse_voice_store,
-            mse_features_plot, mse_posture_plot, mse_voice_plot,
+            execution_time, results_tab, description_tab,
+            results_text, mse_features_plot, mse_posture_plot, mse_voice_plot,
             mse_features_hist, mse_posture_hist, mse_voice_hist,
             mse_features_heatmap, mse_posture_heatmap, mse_voice_heatmap,
             anomaly_frames_features, anomaly_frames_posture,
-            face_samples_most_frequent,
-            aligned_faces_folder_store, frames_folder_store,
-            mse_heatmap_embeddings_store, mse_heatmap_posture_store, mse_heatmap_voice_store,
-            heatmap_video, combined_mse_plot, correlation_heatmap_plot
+            face_samples_most_frequent, heatmap_video, combined_mse_plot,
+            correlation_heatmap_plot
         ]
-    ).then(
-        show_results,
-        inputs=None,
-        outputs=results_group
     )
 
 if __name__ == "__main__":
-    iface.launch()
+    iface.launch()