Update app.py

app.py

@@ -2,13 +2,10 @@ import gradio as gr
 import time
 from video_processing import process_video
 from PIL import Image
-import matplotlib.pyplot as plt
-import numpy as np
-import pandas as pd
+import matplotlib
 
-# Ensure high DPI plots
-plt.rcParams['figure.dpi'] = 300
-plt.rcParams['savefig.dpi'] = 300
+matplotlib.rcParams['figure.dpi'] = 300
+matplotlib.rcParams['savefig.dpi'] = 300
 
 def process_and_show_completion(video_input_path, anomaly_threshold_input, fps, progress=gr.Progress()):
     try:
@@ -36,11 +33,14 @@ def process_and_show_completion(video_input_path, anomaly_threshold_input, fps,
 
         output = [
             exec_time, results_summary,
+            df, mse_embeddings, mse_posture, mse_voice,
             mse_plot_embeddings, mse_plot_posture, mse_plot_voice,
             mse_histogram_embeddings, mse_histogram_posture, mse_histogram_voice,
             mse_heatmap_embeddings, mse_heatmap_posture, mse_heatmap_voice,
             anomaly_faces_embeddings_pil, anomaly_frames_posture_pil,
             face_samples_frequent,
+            aligned_faces_folder, frames_folder,
+            mse_embeddings, mse_posture, mse_voice,
             heatmap_video_path, combined_mse_plot, correlation_heatmap
         ]
 
@@ -51,44 +51,10 @@ def process_and_show_completion(video_input_path, anomaly_threshold_input, fps,
         print(error_message)
         import traceback
         traceback.print_exc()
-        return [error_message] + [None] * 16
+        return [error_message] + [None] * 27
 
-def on_button_click(video, threshold, fps):
-    start_time = time.time()
-    
-    # Show execution time immediately and hide description
-    yield {
-        execution_time: gr.update(visible=True, value=0),
-        description: gr.update(visible=False),
-        results: gr.update(visible=True)
-    }
-    
-    process_results = process_and_show_completion(video, threshold, fps)
-    end_time = time.time()
-    exec_time = end_time - start_time
-    
-    return {
-        execution_time: gr.update(visible=True, value=exec_time),
-        results_text: process_results[1],
-        mse_features_plot: process_results[2],
-        mse_posture_plot: process_results[3],
-        mse_voice_plot: process_results[4],
-        mse_features_hist: process_results[5],
-        mse_posture_hist: process_results[6],
-        mse_voice_hist: process_results[7],
-        mse_features_heatmap: process_results[8],
-        mse_posture_heatmap: process_results[9],
-        mse_voice_heatmap: process_results[10],
-        anomaly_frames_features: process_results[11],
-        anomaly_frames_posture: process_results[12],
-        face_samples_most_frequent: process_results[13],
-        heatmap_video: process_results[14],
-        combined_mse_plot: process_results[15],
-        correlation_heatmap_plot: process_results[16],
-        video_display_facial: video,
-        video_display_body: video,
-        video_display_voice: video
-    }
+def show_results(outputs):
+    return gr.Group(visible=True)
 
 with gr.Blocks() as iface:
     gr.Markdown("""
@@ -98,92 +64,71 @@ with gr.Blocks() as iface:
     It extracts faces, postures, and voice from video frames, and analyzes them to identify anomalies using time series analysis and a variational autoencoder (VAE) approach.
     """)
 
-    video_input = gr.Video(label="Input Video", visible=True)
+    with gr.Row():
+        video_input = gr.Video()
 
     anomaly_threshold = gr.Slider(minimum=1, maximum=5, step=0.1, value=3, label="Anomaly Detection Threshold (Standard deviation)")
     fps_slider = gr.Slider(minimum=5, maximum=20, step=1, value=10, label="Frames Per Second (FPS)")
     process_btn = gr.Button("Detect Anomalies")
+    progress_bar = gr.Progress()
+    execution_time = gr.Number(label="Execution Time (seconds)")
+
+    with gr.Group(visible=False) as results_group:
+        with gr.Tabs():
+            with gr.TabItem("Facial Features"):
+                results_text = gr.TextArea(label="Faces Breakdown", lines=5)
+                mse_features_plot = gr.Plot(label="MSE: Facial Features")
+                mse_features_hist = gr.Plot(label="MSE Distribution: Facial Features")
+                mse_features_heatmap = gr.Plot(label="MSE Heatmap: Facial Features")
+                anomaly_frames_features = gr.Gallery(label="Anomaly Frames (Facial Features)", columns=6, rows=2, height="auto")
+                face_samples_most_frequent = gr.Gallery(label="Most Frequent Person Samples", columns=10, rows=2, height="auto")
+
+            with gr.TabItem("Body Posture"):
+                mse_posture_plot = gr.Plot(label="MSE: Body Posture")
+                mse_posture_hist = gr.Plot(label="MSE Distribution: Body Posture")
+                mse_posture_heatmap = gr.Plot(label="MSE Heatmap: Body Posture")
+                anomaly_frames_posture = gr.Gallery(label="Anomaly Frames (Body Posture)", columns=6, rows=2, height="auto")
+
+            with gr.TabItem("Voice"):
+                mse_voice_plot = gr.Plot(label="MSE: Voice")
+                mse_voice_hist = gr.Plot(label="MSE Distribution: Voice")
+                mse_voice_heatmap = gr.Plot(label="MSE Heatmap: Voice")
+
+            with gr.TabItem("Combined"):
+                heatmap_video = gr.Video(label="Video with Anomaly Heatmap")
+                combined_mse_plot = gr.Plot(label="Combined MSE Plot")
+                correlation_heatmap_plot = gr.Plot(label="Correlation Heatmap")
+                
+    df_store = gr.State()
+    mse_features_store = gr.State()
+    mse_posture_store = gr.State()
+    mse_voice_store = gr.State()
+    aligned_faces_folder_store = gr.State()
+    frames_folder_store = gr.State()
+    mse_heatmap_embeddings_store = gr.State()
+    mse_heatmap_posture_store = gr.State()
+    mse_heatmap_voice_store = gr.State()
     
-    execution_time = gr.Number(label="Execution Time (seconds)", visible=False)
-
-    description = gr.Markdown(visible=True, value="""
-    # Multimodal Behavioral Anomalies Detection
-    
-    The purpose of this tool is to detect anomalies in facial expressions, body language, and voice over the timeline of a video.   
-    
-    It extracts faces, postures, and voice features from video frames, detects unique facial features, body postures, and speaker embeddings, and analyzes them to identify anomalies using time series analysis, specifically utilizing a variational autoencoder (VAE) approach.   
-    
-    ## Applications
-    
-    - Identify suspicious behavior in surveillance footage.
-    - Analyze micro-expressions.
-    - Monitor and assess emotional states in communications.
-    - Evaluate changes in vocal tone and speech patterns.
-    
-    ## Features
-    
-    - **Face Extraction**: Extracts faces from video frames using the MTCNN model.
-    - **Feature Embeddings**: Extracts facial feature embeddings using the InceptionResnetV1 model.
-    - **Body Posture Analysis**: Evaluates body postures using MediaPipe Pose.
-    - **Voice Analysis**: Extracts and segment speaker embeddings from audio using PyAnnote.
-    - **Anomaly Detection**: Uses Variational Autoencoder (VAE) to detect anomalies in facial expressions, body postures, and voice features over time.
-    - **Visualization**: Represents changes in facial expressions, body postures, and vocal tone over time, marking anomaly key points.
-    
-    ## Limitations
-    
-    - **Evaluation Challenges**: Since this is an unsupervised method, there is no labeled data to compare against.
-    - **Subjectivity**: The concept of what constitutes an "anomaly" can be subjective and context-dependent.
-    - **Lighting and Resolution**: Variability in lighting conditions and camera resolution can affect the quality of detected features.
-    - **Audio Quality**: Background noise and poor audio quality can affect the accuracy of voice analysis.
-    - **Generalization**: The model may not generalize well to all types of videos and contexts.
-    - **Computationally Intensive**: Processing high-resolution video frames can be computationally demanding.
-    
-    ## Conclusion
-    This tool offers solutions for detecting behavioral anomalies in video content. However, users should be aware of its limitations and interpret results with caution.
-    """)
-
-    results = gr.Tabs(visible=False)
-    with results:
-        with gr.TabItem("Facial Features"):
-            video_display_facial = gr.Video(label="Input Video")
-            results_text = gr.TextArea(label="Faces Breakdown", lines=5)
-            mse_features_plot = gr.Plot(label="MSE: Facial Features")
-            mse_features_hist = gr.Plot(label="MSE Distribution: Facial Features")
-            mse_features_heatmap = gr.Plot(label="MSE Heatmap: Facial Features")
-            anomaly_frames_features = gr.Gallery(label="Anomaly Frames (Facial Features)", columns=6, rows=2, height="auto")
-            face_samples_most_frequent = gr.Gallery(label="Most Frequent Person Samples", columns=10, rows=2, height="auto")
-
-        with gr.TabItem("Body Posture"):
-            video_display_body = gr.Video(label="Input Video")
-            mse_posture_plot = gr.Plot(label="MSE: Body Posture")
-            mse_posture_hist = gr.Plot(label="MSE Distribution: Body Posture")
-            mse_posture_heatmap = gr.Plot(label="MSE Heatmap: Body Posture")
-            anomaly_frames_posture = gr.Gallery(label="Anomaly Frames (Body Posture)", columns=6, rows=2, height="auto")
-
-        with gr.TabItem("Voice"):
-            video_display_voice = gr.Video(label="Input Video")
-            mse_voice_plot = gr.Plot(label="MSE: Voice")
-            mse_voice_hist = gr.Plot(label="MSE Distribution: Voice")
-            mse_voice_heatmap = gr.Plot(label="MSE Heatmap: Voice")
-
-        with gr.TabItem("Combined"):
-            heatmap_video = gr.Video(label="Video with Anomaly Heatmap")
-            combined_mse_plot = gr.Plot(label="Combined MSE Plot")
-            correlation_heatmap_plot = gr.Plot(label="Correlation Heatmap")
-
     process_btn.click(
-        fn=on_button_click,
+        process_and_show_completion,
         inputs=[video_input, anomaly_threshold, fps_slider],
         outputs=[
-            execution_time, description, results,
-            results_text, mse_features_plot, mse_posture_plot, mse_voice_plot,
+            execution_time, results_text, df_store,
+            mse_features_store, mse_posture_store, mse_voice_store,
+            mse_features_plot, mse_posture_plot, mse_voice_plot,
             mse_features_hist, mse_posture_hist, mse_voice_hist,
             mse_features_heatmap, mse_posture_heatmap, mse_voice_heatmap,
             anomaly_frames_features, anomaly_frames_posture,
-            face_samples_most_frequent, heatmap_video, combined_mse_plot,
-            correlation_heatmap_plot, video_display_facial, video_display_body, video_display_voice
+            face_samples_most_frequent,
+            aligned_faces_folder_store, frames_folder_store,
+            mse_heatmap_embeddings_store, mse_heatmap_posture_store, mse_heatmap_voice_store,
+            heatmap_video, combined_mse_plot, correlation_heatmap_plot
         ]
+    ).then(
+        show_results,
+        inputs=None,
+        outputs=results_group
     )
 
 if __name__ == "__main__":
-    iface.launch(share=True)
+    iface.launch()