|
import gradio as gr |
|
import time |
|
from video_processing import process_video |
|
from PIL import Image |
|
import matplotlib |
|
|
|
matplotlib.rcParams['figure.dpi'] = 300 |
|
matplotlib.rcParams['savefig.dpi'] = 300 |
|
|
|
def process_and_show_completion(video_input_path, anomaly_threshold_input, fps, progress=gr.Progress()): |
|
|
|
|
|
def show_results(outputs): |
|
return [gr.Tab.update(visible=True) for _ in range(4)] + [gr.Tab.update(visible=False)], gr.Group(visible=True) |
|
|
|
def hide_description_show_results(): |
|
return [gr.Tab.update(visible=False)] + [gr.Tab.update(visible=True) for _ in range(4)] |
|
|
|
with gr.Blocks() as iface: |
|
with gr.Row(): |
|
video_input = gr.Video(label="Input Video", visible=False) |
|
|
|
anomaly_threshold = gr.Slider(minimum=1, maximum=5, step=0.1, value=3, label="Anomaly Detection Threshold (Standard deviation)") |
|
fps_slider = gr.Slider(minimum=5, maximum=20, step=1, value=10, label="Frames Per Second (FPS)") |
|
process_btn = gr.Button("Detect Anomalies") |
|
progress_bar = gr.Progress() |
|
|
|
execution_time_group = gr.Group(visible=False) |
|
with execution_time_group: |
|
execution_time = gr.Number(label="Execution Time (seconds)") |
|
|
|
with gr.Tabs() as all_tabs: |
|
with gr.Tab("Description", visible=True): |
|
gr.Markdown(""" |
|
# Multimodal Behavioral Anomalies Detection |
|
|
|
This tool detects anomalies in facial expressions, body language, and voice over the timeline of a video. |
|
It extracts faces, postures, and voice from video frames, and analyzes them to identify anomalies using time series analysis and a variational autoencoder (VAE) approach. |
|
""") |
|
|
|
with gr.Tab("Facial Features", visible=False): |
|
results_text = gr.TextArea(label="Faces Breakdown", lines=5) |
|
mse_features_plot = gr.Plot(label="MSE: Facial Features") |
|
mse_features_hist = gr.Plot(label="MSE Distribution: Facial Features") |
|
mse_features_heatmap = gr.Plot(label="MSE Heatmap: Facial Features") |
|
anomaly_frames_features = gr.Gallery(label="Anomaly Frames (Facial Features)", columns=6, rows=2, height="auto") |
|
face_samples_most_frequent = gr.Gallery(label="Most Frequent Person Samples", columns=10, rows=2, height="auto") |
|
|
|
with gr.Tab("Body Posture", visible=False): |
|
mse_posture_plot = gr.Plot(label="MSE: Body Posture") |
|
mse_posture_hist = gr.Plot(label="MSE Distribution: Body Posture") |
|
mse_posture_heatmap = gr.Plot(label="MSE Heatmap: Body Posture") |
|
anomaly_frames_posture = gr.Gallery(label="Anomaly Frames (Body Posture)", columns=6, rows=2, height="auto") |
|
|
|
with gr.Tab("Voice", visible=False): |
|
mse_voice_plot = gr.Plot(label="MSE: Voice") |
|
mse_voice_hist = gr.Plot(label="MSE Distribution: Voice") |
|
mse_voice_heatmap = gr.Plot(label="MSE Heatmap: Voice") |
|
|
|
with gr.Tab("Combined", visible=False): |
|
heatmap_video = gr.Video(label="Video with Anomaly Heatmap") |
|
combined_mse_plot = gr.Plot(label="Combined MSE Plot") |
|
correlation_heatmap_plot = gr.Plot(label="Correlation Heatmap") |
|
|
|
df_store = gr.State() |
|
mse_features_store = gr.State() |
|
mse_posture_store = gr.State() |
|
mse_voice_store = gr.State() |
|
aligned_faces_folder_store = gr.State() |
|
frames_folder_store = gr.State() |
|
mse_heatmap_embeddings_store = gr.State() |
|
mse_heatmap_posture_store = gr.State() |
|
mse_heatmap_voice_store = gr.State() |
|
|
|
process_btn.click( |
|
hide_description_show_results, |
|
inputs=None, |
|
outputs=all_tabs.children |
|
).then( |
|
process_and_show_completion, |
|
inputs=[video_input, anomaly_threshold, fps_slider], |
|
outputs=[ |
|
execution_time, results_text, df_store, |
|
mse_features_store, mse_posture_store, mse_voice_store, |
|
mse_features_plot, mse_posture_plot, mse_voice_plot, |
|
mse_features_hist, mse_posture_hist, mse_voice_hist, |
|
mse_features_heatmap, mse_posture_heatmap, mse_voice_heatmap, |
|
anomaly_frames_features, anomaly_frames_posture, |
|
face_samples_most_frequent, |
|
aligned_faces_folder_store, frames_folder_store, |
|
mse_heatmap_embeddings_store, mse_heatmap_posture_store, mse_heatmap_voice_store, |
|
heatmap_video, combined_mse_plot, correlation_heatmap_plot |
|
] |
|
).then( |
|
show_results, |
|
inputs=None, |
|
outputs=[all_tabs.children, execution_time_group] |
|
) |
|
|
|
if __name__ == "__main__": |
|
iface.launch() |
