with gr.Blocks() as iface: gr.Markdown(""" # Multimodal Behavioral Anomalies Detection This tool detects anomalies in facial expressions, body language, and voice over the timeline of a video. It extracts faces, postures, and voice from video frames, and analyzes them to identify anomalies using time series analysis and a variational autoencoder (VAE) approach. """) with gr.Row(): video_input = gr.Video() anomaly_threshold = gr.Slider(minimum=1, maximum=5, step=0.1, value=3, label="Anomaly Detection Threshold (Standard deviation)") fps_slider = gr.Slider(minimum=5, maximum=20, step=1, value=10, label="Frames Per Second (FPS)") process_btn = gr.Button("Detect Anomalies") progress_bar = gr.Progress() execution_time = gr.Number(label="Execution Time (seconds)") with gr.Group(visible=False) as results_group: results_text = gr.TextArea(label="Anomaly Detection Results", lines=4) with gr.Tabs(): with gr.TabItem("Facial Features"): mse_features_plot = gr.Plot(label="MSE: Facial Features") mse_features_hist = gr.Plot(label="MSE Distribution: Facial Features") mse_features_heatmap = gr.Plot(label="MSE Heatmap: Facial Features") anomaly_frames_features = gr.Gallery(label="Anomaly Frames (Facial Features)", columns=6, rows=2, height="auto") face_samples_most_frequent = gr.Gallery(label="Most Frequent Person Samples", columns=10, rows=2, height="auto") with gr.TabItem("Body Posture"): mse_posture_plot = gr.Plot(label="MSE: Body Posture") mse_posture_hist = gr.Plot(label="MSE Distribution: Body Posture") mse_posture_heatmap = gr.Plot(label="MSE Heatmap: Body Posture") anomaly_frames_posture = gr.Gallery(label="Anomaly Frames (Body Posture)", columns=6, rows=2, height="auto") with gr.TabItem("Voice"): mse_voice_plot = gr.Plot(label="MSE: Voice") mse_voice_hist = gr.Plot(label="MSE Distribution: Voice") mse_voice_heatmap = gr.Plot(label="MSE Heatmap: Voice") with gr.TabItem("Video with Heatmap"): heatmap_video = gr.Video(label="Video with Anomaly Heatmap") df_store = gr.State() mse_features_store = gr.State() mse_posture_store = gr.State() mse_voice_store = gr.State() aligned_faces_folder_store = gr.State() frames_folder_store = gr.State() mse_heatmap_embeddings_store = gr.State() mse_heatmap_posture_store = gr.State() mse_heatmap_voice_store = gr.State() def show_results(outputs): return gr.Group(visible=True) process_btn.click( process_and_show_completion, inputs=[video_input, anomaly_threshold, fps_slider], outputs=[ execution_time, results_text, df_store, mse_features_store, mse_posture_store, mse_voice_store, mse_features_plot, mse_posture_plot, mse_voice_plot, mse_features_hist, mse_posture_hist, mse_voice_hist, mse_features_heatmap, mse_posture_heatmap, mse_voice_heatmap, anomaly_frames_features, anomaly_frames_posture, face_samples_most_frequent, aligned_faces_folder_store, frames_folder_store, mse_heatmap_embeddings_store, mse_heatmap_posture_store, mse_heatmap_voice_store, heatmap_video ] ).then( show_results, inputs=None, outputs=results_group ) if __name__ == "__main__": iface.launch()