Update app.py

app.py

@@ -4,7 +4,7 @@ import numpy as np
 import torch
 import torch.nn as nn
 import torch.optim as optim
-import torchvision
+import seaborn as sns
 from facenet_pytorch import InceptionResnetV1, MTCNN
 import mediapipe as mp
 from fer import FER
@@ -12,6 +12,7 @@ from sklearn.cluster import KMeans
 from sklearn.preprocessing import StandardScaler, MinMaxScaler
 import umap
 import pandas as pd
+import matplotlib
 import matplotlib.pyplot as plt
 from matplotlib.ticker import MaxNLocator
 from moviepy.editor import VideoFileClip
@@ -19,8 +20,7 @@ from PIL import Image
 import gradio as gr
 import tempfile
 import shutil
-import subprocess
-import fractions
+
 
 
 # Suppress TensorFlow warnings
@@ -29,6 +29,9 @@ import tensorflow as tf
 
 tf.get_logger().setLevel('ERROR')
 
+matplotlib.rcParams['figure.dpi'] = 400
+matplotlib.rcParams['savefig.dpi'] = 400
+
 # Initialize models and other global variables
 device = 'cuda'
 
@@ -40,8 +43,8 @@ face_mesh = mp_face_mesh.FaceMesh(static_image_mode=False, max_num_faces=1, min_
 emotion_detector = FER(mtcnn=False)
 
 
-def frame_to_timecode(frame_num, original_fps, desired_fps):
-    total_seconds = frame_num / original_fps
+def frame_to_timecode(frame_num, total_frames, duration):
+    total_seconds = (frame_num / total_frames) * duration
     hours = int(total_seconds // 3600)
     minutes = int((total_seconds % 3600) // 60)
     seconds = int(total_seconds % 60)
@@ -116,8 +119,8 @@ def extract_frames(video_path, output_folder, desired_fps, progress_callback=Non
 
         # Report progress
         if progress_callback:
-            progress = frame_count / total_frames_to_extract
-            progress_callback(progress, f"Extracting frame {frame_count} of {total_frames_to_extract}")
+            progress = min(100, (frame_count / total_frames_to_extract) * 100)  # Ensure it doesn't exceed 100%
+            progress_callback(progress, f"Extracting frame")
 
         if frame_count >= total_frames_to_extract:
             break
@@ -190,7 +193,7 @@ def organize_faces_by_person(embeddings_by_frame, clusters, aligned_faces_folder
 
 
 def save_person_data_to_csv(embeddings_by_frame, emotions_by_frame, clusters, desired_fps, original_fps, output_folder,
-                            num_components):
+                            num_components, video_duration):
     emotions = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral']
     person_data = {}
 
@@ -215,8 +218,9 @@ def save_person_data_to_csv(embeddings_by_frame, emotions_by_frame, clusters, de
     scaler = MinMaxScaler(feature_range=(0, 1))
     embeddings_reduced_normalized = scaler.fit_transform(embeddings_reduced)
 
-    timecodes = [frame_to_timecode(frame, original_fps, desired_fps) for frame in frames]
-    times_in_minutes = [frame / (original_fps * 60) for frame in frames]
+    total_frames = max(frames)
+    timecodes = [frame_to_timecode(frame, total_frames, video_duration) for frame in frames]
+    times_in_minutes = [frame / total_frames * video_duration / 60 for frame in frames]
 
     df_data = {
         'Frame': frames,
@@ -253,7 +257,7 @@ class LSTMAutoencoder(nn.Module):
 
 
 def lstm_anomaly_detection(X, feature_columns, num_anomalies=10, epochs=100, batch_size=64):
-    device = 'cuda' if torch.cuda.is_available() else 'cpu'
+    device = 'cuda'
 
     X = torch.FloatTensor(X).to(device)
 
@@ -316,42 +320,80 @@ def lstm_anomaly_detection(X, feature_columns, num_anomalies=10, epochs=100, bat
             model)
 
 
-def plot_anomaly_scores(df, anomaly_scores, top_indices, title):
+def plot_emotion(df, emotion, num_anomalies, color):
+    plt.figure(figsize=(16, 8), dpi=400)  # Increase DPI for higher quality
     fig, ax = plt.subplots(figsize=(16, 8))
-    bars = ax.bar(range(len(df)), anomaly_scores, width=0.8, color='skyblue')
-    for i in top_indices:
-        bars[i].set_color('red')
-    ax.set_xlabel('Timecode')
-    ax.set_ylabel('Anomaly Score')
-    ax.set_title(f'Anomaly Scores Over Time ({title})')
-    ax.xaxis.set_major_locator(MaxNLocator(nbins=100))
-    ticks = ax.get_xticks()
-    ax.set_xticklabels([df['Timecode'].iloc[int(tick)] if tick >= 0 and tick < len(df) else '' for tick in ticks],
-                       rotation=90, ha='right')
-    plt.tight_layout()
-    return fig
 
+    # Convert timecodes to seconds for proper plotting
+    df['Seconds'] = df['Timecode'].apply(
+        lambda x: sum(float(t) * 60 ** i for i, t in enumerate(reversed(x.split(':')))))
 
-def plot_emotion(df, emotion, num_anomalies, color):
-    fig, ax = plt.subplots(figsize=(16, 8))
-    values = df[emotion].values
-    bars = ax.bar(range(len(df)), values, width=0.9, color=color)
-    top_indices = np.argsort(values)[-num_anomalies:][::-1]
-    for i in top_indices:
-        bars[i].set_color('red')
-    ax.set_xlabel('Timecode')
+    # Create a DataFrame for seaborn
+    plot_df = pd.DataFrame({
+        'Seconds': df['Seconds'],
+        'Emotion Score': df[emotion]
+    })
+
+    # Plot using seaborn
+    sns.lineplot(x='Seconds', y='Emotion Score', data=plot_df, ax=ax, color=color)
+
+    # Highlight top anomalies
+    top_indices = np.argsort(df[emotion].values)[-num_anomalies:][::-1]
+    ax.scatter(df['Seconds'].iloc[top_indices], df[emotion].iloc[top_indices], color='red', s=50, zorder=5)
+
+    # Set x-axis
+    max_seconds = df['Seconds'].max()
+    ax.set_xlim(0, max_seconds)
+    num_ticks = 80  # Reduce number of ticks for emotion graphs
+    ax.set_xticks(np.linspace(0, max_seconds, num_ticks))
+    ax.set_xticklabels([f"{int(x // 60):02d}:{int(x % 60):02d}" for x in ax.get_xticks()], rotation=90, ha='right')
+
+    ax.set_xlabel('Time')
     ax.set_ylabel(f'{emotion.capitalize()} Score')
-    ax.set_title(f'{emotion.capitalize()} Anomalies Over Time (Top {num_anomalies} in Red)')
-    ax.xaxis.set_major_locator(MaxNLocator(nbins=100))
-    ticks = ax.get_xticks()
-    ax.set_xticklabels([df['Timecode'].iloc[int(tick)] if tick >= 0 and tick < len(df) else '' for tick in ticks],
-                       rotation=90, ha='right')
+    ax.set_title(f'{emotion.capitalize()} Scores Over Time (Top {num_anomalies} in Red)')
+
+    # Add grid
+    ax.grid(True, linestyle='--', alpha=0.7)
+
     plt.tight_layout()
     return fig
 
+def plot_anomaly_scores(df, anomaly_scores, top_indices, title):
+    plt.figure(figsize=(16, 8), dpi=400)  # Increase DPI for higher quality
+    fig, ax = plt.subplots(figsize=(16, 8))
+
+    # Convert timecodes to seconds for proper plotting
+    df['Seconds'] = df['Timecode'].apply(
+        lambda x: sum(float(t) * 60 ** i for i, t in enumerate(reversed(x.split(':')))))
+
+    # Create a DataFrame for seaborn
+    plot_df = pd.DataFrame({
+        'Seconds': df['Seconds'],
+        'Anomaly Score': anomaly_scores
+    })
+
+    # Plot using seaborn
+    sns.lineplot(x='Seconds', y='Anomaly Score', data=plot_df, ax=ax)
+
+    # Highlight top anomalies
+    ax.scatter(df['Seconds'].iloc[top_indices], anomaly_scores[top_indices], color='red', s=50, zorder=5)
+
+    # Set x-axis
+    max_seconds = df['Seconds'].max()
+    ax.set_xlim(0, max_seconds)
+    num_ticks = 80  # Increase number of ticks for anomaly score graphs
+    ax.set_xticks(np.linspace(0, max_seconds, num_ticks))
+    ax.set_xticklabels([f"{int(x // 60):02d}:{int(x % 60):02d}" for x in ax.get_xticks()], rotation=90, ha='right')
+
+    ax.set_xlabel('Time')
+    ax.set_ylabel('Anomaly Score')
+    ax.set_title(f'Anomaly Scores Over Time ({title})')
 
-import base64
+    # Add grid
+    ax.grid(True, linestyle='--', alpha=0.7)
 
+    plt.tight_layout()
+    return fig
 
 def get_random_face_sample(organized_faces_folder, largest_cluster, output_folder):
     person_folder = os.path.join(organized_faces_folder, f"person_{largest_cluster}")
@@ -363,7 +405,7 @@ def get_random_face_sample(organized_faces_folder, largest_cluster, output_folde
 
         # Read the image and resize it to be smaller
         face_img = cv2.imread(face_path)
-        small_face = cv2.resize(face_img, (100, 100))  # Resize to NxN pixels
+        small_face = cv2.resize(face_img, (160, 160))  # Resize to NxN pixels
         cv2.imwrite(output_path, small_face)
 
         return output_path
@@ -380,16 +422,20 @@ def process_video(video_path, num_anomalies, num_components, desired_fps, batch_
         os.makedirs(aligned_faces_folder, exist_ok=True)
         os.makedirs(organized_faces_folder, exist_ok=True)
 
-        progress(0.05, "Starting frame extraction")
+
+        clip = VideoFileClip(video_path)
+        video_duration = clip.duration
+        clip.close()
+
+        progress(0, "Starting frame extraction")
         frames_folder = os.path.join(temp_dir, 'extracted_frames')
 
         def extraction_progress(percent, message):
-            # Adjust the progress to fit within the 5% to 30% range of the overall process
-            overall_progress = 0.05 + (percent * 0.25)
-            progress(overall_progress, message)
+            progress(percent / 100, f"Extracting frames")
 
         frame_count, original_fps = extract_frames(video_path, frames_folder, desired_fps, extraction_progress)
 
+        progress(1, "Frame extraction complete")
 
         progress(0.3, "Processing frames")
         embeddings_by_frame, emotions_by_frame = process_frames(frames_folder, aligned_faces_folder, frame_count,
@@ -407,7 +453,7 @@ def process_video(video_path, num_anomalies, num_components, desired_fps, batch_
 
         progress(0.8, "Saving person data")
         df, largest_cluster = save_person_data_to_csv(embeddings_by_frame, emotions_by_frame, clusters, desired_fps,
-                                                      original_fps, temp_dir, num_components)
+                                                      original_fps, temp_dir, num_components, video_duration)
 
         progress(0.9, "Performing anomaly detection")
         feature_columns = [col for col in df.columns if
@@ -472,12 +518,11 @@ iface = gr.Interface(
         gr.Video(),
         gr.Slider(minimum=1, maximum=20, step=1, value=5, label="Number of Anomalies"),
         gr.Slider(minimum=1, maximum=20, step=1, value=10, label="Number of Components"),
-        gr.Slider(minimum=1, maximum=30, step=1, value=20, label="Desired FPS"),
-        gr.Slider(minimum=1, maximum=64, step=1, value=16, label="Batch Size")
+        gr.Slider(minimum=1, maximum=20, step=1, value=10, label="Desired FPS"),
+        gr.Slider(minimum=1, maximum=64, step=1, value=8, label="Batch Size")
     ],
     outputs=[
         gr.Textbox(label="Anomaly Detection Results"),
-        gr.Image(type="filepath", label="Random Face Sample of Most Frequent Person"),
         gr.Plot(label="Anomaly Scores (All Features)"),
         gr.Plot(label="Anomaly Scores (Components Only)"),
         gr.Plot(label="Fear Anomalies"),
@@ -485,7 +530,8 @@ iface = gr.Interface(
         gr.Plot(label="Angry Anomalies"),
         gr.Plot(label="Happy Anomalies"),
         gr.Plot(label="Surprise Anomalies"),
-        gr.Plot(label="Neutral Anomalies")
+        gr.Plot(label="Neutral Anomalies"),
+        gr.Image(type="filepath", label="Random Face Sample of Most Frequent Person"),
     ],
     title="Facial Expressions Anomaly Detection",
     description="""