Update app.py

app.py

@@ -8,32 +8,38 @@ import torch.optim as optim
 from facenet_pytorch import InceptionResnetV1, MTCNN
 import tensorflow as tf
 import mediapipe as mp
-from fer import FER
 from sklearn.cluster import DBSCAN
 from sklearn.preprocessing import StandardScaler, MinMaxScaler
 import pandas as pd
 import matplotlib
 import matplotlib.pyplot as plt
+import seaborn as sns
 from matplotlib.patches import Rectangle
 from moviepy.editor import VideoFileClip
-from PIL import Image
+from PIL import Image, ImageDraw, ImageFont
 import gradio as gr
 import tempfile
 import shutil
-import copy
 import time
 
-matplotlib.rcParams['figure.dpi'] = 500
-matplotlib.rcParams['savefig.dpi'] = 500
+
+matplotlib.rcParams['figure.dpi'] = 400
+matplotlib.rcParams['savefig.dpi'] = 400
 
 # Initialize models and other global variables
-device = 'cuda' if torch.cuda.is_available() else 'cpu'
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+FIXED_FPS = 20
 
-mtcnn = MTCNN(keep_all=False, device=device, thresholds=[0.95, 0.95, 0.95], min_face_size=80)
+mtcnn = MTCNN(keep_all=False, device=device, thresholds=[0.95, 0.95, 0.95], min_face_size=100)
 model = InceptionResnetV1(pretrained='vggface2').eval().to(device)
+
 mp_face_mesh = mp.solutions.face_mesh
-face_mesh = mp_face_mesh.FaceMesh(static_image_mode=False, max_num_faces=1, min_detection_confidence=0.5)
-emotion_detector = FER(mtcnn=False)
+face_mesh = mp_face_mesh.FaceMesh(static_image_mode=False, max_num_faces=1, min_detection_confidence=0.8)
+
+mp_pose = mp.solutions.pose
+mp_drawing = mp.solutions.drawing_utils
+pose = mp_pose.Pose(static_image_mode=False, min_detection_confidence=0.8, min_tracking_confidence=0.8)
 
 def frame_to_timecode(frame_num, total_frames, duration):
     total_seconds = (frame_num / total_frames) * duration
@@ -53,20 +59,13 @@ def timecode_to_seconds(timecode):
     h, m, s = map(int, timecode.split(':'))
     return h * 3600 + m * 60 + s
 
-def get_face_embedding_and_emotion(face_img):
+def get_face_embedding(face_img):
     face_tensor = torch.tensor(face_img).permute(2, 0, 1).unsqueeze(0).float() / 255
     face_tensor = (face_tensor - 0.5) / 0.5
     face_tensor = face_tensor.to(device)
     with torch.no_grad():
         embedding = model(face_tensor)
-
-    emotions = emotion_detector.detect_emotions(face_img)
-    if emotions:
-        emotion_dict = emotions[0]['emotions']
-    else:
-        emotion_dict = {e: 0 for e in ['angry', 'disgust', 'fear', 'sad', 'happy']}
-
-    return embedding.cpu().numpy().flatten(), emotion_dict
+    return embedding.cpu().numpy().flatten()
 
 def alignFace(img):
     img_raw = img.copy()
@@ -93,6 +92,53 @@ def alignFace(img):
     new_img = cv2.warpAffine(img_raw, rotation_matrix, (width, height))
     return new_img
 
+def calculate_posture_score(frame):
+    image_height, image_width, _ = frame.shape
+    results = pose.process(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+
+    if not results.pose_landmarks:
+        return None, None
+
+    landmarks = results.pose_landmarks.landmark
+
+    # Use only body landmarks
+    left_shoulder = landmarks[mp_pose.PoseLandmark.LEFT_SHOULDER.value]
+    right_shoulder = landmarks[mp_pose.PoseLandmark.RIGHT_SHOULDER.value]
+    left_hip = landmarks[mp_pose.PoseLandmark.LEFT_HIP.value]
+    right_hip = landmarks[mp_pose.PoseLandmark.RIGHT_HIP.value]
+    left_knee = landmarks[mp_pose.PoseLandmark.LEFT_KNEE.value]
+    right_knee = landmarks[mp_pose.PoseLandmark.RIGHT_KNEE.value]
+
+    # Calculate angles
+    shoulder_angle = abs(math.degrees(math.atan2(right_shoulder.y - left_shoulder.y, right_shoulder.x - left_shoulder.x)))
+    hip_angle = abs(math.degrees(math.atan2(right_hip.y - left_hip.y, right_hip.x - left_hip.x)))
+    knee_angle = abs(math.degrees(math.atan2(right_knee.y - left_knee.y, right_knee.x - left_knee.x)))
+
+    # Calculate vertical alignment
+    shoulder_hip_alignment = abs((left_shoulder.y + right_shoulder.y) / 2 - (left_hip.y + right_hip.y) / 2)
+    hip_knee_alignment = abs((left_hip.y + right_hip.y) / 2 - (left_knee.y + right_knee.y) / 2)
+    # Add head landmarks
+    nose = landmarks[mp_pose.PoseLandmark.NOSE.value]
+    left_ear = landmarks[mp_pose.PoseLandmark.LEFT_EAR.value]
+    right_ear = landmarks[mp_pose.PoseLandmark.RIGHT_EAR.value]
+    # Calculate head tilt
+    head_tilt = abs(math.degrees(math.atan2(right_ear.y - left_ear.y, right_ear.x - left_ear.x)))
+    # Calculate head position relative to shoulders
+    head_position = abs((nose.y - (left_shoulder.y + right_shoulder.y) / 2) /
+                        ((left_shoulder.y + right_shoulder.y) / 2 - (left_hip.y + right_hip.y) / 2))
+
+    # Combine metrics into a single posture score (you may need to adjust these weights)
+    posture_score = (
+        (1 - abs(shoulder_angle - hip_angle) / 90) * 0.3 +
+        (1 - abs(hip_angle - knee_angle) / 90) * 0.2 +
+        (1 - shoulder_hip_alignment) * 0.1 +
+        (1 - hip_knee_alignment) * 0.1 +
+        (1 - abs(head_tilt - 90) / 90) * 0.15 +
+        (1 - head_position) * 0.15
+    )
+
+    return posture_score, results.pose_landmarks
+
 def extract_frames(video_path, output_folder, desired_fps, progress_callback=None):
     os.makedirs(output_folder, exist_ok=True)
     clip = VideoFileClip(video_path)
@@ -130,9 +176,12 @@ def is_frontal_face(landmarks, threshold=40):
     angle_degrees = math.degrees(angle)
     return abs(180 - angle_degrees) < threshold
 
+
 def process_frames(frames_folder, aligned_faces_folder, frame_count, progress, batch_size):
     embeddings_by_frame = {}
     emotions_by_frame = {}
+    posture_scores_by_frame = {}
+    posture_landmarks_by_frame = {}
     aligned_face_paths = []
     frame_files = sorted([f for f in os.listdir(frames_folder) if f.endswith('.jpg')])
 
@@ -154,6 +203,12 @@ def process_frames(frames_folder, aligned_faces_folder, frame_count, progress, b
 
             for j, (frame, frame_num, boxes, probs) in enumerate(
                     zip(batch_frames, batch_nums, batch_boxes, batch_probs)):
+
+                # Calculate posture score for the full frame
+                posture_score, posture_landmarks = calculate_posture_score(frame)
+                posture_scores_by_frame[frame_num] = posture_score
+                posture_landmarks_by_frame[frame_num] = posture_landmarks
+
                 if boxes is not None and len(boxes) > 0 and probs[0] >= 0.99:
                     x1, y1, x2, y2 = [int(b) for b in boxes[0]]
                     face = frame[y1:y2, x1:x2]
@@ -166,14 +221,14 @@ def process_frames(frames_folder, aligned_faces_folder, frame_count, progress, b
                                 output_path = os.path.join(aligned_faces_folder, f"frame_{frame_num}_face.jpg")
                                 cv2.imwrite(output_path, aligned_face_resized)
                                 aligned_face_paths.append(output_path)
-                                embedding, emotion = get_face_embedding_and_emotion(aligned_face_resized)
+                                embedding = get_face_embedding(aligned_face_resized)
                                 embeddings_by_frame[frame_num] = embedding
-                                emotions_by_frame[frame_num] = emotion
 
         progress((i + len(batch_files)) / len(frame_files),
                  f"Processing frames {i + 1} to {min(i + len(batch_files), len(frame_files))} of {len(frame_files)}")
 
-    return embeddings_by_frame, emotions_by_frame, aligned_face_paths
+    return embeddings_by_frame, posture_scores_by_frame, posture_landmarks_by_frame, aligned_face_paths
+
 
 def cluster_faces(embeddings):
     if len(embeddings) < 2:
@@ -198,20 +253,19 @@ def organize_faces_by_person(embeddings_by_frame, clusters, aligned_faces_folder
         dst = os.path.join(person_folder, f"frame_{frame_num}_face.jpg")
         shutil.copy(src, dst)
 
-def save_person_data_to_csv(embeddings_by_frame, emotions_by_frame, clusters, desired_fps, original_fps, output_folder, video_duration):
-    emotions = ['angry', 'disgust', 'fear', 'sad', 'happy']
+def save_person_data_to_csv(embeddings_by_frame, clusters, desired_fps, original_fps, output_folder, video_duration):
     person_data = {}
 
-    for (frame_num, embedding), (_, emotion_dict), cluster in zip(embeddings_by_frame.items(), emotions_by_frame.items(), clusters):
+    for (frame_num, embedding), cluster in zip(embeddings_by_frame.items(), clusters):
         if cluster not in person_data:
             person_data[cluster] = []
-        person_data[cluster].append((frame_num, embedding, {e: emotion_dict[e] for e in emotions}))
+        person_data[cluster].append((frame_num, embedding))
 
     largest_cluster = max(person_data, key=lambda k: len(person_data[k]))
 
     data = person_data[largest_cluster]
     data.sort(key=lambda x: x[0])
-    frames, embeddings, emotions_data = zip(*data)
+    frames, embeddings = zip(*data)
 
     embeddings_array = np.array(embeddings)
     np.save(os.path.join(output_folder, 'face_embeddings.npy'), embeddings_array)
@@ -228,9 +282,6 @@ def save_person_data_to_csv(embeddings_by_frame, emotions_by_frame, clusters, de
     for i in range(len(embeddings[0])):
         df_data[f'Raw_Embedding_{i}'] = [embedding[i] for embedding in embeddings]
 
-    for emotion in emotions:
-        df_data[emotion] = [e[emotion] for e in emotions_data]
-
     df = pd.DataFrame(df_data)
 
     return df, largest_cluster
@@ -270,62 +321,53 @@ def determine_anomalies(mse_values, threshold):
     anomalies = mse_values > (mean + threshold * std)
     return anomalies
 
-def anomaly_detection(X_emotions, X_embeddings, epochs=200, batch_size=8, patience=3):
-    device = 'cuda' if torch.cuda.is_available() else 'cpu'
-
-    # Normalize emotions
-    scaler_emotions = MinMaxScaler()
-    X_emotions_scaled = scaler_emotions.fit_transform(X_emotions)
-
-    # Process emotions
-    X_emotions_scaled = torch.FloatTensor(X_emotions_scaled).to(device)
-    if X_emotions_scaled.dim() == 2:
-        X_emotions_scaled = X_emotions_scaled.unsqueeze(0)
-
-    model_emotions = Autoencoder(input_size=X_emotions_scaled.shape[2]).to(device)
-    criterion = nn.MSELoss()
-    optimizer_emotions = optim.Adam(model_emotions.parameters())
-
-    # Train emotions model
-    for epoch in range(epochs):
-        model_emotions.train()
-        optimizer_emotions.zero_grad()
-        output_emotions = model_emotions(X_emotions_scaled)
-        loss_emotions = criterion(output_emotions, X_emotions_scaled)
-        loss_emotions.backward()
-        optimizer_emotions.step()
+def anomaly_detection(X_embeddings, X_posture, epochs=200, batch_size=8, patience=3):
+    # Normalize posture
+    scaler_posture = MinMaxScaler()
+    X_posture_scaled = scaler_posture.fit_transform(X_posture.reshape(-1, 1))
 
     # Process facial embeddings
     X_embeddings = torch.FloatTensor(X_embeddings).to(device)
     if X_embeddings.dim() == 2:
         X_embeddings = X_embeddings.unsqueeze(0)
 
+    # Process posture
+    X_posture_scaled = torch.FloatTensor(X_posture_scaled).to(device)
+    if X_posture_scaled.dim() == 2:
+        X_posture_scaled = X_posture_scaled.unsqueeze(0)
+
     model_embeddings = Autoencoder(input_size=X_embeddings.shape[2]).to(device)
+    model_posture = Autoencoder(input_size=X_posture_scaled.shape[2]).to(device)
+
+    criterion = nn.MSELoss()
     optimizer_embeddings = optim.Adam(model_embeddings.parameters())
+    optimizer_posture = optim.Adam(model_posture.parameters())
 
-    # Train embeddings model
+    # Train models
     for epoch in range(epochs):
-        model_embeddings.train()
-        optimizer_embeddings.zero_grad()
-        output_embeddings = model_embeddings(X_embeddings)
-        loss_embeddings = criterion(output_embeddings, X_embeddings)
-        loss_embeddings.backward()
-        optimizer_embeddings.step()
-
-    # Compute MSE for emotions and embeddings
-    model_emotions.eval()
+        for model, optimizer, X in [(model_embeddings, optimizer_embeddings, X_embeddings),
+                                    (model_posture, optimizer_posture, X_posture_scaled)]:
+            model.train()
+            optimizer.zero_grad()
+            output = model(X)
+            loss = criterion(output, X)
+            loss.backward()
+            optimizer.step()
+
+    # Compute MSE for embeddings and posture
     model_embeddings.eval()
+    model_posture.eval()
     with torch.no_grad():
-        reconstructed_emotions = model_emotions(X_emotions_scaled).cpu().numpy()
         reconstructed_embeddings = model_embeddings(X_embeddings).cpu().numpy()
+        reconstructed_posture = model_posture(X_posture_scaled).cpu().numpy()
 
-        mse_emotions = np.mean(np.power(X_emotions_scaled.cpu().numpy() - reconstructed_emotions, 2), axis=2).squeeze()
         mse_embeddings = np.mean(np.power(X_embeddings.cpu().numpy() - reconstructed_embeddings, 2), axis=2).squeeze()
+        mse_posture = np.mean(np.power(X_posture_scaled.cpu().numpy() - reconstructed_posture, 2), axis=2).squeeze()
 
-    return mse_emotions, mse_embeddings
+    return mse_embeddings, mse_posture
 
-def plot_mse(df, mse_values, title, color='blue', time_threshold=3, anomaly_threshold=4):
-    plt.figure(figsize=(16, 8), dpi=500)
+def plot_mse(df, mse_values, title, color='navy', time_threshold=3, anomaly_threshold=4):
+    plt.figure(figsize=(16, 8), dpi=400)
     fig, ax = plt.subplots(figsize=(16, 8))
 
     if 'Seconds' not in df.columns:
@@ -347,12 +389,11 @@ def plot_mse(df, mse_values, title, color='blue', time_threshold=3, anomaly_thre
     median = np.median(mse_values)
 
     ax.scatter(df['Seconds'], mse_values, color=color, alpha=0.3, s=5)
-    ax.plot(df['Seconds'], mean, color=color, linewidth=2)
-    ax.fill_between(df['Seconds'], mean - std, mean + std, color=color, alpha=0.2)
+    ax.plot(df['Seconds'], mean, color=color, linewidth=0.5)
+    ax.fill_between(df['Seconds'], mean - std, mean + std, color=color, alpha=0.1)
 
     # Add median line
-    ax.axhline(y=median, color='black', linestyle='--', label='Baseline')
-    ax.text(ax.get_xlim()[1], median, 'Baseline', verticalalignment='center', horizontalalignment='left', color='black')
+    ax.axhline(y=median, color='black', linestyle='--', label='Median Baseline')
 
     # Add threshold line
     threshold = np.mean(mse_values) + anomaly_threshold * np.std(mse_values)
@@ -362,7 +403,7 @@ def plot_mse(df, mse_values, title, color='blue', time_threshold=3, anomaly_thre
     anomalies = determine_anomalies(mse_values, anomaly_threshold)
     anomaly_frames = df['Frame'].iloc[anomalies].tolist()
 
-    ax.scatter(df['Seconds'].iloc[anomalies], mse_values[anomalies], color='red', s=25, zorder=5)
+    ax.scatter(df['Seconds'].iloc[anomalies], mse_values[anomalies], color='red', s=20, zorder=5)
 
     anomaly_data = list(zip(df['Timecode'].iloc[anomalies],
                             df['Seconds'].iloc[anomalies],
@@ -384,7 +425,7 @@ def plot_mse(df, mse_values, title, color='blue', time_threshold=3, anomaly_thre
         start_sec = group[0][1]
         end_sec = group[-1][1]
         rect = Rectangle((start_sec, ax.get_ylim()[0]), end_sec - start_sec, ax.get_ylim()[1] - ax.get_ylim()[0],
-                         facecolor='red', alpha=0.3, zorder=1)
+                         facecolor='red', alpha=0.2, zorder=1)
         ax.add_patch(rect)
 
     for group in grouped_anomalies:
@@ -412,8 +453,8 @@ def plot_mse(df, mse_values, title, color='blue', time_threshold=3, anomaly_thre
     return fig, anomaly_frames
 
 def plot_mse_histogram(mse_values, title, anomaly_threshold, color='blue'):
-    plt.figure(figsize=(16, 8), dpi=500)
-    fig, ax = plt.subplots(figsize=(16, 8))
+    plt.figure(figsize=(16, 4), dpi=400)
+    fig, ax = plt.subplots(figsize=(16, 4))
 
     ax.hist(mse_values, bins=100, edgecolor='black', color=color, alpha=0.7)
     ax.set_xlabel('Mean Squared Error')
@@ -440,56 +481,152 @@ def plot_mse_histogram(mse_values, title, anomaly_threshold, color='blue'):
     return fig
 
 
-def plot_emotion(df, emotion, color, anomaly_threshold):
-    plt.figure(figsize=(16, 8), dpi=500)
+def plot_posture(df, posture_scores, color='blue', anomaly_threshold=4):
+    plt.figure(figsize=(16, 8), dpi=400)
     fig, ax = plt.subplots(figsize=(16, 8))
 
     df['Seconds'] = df['Timecode'].apply(
         lambda x: sum(float(t) * 60 ** i for i, t in enumerate(reversed(x.split(':')))))
 
-    mean = df[emotion].rolling(window=10).mean()
-    std = df[emotion].rolling(window=10).std()
-    median = df[emotion].median()
-
-    ax.scatter(df['Seconds'], df[emotion], color=color, alpha=0.3, s=5)
-    ax.plot(df['Seconds'], mean, color=color, linewidth=2)
-    ax.fill_between(df['Seconds'], mean - std, mean + std, color=color, alpha=0.2)
+    posture_data = [(frame, score) for frame, score in posture_scores.items() if score is not None]
+    posture_frames, posture_scores = zip(*posture_data)
 
-    # Add median line
-    ax.axhline(y=median, color='black', linestyle='--', label='Baseline')
-    ax.text(ax.get_xlim()[1], median, 'Baseline', verticalalignment='center', horizontalalignment='left', color='black')
+    # Create a new dataframe for posture data
+    posture_df = pd.DataFrame({'Frame': posture_frames, 'Score': posture_scores})
+    posture_df = posture_df.merge(df[['Frame', 'Seconds']], on='Frame', how='inner')
 
-    # Convert anomaly threshold to probability
-    probability_threshold = (anomaly_threshold - 1) / 6  # Convert 1-7 scale to 0-1 probability
+    ax.scatter(posture_df['Seconds'], posture_df['Score'], color=color, alpha=0.3, s=5)
+    mean = posture_df['Score'].rolling(window=10).mean()
+    ax.plot(posture_df['Seconds'], mean, color=color, linewidth=0.5)
 
-    # Add threshold line and detect anomalies
-    ax.axhline(y=probability_threshold, color='red', linestyle='--', label=f'Threshold: {probability_threshold:.2f}')
-    ax.text(ax.get_xlim()[1], probability_threshold, f'Threshold: {probability_threshold:.2f}',
-            verticalalignment='center', horizontalalignment='left', color='red')
+    ax.set_xlabel('Timecode')
+    ax.set_ylabel('Posture Score')
+    ax.set_title("Body Posture Over Time")
 
-    # Detect and highlight anomalies
-    anomalies = df[emotion] >= probability_threshold
-    ax.scatter(df['Seconds'][anomalies], df[emotion][anomalies], color='red', s=25, zorder=5)
+    ax.grid(True, linestyle='--', alpha=0.7)
 
     max_seconds = df['Seconds'].max()
-    num_ticks = 100
+    num_ticks = 80
     tick_locations = np.linspace(0, max_seconds, num_ticks)
     tick_labels = [seconds_to_timecode(int(s)) for s in tick_locations]
 
     ax.set_xticks(tick_locations)
     ax.set_xticklabels(tick_labels, rotation=90, ha='center', fontsize=6)
 
-    ax.set_xlabel('Timecode')
-    ax.set_ylabel('Emotion Probability')
-    ax.set_title(f"{emotion.capitalize()} Over Time")
-
-    ax.grid(True, linestyle='--', alpha=0.7)
-    ax.legend()
     plt.tight_layout()
     plt.close()
     return fig
 
-def get_all_face_samples(organized_faces_folder, output_folder, largest_cluster, max_samples=500):
+
+def plot_mse_heatmap(mse_values, title, df):
+    plt.figure(figsize=(20, 10), dpi=400)
+    fig, ax = plt.subplots(figsize=(20, 10))
+
+    # Reshape MSE values to 2D array for heatmap
+    mse_2d = mse_values.reshape(1, -1)
+
+    # Create heatmap
+    sns.heatmap(mse_2d, cmap='YlOrRd', cbar_kws={'label': 'MSE'}, ax=ax)
+
+    # Set x-axis ticks to timecodes
+    num_ticks = 60
+    tick_locations = np.linspace(0, len(mse_values) - 1, num_ticks).astype(int)
+    tick_labels = [df['Timecode'].iloc[i] for i in tick_locations]
+
+    ax.set_xticks(tick_locations)
+    ax.set_xticklabels(tick_labels, rotation=90, ha='center', va='top')  # Adjusted rotation and alignment
+
+    ax.set_title(title)
+
+    # Remove y-axis labels
+    ax.set_yticks([])
+
+    plt.tight_layout()  # Ensure all elements fit within the figure
+    plt.close()
+    return fig
+
+def draw_pose_landmarks(frame, landmarks):
+    annotated_frame = frame.copy()
+    # Include relevant landmarks for head position and body
+    body_landmarks = [
+        mp_pose.PoseLandmark.NOSE,
+        mp_pose.PoseLandmark.LEFT_SHOULDER,
+        mp_pose.PoseLandmark.RIGHT_SHOULDER,
+        mp_pose.PoseLandmark.LEFT_EAR,
+        mp_pose.PoseLandmark.RIGHT_EAR,
+        mp_pose.PoseLandmark.LEFT_ELBOW,
+        mp_pose.PoseLandmark.RIGHT_ELBOW,
+        mp_pose.PoseLandmark.LEFT_WRIST,
+        mp_pose.PoseLandmark.RIGHT_WRIST,
+        mp_pose.PoseLandmark.LEFT_HIP,
+        mp_pose.PoseLandmark.RIGHT_HIP,
+        mp_pose.PoseLandmark.LEFT_KNEE,
+        mp_pose.PoseLandmark.RIGHT_KNEE,
+        mp_pose.PoseLandmark.LEFT_ANKLE,
+        mp_pose.PoseLandmark.RIGHT_ANKLE
+    ]
+
+    # Connections for head position and body
+    body_connections = [
+        (mp_pose.PoseLandmark.LEFT_EAR, mp_pose.PoseLandmark.LEFT_SHOULDER),
+        (mp_pose.PoseLandmark.RIGHT_EAR, mp_pose.PoseLandmark.RIGHT_SHOULDER),
+        (mp_pose.PoseLandmark.NOSE, mp_pose.PoseLandmark.LEFT_SHOULDER),
+        (mp_pose.PoseLandmark.NOSE, mp_pose.PoseLandmark.RIGHT_SHOULDER),
+        (mp_pose.PoseLandmark.LEFT_SHOULDER, mp_pose.PoseLandmark.RIGHT_SHOULDER),
+        (mp_pose.PoseLandmark.LEFT_SHOULDER, mp_pose.PoseLandmark.LEFT_ELBOW),
+        (mp_pose.PoseLandmark.RIGHT_SHOULDER, mp_pose.PoseLandmark.RIGHT_ELBOW),
+        (mp_pose.PoseLandmark.LEFT_ELBOW, mp_pose.PoseLandmark.LEFT_WRIST),
+        (mp_pose.PoseLandmark.RIGHT_ELBOW, mp_pose.PoseLandmark.RIGHT_WRIST),
+        (mp_pose.PoseLandmark.LEFT_SHOULDER, mp_pose.PoseLandmark.LEFT_HIP),
+        (mp_pose.PoseLandmark.RIGHT_SHOULDER, mp_pose.PoseLandmark.RIGHT_HIP),
+        (mp_pose.PoseLandmark.LEFT_HIP, mp_pose.PoseLandmark.RIGHT_HIP),
+        (mp_pose.PoseLandmark.LEFT_HIP, mp_pose.PoseLandmark.LEFT_KNEE),
+        (mp_pose.PoseLandmark.RIGHT_HIP, mp_pose.PoseLandmark.RIGHT_KNEE),
+        (mp_pose.PoseLandmark.LEFT_KNEE, mp_pose.PoseLandmark.LEFT_ANKLE),
+        (mp_pose.PoseLandmark.RIGHT_KNEE, mp_pose.PoseLandmark.RIGHT_ANKLE)
+    ]
+
+    # Draw landmarks
+    for landmark in body_landmarks:
+        if landmark in landmarks.landmark:
+            lm = landmarks.landmark[landmark]
+            h, w, _ = annotated_frame.shape
+            cx, cy = int(lm.x * w), int(lm.y * h)
+            cv2.circle(annotated_frame, (cx, cy), 5, (245, 117, 66), -1)
+
+    # Draw connections
+    for connection in body_connections:
+        start_lm = landmarks.landmark[connection[0]]
+        end_lm = landmarks.landmark[connection[1]]
+        h, w, _ = annotated_frame.shape
+        start_point = (int(start_lm.x * w), int(start_lm.y * h))
+        end_point = (int(end_lm.x * w), int(end_lm.y * h))
+        cv2.line(annotated_frame, start_point, end_point, (245, 66, 230), 2)
+
+    # Highlight head tilt
+    left_ear = landmarks.landmark[mp_pose.PoseLandmark.LEFT_EAR]
+    right_ear = landmarks.landmark[mp_pose.PoseLandmark.RIGHT_EAR]
+    nose = landmarks.landmark[mp_pose.PoseLandmark.NOSE]
+
+    h, w, _ = annotated_frame.shape
+    left_ear_point = (int(left_ear.x * w), int(left_ear.y * h))
+    right_ear_point = (int(right_ear.x * w), int(right_ear.y * h))
+    nose_point = (int(nose.x * w), int(nose.y * h))
+
+    # Draw a line between ears to show head tilt
+    cv2.line(annotated_frame, left_ear_point, right_ear_point, (0, 255, 0), 2)
+
+    # Draw a line from nose to the midpoint between shoulders to show head forward/backward tilt
+    left_shoulder = landmarks.landmark[mp_pose.PoseLandmark.LEFT_SHOULDER]
+    right_shoulder = landmarks.landmark[mp_pose.PoseLandmark.RIGHT_SHOULDER]
+    shoulder_mid_x = (left_shoulder.x + right_shoulder.x) / 2
+    shoulder_mid_y = (left_shoulder.y + right_shoulder.y) / 2
+    shoulder_mid_point = (int(shoulder_mid_x * w), int(shoulder_mid_y * h))
+    cv2.line(annotated_frame, nose_point, shoulder_mid_point, (0, 255, 0), 2)
+
+    return annotated_frame
+
+def get_all_face_samples(organized_faces_folder, output_folder, largest_cluster, max_samples=400):
     face_samples = {"most_frequent": [], "others": []}
     for cluster_folder in sorted(os.listdir(organized_faces_folder)):
         if cluster_folder.startswith("person_"):
@@ -523,12 +660,18 @@ def get_all_face_samples(organized_faces_folder, output_folder, largest_cluster,
                                 break
     return face_samples
 
+
 def process_video(video_path, anomaly_threshold, desired_fps, progress=gr.Progress()):
     start_time = time.time()
     output_folder = "output"
     os.makedirs(output_folder, exist_ok=True)
     batch_size = 16
 
+    GRAPH_COLORS = {
+        'facial_embeddings': 'navy',
+        'body_posture': 'purple'
+    }
+
     with tempfile.TemporaryDirectory() as temp_dir:
         aligned_faces_folder = os.path.join(temp_dir, 'aligned_faces')
         organized_faces_folder = os.path.join(temp_dir, 'organized_faces')
@@ -549,12 +692,13 @@ def process_video(video_path, anomaly_threshold, desired_fps, progress=gr.Progre
 
         progress(1, "Frame extraction complete")
         progress(0.3, "Processing frames")
-        embeddings_by_frame, emotions_by_frame, aligned_face_paths = process_frames(frames_folder, aligned_faces_folder,
-                                                                                    frame_count,
-                                                                                    progress, batch_size)
+        embeddings_by_frame, posture_scores_by_frame, posture_landmarks_by_frame, aligned_face_paths = process_frames(
+            frames_folder, aligned_faces_folder,
+            frame_count,
+            progress, batch_size)
 
         if not aligned_face_paths:
-            return ("No faces were extracted from the video.",) + (None,) * 10
+            raise ValueError("No faces were extracted from the video.")
 
         progress(0.6, "Clustering faces")
         embeddings = [embedding for _, embedding in embeddings_by_frame.items()]
@@ -565,7 +709,7 @@ def process_video(video_path, anomaly_threshold, desired_fps, progress=gr.Progre
         organize_faces_by_person(embeddings_by_frame, clusters, aligned_faces_folder, organized_faces_folder)
 
         progress(0.8, "Saving person data")
-        df, largest_cluster = save_person_data_to_csv(embeddings_by_frame, emotions_by_frame, clusters, desired_fps,
+        df, largest_cluster = save_person_data_to_csv(embeddings_by_frame, clusters, desired_fps,
                                                       original_fps, temp_dir, video_duration)
 
         # Add 'Seconds' column to df
@@ -576,107 +720,220 @@ def process_video(video_path, anomaly_threshold, desired_fps, progress=gr.Progre
         face_samples = get_all_face_samples(organized_faces_folder, output_folder, largest_cluster)
 
         progress(0.9, "Performing anomaly detection")
-        emotion_columns = ['angry', 'disgust', 'fear', 'sad', 'happy']
         embedding_columns = [col for col in df.columns if col.startswith('Raw_Embedding_')]
 
-        X_emotions = df[emotion_columns].values
         X_embeddings = df[embedding_columns].values
 
         try:
-            mse_emotions, mse_embeddings = anomaly_detection(X_emotions, X_embeddings, batch_size=batch_size)
+            X_posture = np.array([posture_scores_by_frame.get(frame, None) for frame in df['Frame']])
+            X_posture = X_posture[X_posture != None].reshape(-1, 1)  # Remove None values and reshape
+
+            # Ensure X_posture is not empty
+            if len(X_posture) == 0:
+                raise ValueError("No valid posture data found")
+
+            mse_embeddings, mse_posture = anomaly_detection(X_embeddings, X_posture, batch_size=batch_size)
 
             progress(0.95, "Generating plots")
-            mse_plot_embeddings, anomaly_frames_embeddings = plot_mse(df, mse_embeddings, "Facial Embeddings",
-                                                                      color='green',
+            mse_plot_embeddings, anomaly_frames_embeddings = plot_mse(df, mse_embeddings, "Facial Features",
+                                                                      color=GRAPH_COLORS['facial_embeddings'],
                                                                       anomaly_threshold=anomaly_threshold)
-            mse_histogram_embeddings = plot_mse_histogram(mse_embeddings, "MSE Distribution: Facial Embeddings",
-                                                          anomaly_threshold, color='green')
 
-            # Add emotion plots
-            emotion_plots = []
-            for emotion, color in zip(emotion_columns, ['purple', 'brown', 'green', 'orange', 'darkblue']):
-                emotion_plot = plot_emotion(df, emotion, color, anomaly_threshold)
-                emotion_plots.append(emotion_plot)
+            mse_histogram_embeddings = plot_mse_histogram(mse_embeddings, "MSE Distribution: Facial Features",
+                                                          anomaly_threshold, color=GRAPH_COLORS['facial_embeddings'])
 
-            mse_var_emotions = np.var(mse_emotions)
-            mse_var_embeddings = np.var(mse_embeddings)
+            mse_plot_posture, anomaly_frames_posture = plot_mse(df, mse_posture, "Body Posture",
+                                                                color=GRAPH_COLORS['body_posture'],
+                                                                anomaly_threshold=anomaly_threshold)
+
+            mse_histogram_posture = plot_mse_histogram(mse_posture, "MSE Distribution: Body Posture",
+                                                       anomaly_threshold, color=GRAPH_COLORS['body_posture'])
+
+            mse_heatmap_embeddings = plot_mse_heatmap(mse_embeddings, "Facial Features MSE Heatmap", df)
+            mse_heatmap_posture = plot_mse_heatmap(mse_posture, "Body Posture MSE Heatmap", df)
 
         except Exception as e:
             print(f"Error details: {str(e)}")
-            return (f"Error in anomaly detection: {str(e)}",) + (None,) * 15
+            import traceback
+            traceback.print_exc()
+            return (f"Error in video processing: {str(e)}",) + (None,) * 14
 
         progress(1.0, "Preparing results")
-        results = f"Number of persons/clusters detected: {num_clusters}\n\n"
-        results += f"Breakdown of persons/clusters:\n"
+        results = f"Number of persons detected: {num_clusters}\n\n"
+        results += "Breakdown:\n"
         for cluster_id in range(num_clusters):
-            results += f"Person/Cluster {cluster_id + 1}: {len([c for c in clusters if c == cluster_id])} frames\n"
+            face_count = len([c for c in clusters if c == cluster_id])
+            results += f"Person {cluster_id + 1}: {face_count} face frames\n"
 
         end_time = time.time()
         execution_time = end_time - start_time
 
-        # Load anomaly frames as images
-        anomaly_faces_embeddings = [
-            cv2.imread(os.path.join(aligned_faces_folder, f"frame_{frame}_face.jpg"))
-            for frame in anomaly_frames_embeddings
-            if os.path.exists(os.path.join(aligned_faces_folder, f"frame_{frame}_face.jpg"))
-        ]
-        anomaly_faces_embeddings = [cv2.cvtColor(face, cv2.COLOR_BGR2RGB) for face in anomaly_faces_embeddings if face is not None]
+        def add_timecode_to_image(image, timecode):
+            img_pil = Image.fromarray(image)
+            draw = ImageDraw.Draw(img_pil)
+            font = ImageFont.truetype("arial.ttf", 15)
+            draw.text((10, 10), timecode, (255, 0, 0), font=font)
+            return np.array(img_pil)
+
+        # In the process_video function, update the anomaly frame processing:
+        anomaly_faces_embeddings = []
+        for frame in anomaly_frames_embeddings:
+            face_path = os.path.join(aligned_faces_folder, f"frame_{frame}_face.jpg")
+            if os.path.exists(face_path):
+                face_img = cv2.imread(face_path)
+                if face_img is not None:
+                    face_img = cv2.cvtColor(face_img, cv2.COLOR_BGR2RGB)
+                    timecode = df[df['Frame'] == frame]['Timecode'].iloc[0]
+                    face_img_with_timecode = add_timecode_to_image(face_img, timecode)
+                    anomaly_faces_embeddings.append(face_img_with_timecode)
+
+        anomaly_frames_posture_images = []
+        for frame in anomaly_frames_posture:
+            frame_path = os.path.join(frames_folder, f"frame_{frame:04d}.jpg")
+            if os.path.exists(frame_path):
+                frame_img = cv2.imread(frame_path)
+                if frame_img is not None:
+                    frame_img = cv2.cvtColor(frame_img, cv2.COLOR_BGR2RGB)
+                    pose_results = pose.process(frame_img)
+                    if pose_results.pose_landmarks:
+                        frame_img = draw_pose_landmarks(frame_img, pose_results.pose_landmarks)
+                    timecode = df[df['Frame'] == frame]['Timecode'].iloc[0]
+                    frame_img_with_timecode = add_timecode_to_image(frame_img, timecode)
+                    anomaly_frames_posture_images.append(frame_img_with_timecode)
 
         return (
             execution_time,
             results,
             df,
             mse_embeddings,
-            mse_emotions,
+            mse_posture,
             mse_plot_embeddings,
             mse_histogram_embeddings,
-            *emotion_plots,
+            mse_plot_posture,
+            mse_histogram_posture,
+            mse_heatmap_embeddings,
+            mse_heatmap_posture,
             face_samples["most_frequent"],
             face_samples["others"],
             anomaly_faces_embeddings,
-            aligned_faces_folder
+            anomaly_frames_posture_images,
+            aligned_faces_folder,
+            frames_folder
         )
 
+
 with gr.Blocks() as iface:
-    gr.Markdown("# Facial Expressions Anomaly Detection")
+    gr.Markdown("""
+    # Facial Expression and Body Language Anomaly Detection
+
+    This application analyzes videos to detect anomalies in facial features and body language. 
+    It processes the video frames to extract facial embeddings and body posture, 
+    then uses machine learning techniques to identify unusual patterns or deviations from the norm.
+
+    For more information, visit: [https://github.com/reab5555/Facial-Expression-Anomaly-Detection](https://github.com/reab5555/Facial-Expression-Anomaly-Detection)
+    """)
 
     with gr.Row():
         video_input = gr.Video()
-        anomaly_threshold = gr.Slider(minimum=1, maximum=7, step=0.1, value=4.5, label="Anomaly Detection Threshold")
-        fps_slider = gr.Slider(minimum=10, maximum=20, step=5, value=20, label="Frames Per Second")
 
+    anomaly_threshold = gr.Slider(minimum=1, maximum=5, step=0.1, value=3.5, label="Anomaly Detection Threshold")
     process_btn = gr.Button("Process Video")
-
+    progress_bar = gr.Progress()
     execution_time = gr.Number(label="Execution Time (seconds)")
-    results_text = gr.Textbox(label="Anomaly Detection Results")
 
-    anomaly_frames_embeddings = gr.Gallery(label="Anomaly Frames (Facial Embeddings)", columns=6, rows=2, height="auto")
+    with gr.Group(visible=False) as results_group:
+        results_text = gr.TextArea(label="Anomaly Detection Results", lines=6)
 
-    mse_embeddings_plot = gr.Plot(label="MSE: Facial Embeddings")
-    mse_embeddings_hist = gr.Plot(label="MSE Distribution: Facial Embeddings")
+        with gr.Tab("Facial Features"):
+            mse_features_plot = gr.Plot(label="MSE: Facial Features")
+            mse_features_hist = gr.Plot(label="MSE Distribution: Facial Features")
+            mse_features_heatmap = gr.Plot(label="MSE Heatmap: Facial Features")
+            anomaly_frames_features = gr.Gallery(label="Anomaly Frames (Facial Features)", columns=6, rows=2, height="auto")
 
-    # Add emotion plots
-    emotion_plots = [gr.Plot(label=f"{emotion.capitalize()} Over Time") for emotion in ['angry', 'disgust', 'fear', 'sad', 'happy']]
+        with gr.Tab("Body Posture"):
+            mse_posture_plot = gr.Plot(label="MSE: Body Posture")
+            mse_posture_hist = gr.Plot(label="MSE Distribution: Body Posture")
+            mse_posture_heatmap = gr.Plot(label="MSE Heatmap: Body Posture")
+            anomaly_frames_posture = gr.Gallery(label="Anomaly Frames (Body Posture)", columns=6, rows=2, height="auto")
 
-    face_samples_most_frequent = gr.Gallery(label="Most Frequent Person Samples (Target)", columns=6, rows=2, height="auto")
-    face_samples_others = gr.Gallery(label="Other Persons Samples", columns=6, rows=1, height="auto")
+        with gr.Tab("Face Samples"):
+            face_samples_most_frequent = gr.Gallery(label="Most Frequent Person Samples (Target)", columns=6, rows=2, height="auto")
+            face_samples_others = gr.Gallery(label="Other Persons Samples", columns=6, rows=1, height="auto")
 
     # Hidden components to store intermediate results
     df_store = gr.State()
-    mse_emotions_store = gr.State()
-    mse_embeddings_store = gr.State()
+    mse_features_store = gr.State()
+    mse_posture_store = gr.State()
     aligned_faces_folder_store = gr.State()
+    frames_folder_store = gr.State()
+    mse_heatmap_embeddings_store = gr.State()
+    mse_heatmap_posture_store = gr.State()
+
+    def process_and_show_completion(video_input_path, anomaly_threshold_input):
+        try:
+            print("Starting video processing...")
+            results = process_video(video_input_path, anomaly_threshold_input, FIXED_FPS, progress=progress_bar)
+            print("Video processing completed.")
+
+            if isinstance(results[0], str) and results[0].startswith("Error"):
+                print(f"Error occurred: {results[0]}")
+                return [results[0]] + [None] * 18  # Update this line to match the number of outputs
+
+            exec_time, results_summary, df, mse_embeddings, mse_posture, \
+                mse_plot_embeddings, mse_histogram_embeddings, \
+                mse_plot_posture, mse_histogram_posture, \
+                mse_heatmap_embeddings, mse_heatmap_posture, \
+                face_samples_frequent, face_samples_other, \
+                anomaly_faces_embeddings, anomaly_frames_posture_images, \
+                aligned_faces_folder, frames_folder = results
+
+            # Convert numpy arrays to PIL Images for the galleries
+            anomaly_faces_embeddings_pil = [Image.fromarray(face) for face in anomaly_faces_embeddings]
+            anomaly_frames_posture_pil = [Image.fromarray(frame) for frame in anomaly_frames_posture_images]
+
+            # Ensure face samples are in the correct format for Gradio
+            face_samples_frequent = [Image.open(path) for path in face_samples_frequent]
+            face_samples_other = [Image.open(path) for path in face_samples_other]
+
+            output = [
+                exec_time, results_summary,
+                df, mse_embeddings, mse_posture,
+                mse_plot_embeddings, mse_plot_posture,
+                mse_histogram_embeddings, mse_histogram_posture,
+                mse_heatmap_embeddings, mse_heatmap_posture,
+                anomaly_faces_embeddings_pil, anomaly_frames_posture_pil,
+                face_samples_frequent, face_samples_other,
+                aligned_faces_folder, frames_folder,
+                mse_embeddings, mse_posture
+            ]
+
+            return output
+
+        except Exception as e:
+            error_message = f"An error occurred: {str(e)}"
+            print(error_message)
+            import traceback
+            traceback.print_exc()
+            return [error_message] + [None] * 18
 
     process_btn.click(
-        process_video,
-        inputs=[video_input, anomaly_threshold, fps_slider],
+        process_and_show_completion,
+        inputs=[video_input, anomaly_threshold],
         outputs=[
-            execution_time, results_text, df_store, mse_embeddings_store, mse_emotions_store,
-            mse_embeddings_plot, mse_embeddings_hist,
-            *emotion_plots,
-            face_samples_most_frequent, face_samples_others, anomaly_frames_embeddings,
-            aligned_faces_folder_store
+            execution_time, results_text, df_store,
+            mse_features_store, mse_posture_store,
+            mse_features_plot, mse_posture_plot,
+            mse_features_hist, mse_posture_hist,
+            mse_features_heatmap, mse_posture_heatmap,
+            anomaly_frames_features, anomaly_frames_posture,
+            face_samples_most_frequent, face_samples_others,
+            aligned_faces_folder_store, frames_folder_store,
+            mse_heatmap_embeddings_store, mse_heatmap_posture_store
         ]
+    ).then(
+        lambda: gr.Group(visible=True),
+        inputs=None,
+        outputs=[results_group]
     )
 
 if __name__ == "__main__":