Initial leaderboard setup

.ipynb_checkpoints/comparison-checkpoint.py

@@ -8,44 +8,62 @@ warnings.filterwarnings("ignore", category=UserWarning, message="y_pred contains
 sns.set_style("whitegrid")
 
 class ModelEvaluator:
-    def __init__(self, df_labels, df_predictions, model_name):
+    def __init__(self, df_labels, df_predictions, model_name, categories = ['main-event', 'location', 'zone', 'light-conditions', 'weather-conditions', 'vehicles-density']):
         """
         Initialize the evaluator with ground truth labels and model predictions.
         """
         self.df_labels = df_labels
         self.df_predictions = df_predictions
         self.model_name = model_name
+        self.categories = categories
         self.metrics_df = self.compute_metrics()
-
+        
     def merge_data(self):
         """Merge ground truth labels with predictions based on 'id'."""
         merged_df = pd.merge(self.df_labels, self.df_predictions, on='id', suffixes=('_true', '_pred'))
+        for category in list(set(self.categories) - set(['main-event'])):
+            valid_values = self.df_labels[f"{category}"].unique().astype(str)
+            merged_df = merged_df[merged_df[f"{category}_pred"].astype(str).isin(valid_values)]
+
         return merged_df
 
     def compute_metrics(self):
         """Compute precision, recall, F1-score, accuracy, and balanced accuracy for each class and category."""
         merged_df = self.merge_data()
-        categories = ['main-event', 'location', 'zone', 'light-conditions', 'weather-conditions', 'vehicles-density']
-
+        categories = self.categories
+    
         results = []
-
+    
         for category in categories:
-            y_true = merged_df[f"{category}_true"].astype(str)
-            y_pred = merged_df[f"{category}_pred"].astype(str)
-
-            labels = sorted(set(y_true) | set(y_pred))
-
-            class_precisions = precision_score(y_true, y_pred, labels=labels, average=None, zero_division=0)
-            class_recalls = recall_score(y_true, y_pred, labels=labels, average=None, zero_division=0)
-            class_f1 = f1_score(y_true, y_pred, labels=labels, average=None, zero_division=0)
-
-            overall_precision = precision_score(y_true, y_pred, average='macro', zero_division=0)
-            overall_recall = recall_score(y_true, y_pred, average='macro', zero_division=0)
-            overall_f1 = f1_score(y_true, y_pred, average='macro', zero_division=0)
+            true_col = f"{category}_true"
+            pred_col = f"{category}_pred"
+    
+            if true_col not in merged_df.columns or pred_col not in merged_df.columns:
+                print(f"Skipping {category} - missing columns")
+                continue
+    
+            y_true = merged_df[true_col].astype(str)
+            y_pred = merged_df[pred_col].astype(str)
+    
+            valid_labels = sorted(set(y_true) | set(y_pred))
+    
+            valid_labels = [label for label in valid_labels if (y_true == label).sum() > 0]
+    
+            if not valid_labels:
+                print(f"Skipping {category} - No valid labels found.")
+                continue
+    
+            class_precisions = precision_score(y_true, y_pred, labels=valid_labels, average=None, zero_division=0)
+            class_recalls = recall_score(y_true, y_pred, labels=valid_labels, average=None, zero_division=0)
+            class_f1 = f1_score(y_true, y_pred, labels=valid_labels, average=None, zero_division=0)
+    
+            overall_precision = precision_score(y_true, y_pred, labels=valid_labels, average='macro', zero_division=0)
+            overall_recall = recall_score(y_true, y_pred, labels=valid_labels, average='macro', zero_division=0)
+            overall_f1 = f1_score(y_true, y_pred, labels=valid_labels, average='macro', zero_division=0)
             overall_accuracy = accuracy_score(y_true, y_pred)
             overall_balanced_acc = balanced_accuracy_score(y_true, y_pred)
-
-            for i, label in enumerate(labels):
+    
+            for i, label in enumerate(valid_labels):
                 results.append({
                     "Model": self.model_name,
                     "Category": category,
@@ -57,7 +75,7 @@ class ModelEvaluator:
                     "Balanced Acc.": np.nan,
                     "Support": (y_true == label).sum()
                 })
-
+    
             results.append({
                 "Model": self.model_name,
                 "Category": category,
@@ -69,9 +87,9 @@ class ModelEvaluator:
                 "Balanced Acc.": overall_balanced_acc,
                 "Support": len(y_true)
             })
-
+    
         df_res = pd.DataFrame(results)
-        return df_res.loc[df_res['Support']>0].reset_index(drop=True)
+        return df_res.loc[df_res['Support'] > 0].reset_index(drop=True)
 
     def get_metrics_df(self):
         """Return the computed metrics DataFrame."""

comparison.py

@@ -8,44 +8,62 @@ warnings.filterwarnings("ignore", category=UserWarning, message="y_pred contains
 sns.set_style("whitegrid")
 
 class ModelEvaluator:
-    def __init__(self, df_labels, df_predictions, model_name):
+    def __init__(self, df_labels, df_predictions, model_name, categories = ['main-event', 'location', 'zone', 'light-conditions', 'weather-conditions', 'vehicles-density']):
         """
         Initialize the evaluator with ground truth labels and model predictions.
         """
         self.df_labels = df_labels
         self.df_predictions = df_predictions
         self.model_name = model_name
+        self.categories = categories
         self.metrics_df = self.compute_metrics()
-
+        
     def merge_data(self):
         """Merge ground truth labels with predictions based on 'id'."""
         merged_df = pd.merge(self.df_labels, self.df_predictions, on='id', suffixes=('_true', '_pred'))
+        for category in list(set(self.categories) - set(['main-event'])):
+            valid_values = self.df_labels[f"{category}"].unique().astype(str)
+            merged_df = merged_df[merged_df[f"{category}_pred"].astype(str).isin(valid_values)]
+
         return merged_df
 
     def compute_metrics(self):
         """Compute precision, recall, F1-score, accuracy, and balanced accuracy for each class and category."""
         merged_df = self.merge_data()
-        categories = ['main-event', 'location', 'zone', 'light-conditions', 'weather-conditions', 'vehicles-density']
-
+        categories = self.categories
+    
         results = []
-
+    
         for category in categories:
-            y_true = merged_df[f"{category}_true"].astype(str)
-            y_pred = merged_df[f"{category}_pred"].astype(str)
-
-            labels = sorted(set(y_true) | set(y_pred))
-
-            class_precisions = precision_score(y_true, y_pred, labels=labels, average=None, zero_division=0)
-            class_recalls = recall_score(y_true, y_pred, labels=labels, average=None, zero_division=0)
-            class_f1 = f1_score(y_true, y_pred, labels=labels, average=None, zero_division=0)
-
-            overall_precision = precision_score(y_true, y_pred, average='macro', zero_division=0)
-            overall_recall = recall_score(y_true, y_pred, average='macro', zero_division=0)
-            overall_f1 = f1_score(y_true, y_pred, average='macro', zero_division=0)
+            true_col = f"{category}_true"
+            pred_col = f"{category}_pred"
+    
+            if true_col not in merged_df.columns or pred_col not in merged_df.columns:
+                print(f"Skipping {category} - missing columns")
+                continue
+    
+            y_true = merged_df[true_col].astype(str)
+            y_pred = merged_df[pred_col].astype(str)
+    
+            valid_labels = sorted(set(y_true) | set(y_pred))
+    
+            valid_labels = [label for label in valid_labels if (y_true == label).sum() > 0]
+    
+            if not valid_labels:
+                print(f"Skipping {category} - No valid labels found.")
+                continue
+    
+            class_precisions = precision_score(y_true, y_pred, labels=valid_labels, average=None, zero_division=0)
+            class_recalls = recall_score(y_true, y_pred, labels=valid_labels, average=None, zero_division=0)
+            class_f1 = f1_score(y_true, y_pred, labels=valid_labels, average=None, zero_division=0)
+    
+            overall_precision = precision_score(y_true, y_pred, labels=valid_labels, average='macro', zero_division=0)
+            overall_recall = recall_score(y_true, y_pred, labels=valid_labels, average='macro', zero_division=0)
+            overall_f1 = f1_score(y_true, y_pred, labels=valid_labels, average='macro', zero_division=0)
             overall_accuracy = accuracy_score(y_true, y_pred)
             overall_balanced_acc = balanced_accuracy_score(y_true, y_pred)
-
-            for i, label in enumerate(labels):
+    
+            for i, label in enumerate(valid_labels):
                 results.append({
                     "Model": self.model_name,
                     "Category": category,
@@ -57,7 +75,7 @@ class ModelEvaluator:
                     "Balanced Acc.": np.nan,
                     "Support": (y_true == label).sum()
                 })
-
+    
             results.append({
                 "Model": self.model_name,
                 "Category": category,
@@ -69,9 +87,9 @@ class ModelEvaluator:
                 "Balanced Acc.": overall_balanced_acc,
                 "Support": len(y_true)
             })
-
+    
         df_res = pd.DataFrame(results)
-        return df_res.loc[df_res['Support']>0].reset_index(drop=True)
+        return df_res.loc[df_res['Support'] > 0].reset_index(drop=True)
 
     def get_metrics_df(self):
         """Return the computed metrics DataFrame."""