Upload 5 files

README.md

@@ -1,13 +1,76 @@
----
-title: Textclassification Leaderboard
-emoji: 👀
-colorFrom: green
-colorTo: purple
-sdk: gradio
-sdk_version: 5.23.0
-app_file: app.py
-pinned: false
-short_description: textclassification-leaderboard
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Text Classification Benchmark Leaderboard
+
+This project provides a **leaderboard** for evaluating **Text Classification** models. Users can upload their model predictions in a CSV format, compare performance metrics against ground truth datasets, and track submissions over time.
+
+## Features
+- **Dataset Selection:** Users can choose a dataset from predefined test sets.
+- **Submission Upload:** Supports CSV files with `file_name` and `label` columns.
+- **Automated Evaluation:** Calculates **Accuracy, Precision, Recall, and F1-score**.
+- **Leaderboard Tracking:** Stores and displays past experiments.
+- **Gradio Interface:** Simple and interactive web interface.
+
+## Requirements
+Ensure you have the following installed before running the project:
+
+```bash
+pip install gradio pandas sqlalchemy scikit-learn
+```
+
+## Setup & Usage
+
+1. **Clone the repository:**
+   ```bash
+   git clone https://github.com/nlp4bia-bsc/text-classification-leaderboard.git
+   cd text-classification-leaderboard
+   ```
+
+2. **Run the application:**
+   ```bash
+   python app.py
+   ```
+
+3. **Access the interface:**
+   The application runs locally. Open your browser and go to:
+   ```
+   http://127.0.0.1:7860/
+   ```
+
+## Submission Format
+Your submission file must be a **CSV** containing the following columns:
+
+| file_name | label  |
+|-----------|--------|
+| doc1.txt  | spam   |
+| doc2.txt  | ham    |
+| doc3.txt  | spam   |
+
+### Evaluation Metrics
+The system calculates:
+- **Accuracy**
+- **Precision (weighted)**
+- **Recall (weighted)**
+- **F1-score (weighted)**
+
+## Directory Structure
+```
+text-classification-leaderboard/
+│── testsets/               # Folder containing test datasets
+│── submissions.db          # SQLite database for storing results
+│── app.py                  # Main application script
+│── README.md               # Project documentation
+```
+
+## Future Improvements
+- Add support for multi-label classification.
+- Expand dataset compatibility with more formats.
+
+## License
+This project is licensed under the **MIT License**. Feel free to contribute and enhance it!
+
+## Contributing
+Pull requests are welcome! If you have suggestions or find issues, please open an issue on the repository.
+
+---
+**Author:** Wesam Alnabki
+**GitHub:** [wesamalnabki](https://github.com/wesamalnabki)
+

app.py

@@ -0,0 +1,149 @@
+import os
+import gradio as gr
+from sqlalchemy import create_engine, Column, Integer, String, Float, DateTime
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+from datetime import datetime
+import pandas as pd
+from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
+
+testsets_root_path = "./testsets/"
+
+# Function to load the dataset
+def load_testsets(testsets_root_path: str) -> dict:
+    datasets_dict = {}
+    for ds in os.listdir(testsets_root_path):
+        if ds.endswith(".csv"):  # Ensure only CSV files are processed
+            csv_path = os.path.join(testsets_root_path, ds)
+            df = pd.read_csv(csv_path)
+            datasets_dict[ds.replace(".csv", "")] = df
+    return datasets_dict
+
+# Database setup
+Base = declarative_base()
+
+class Submission(Base):
+    __tablename__ = 'submissions'
+    id = Column(Integer, primary_key=True)
+    dataset_name = Column(String)
+    submission_name = Column(String)
+    model_link = Column(String)
+    person_name = Column(String)
+    accuracy = Column(Float)
+    precision = Column(Float)
+    recall = Column(Float)
+    f1 = Column(Float)
+    submission_date = Column(DateTime, default=datetime.utcnow)
+
+engine = create_engine('sqlite:///submissions.db')
+Base.metadata.create_all(engine)
+Session = sessionmaker(bind=engine)
+session = Session()
+
+# Function to fetch previous submissions for a selected dataset
+def get_existing_submissions(dataset_name):
+    existing_submissions = session.query(Submission).filter_by(dataset_name=dataset_name).order_by(
+        Submission.submission_date.desc()).all()
+
+    submissions_list = [{
+        "Submission Name": sub.submission_name,
+        "Model Link": sub.model_link,
+        "Person Name": sub.person_name,
+        "Accuracy": sub.accuracy,
+        "Precision": sub.precision,
+        "Recall": sub.recall,
+        "F1": sub.f1,
+        "Submission Date": sub.submission_date.strftime("%Y-%m-%d %H:%M:%S")
+    } for sub in existing_submissions]
+
+    return pd.DataFrame(submissions_list) if submissions_list else pd.DataFrame(columns=[
+        "Submission Name", "Model Link", "Person Name", "Accuracy", "Precision", "Recall", "F1", "Submission Date"
+    ])
+
+# Evaluation function for text classification
+def calculate_metrics(gs, pred):
+    y_true = gs['label']
+    y_pred = pred['label']
+    try:
+        accuracy = accuracy_score(y_true, y_pred)
+        precision = precision_score(y_true, y_pred, average='weighted')
+        recall = recall_score(y_true, y_pred, average='weighted')
+        f1 = f1_score(y_true, y_pred, average='weighted')
+        return accuracy, precision, recall, f1
+    except:
+        return None, None, None, None
+
+def benchmark_interface(dataset_name, submission_file, submission_name, model_link, person_name):
+    if not all([dataset_name, submission_file, submission_name, model_link, person_name]):
+        return {"error": "All fields are required."}, pd.DataFrame()
+
+    dataset_dict = load_testsets(testsets_root_path)
+    df_gs = dataset_dict.get(dataset_name)
+    if df_gs is None:
+        return {"error": "Dataset not found."}, pd.DataFrame()
+
+    # Parse the uploaded submission CSV
+    submission_df = pd.read_csv(submission_file.name)
+
+    # Ensure the columns are present
+    if not all(col in submission_df.columns for col in ['file_name', 'label']):
+        return {"error": "Submission file must contain 'file_name' and 'label' columns."}, pd.DataFrame()
+
+    # Calculate metrics
+    accuracy, precision, recall, f1 = calculate_metrics(gs=df_gs, pred=submission_df)
+    metrics = {'Accuracy': accuracy, 'Precision': precision, 'Recall': recall, 'F1': f1}
+    if f1 is not None:
+        # Save submission to the database
+        new_submission = Submission(
+            dataset_name=dataset_name,
+            submission_name=submission_name,
+            model_link=model_link,
+            person_name=person_name,
+            accuracy=accuracy,
+            precision=precision,
+            recall=recall,
+            f1=f1
+        )
+        session.add(new_submission)
+        session.commit()
+
+    # Fetch updated submissions
+    submissions_df = get_existing_submissions(dataset_name)
+    return metrics, submissions_df
+
+
+def create_gradio_app():
+    dataset_dict = load_testsets(testsets_root_path)
+    dataset_names = list(dataset_dict.keys())
+
+    with gr.Blocks() as demo:
+        gr.Markdown("## Benchmarking Leaderboard for Text Classification")
+        dataset_radio = gr.Radio(choices=dataset_names, label="Select Dataset")
+        submission_file = gr.File(label="Upload Submission CSV")
+        submission_name = gr.Textbox(label="Submission Name")
+        model_link = gr.Textbox(label="Model Link on HuggingFace")
+        person_name = gr.Textbox(label="Person Name")
+        submit_button = gr.Button("Submit")
+        metrics_output = gr.JSON(label="Evaluation Metrics")
+        existing_submissions_output = gr.Dataframe(label="Existing Submissions")
+
+        # When a dataset is selected, fetch previous submissions
+        dataset_radio.change(
+            fn=get_existing_submissions,
+            inputs=[dataset_radio],
+            outputs=[existing_submissions_output]
+        )
+
+        submit_button.click(
+            fn=benchmark_interface,
+            inputs=[dataset_radio, submission_file, submission_name, model_link, person_name],
+            outputs=[metrics_output, existing_submissions_output]
+        )
+    return demo
+
+def main():
+    app = create_gradio_app()
+    app.launch()
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,3 @@
+sqlalchemy
+pandas
+scikit-learn