Update tasks/text.py

tasks/text.py

@@ -2,91 +2,87 @@ from fastapi import APIRouter
 from datetime import datetime
 from datasets import load_dataset
 from sklearn.metrics import accuracy_score
-import random
-
-from .utils.evaluation import TextEvaluationRequest
-from .utils.emissions import tracker, clean_emissions_data, get_space_info
+from sentence_transformers import SentenceTransformer
+import numpy as np
 
 router = APIRouter()
 
-DESCRIPTION = "Random Baseline"
+DESCRIPTION = "Class embeddings with cosine similarity using batching and thresholding"
 ROUTE = "/text"
 
-@router.post(ROUTE, tags=["Text Task"], 
-             description=DESCRIPTION)
-async def evaluate_text(request: TextEvaluationRequest):
+# Load fine-tuned model
+model_name = "pedro-thenewsroom/climate-misinfo-embed"
+embedding_model = SentenceTransformer(model_name)
+
+# Define label mapping
+LABEL_MAPPING = {
+    "0_not_relevant": 0,
+    "1_not_happening": 1,
+    "2_not_human": 2,
+    "3_not_bad": 3,
+    "4_solutions_harmful_unnecessary": 4,
+    "5_science_unreliable": 5,
+    "6_proponents_biased": 6,
+    "7_fossil_fuels_needed": 7,
+}
+
+# Class descriptions for embedding comparison
+class_labels = [
+    "Despite the alarmists' claims of global warming, temperatures have remained steady or even dropped in many areas, proving that climate change is nothing more than natural variability.",
+    "The Earth's climate has always changed due to natural cycles and external factors, and the role of human activity or CO2 emissions in driving these changes is negligible or unsupported by evidence.",
+    "Contrary to the alarmist rhetoric, rising CO2 levels and modest warming are fostering a greener planet, boosting crop yields, and enhancing global prosperity while posing no significant threat to human or environmental health.",
+    "Global climate policies are costly, ineffective, and fail to address the unchecked emissions from developing nations, rendering efforts by industrialized countries futile and economically damaging.",
+    "The so-called consensus on climate change relies on flawed models, manipulated data, and a refusal to address legitimate scientific uncertainties, all to serve a predetermined political narrative.",
+    "Climate change is nothing more than a fabricated agenda pushed by corrupt elites, politicians, and scientists to control the masses, gain wealth, and suppress freedom.",
+    "Fossil fuels have powered centuries of progress, lifted billions out of poverty, and remain the backbone of global energy, while alternatives, though promising, cannot yet match their scale, reliability, or affordability.",
+]
+
+# Precompute normalized class embeddings
+class_embeddings = embedding_model.encode(class_labels, batch_size=8, convert_to_numpy=True, normalize_embeddings=True)
+
+
+@router.post(ROUTE, tags=["Text Task"], description=DESCRIPTION)
+async def evaluate_text(request: dict):
     """
-    Evaluate text classification for climate disinformation detection.
-    
-    Current Model: Random Baseline
-    - Makes random predictions from the label space (0-7)
-    - Used as a baseline for comparison
+    Evaluate text classification using precomputed embeddings and cosine similarity.
     """
-    # Get space info
-    username, space_url = get_space_info()
-
-    # Define the label mapping
-    LABEL_MAPPING = {
-        "0_not_relevant": 0,
-        "1_not_happening": 1,
-        "2_not_human": 2,
-        "3_not_bad": 3,
-        "4_solutions_harmful_unnecessary": 4,
-        "5_science_unreliable": 5,
-        "6_proponents_biased": 6,
-        "7_fossil_fuels_needed": 7
-    }
+    # Load dataset
+    dataset = load_dataset("QuotaClimat/frugalaichallenge-text-train")
+    df_train = dataset["train"].to_pandas()
+    df_test = dataset["test"].to_pandas()
+    df = pd.concat([df_train, df_test], ignore_index=True)
+
+    # Extract quotes and their true labels
+    quotes = df["quote"].tolist()
+    true_labels = df["label"].apply(lambda x: int(x.split("_")[0]) if isinstance(x, str) else 0).tolist()
+
+    # Encode dataset quotes in batches
+    batch_size = 32
+    quote_embeddings = embedding_model.encode(quotes, batch_size=batch_size, convert_to_numpy=True, normalize_embeddings=True)
+
+    # Compute cosine similarity using matrix multiplication (efficient)
+    cosine_similarities = np.dot(quote_embeddings, class_embeddings.T)
+
+    # Get the best match for each sentence
+    best_indices = np.argmax(cosine_similarities, axis=1)
+    best_similarities = np.max(cosine_similarities, axis=1)
+
+    # Apply threshold (0.9) for classification
+    predicted_labels = [best_idx if best_sim > 0.9 else 0 for best_idx, best_sim in zip(best_indices, best_similarities)]
 
-    # Load and prepare the dataset
-    dataset = load_dataset(request.dataset_name)
-
-    # Convert string labels to integers
-    dataset = dataset.map(lambda x: {"label": LABEL_MAPPING[x["label"]]})
-
-    # Split dataset
-    train_test = dataset["train"]
-    test_dataset = dataset["test"]
-    
-    # Start tracking emissions
-    tracker.start()
-    tracker.start_task("inference")
-
-    #--------------------------------------------------------------------------------------------
-    # YOUR MODEL INFERENCE CODE HERE
-    # Update the code below to replace the random baseline by your model inference within the inference pass where the energy consumption and emissions are tracked.
-    #--------------------------------------------------------------------------------------------   
-    
-    # Make random predictions (placeholder for actual model inference)
-    true_labels = test_dataset["label"]
-    predictions = [random.randint(0, 7) for _ in range(len(true_labels))]
-
-    #--------------------------------------------------------------------------------------------
-    # YOUR MODEL INFERENCE STOPS HERE
-    #--------------------------------------------------------------------------------------------   
-
-    
-    # Stop tracking emissions
-    emissions_data = tracker.stop_task()
-    
     # Calculate accuracy
-    accuracy = accuracy_score(true_labels, predictions)
-    
+    accuracy = accuracy_score(true_labels, predicted_labels)
+
     # Prepare results dictionary
     results = {
-        "username": username,
-        "space_url": space_url,
         "submission_timestamp": datetime.now().isoformat(),
         "model_description": DESCRIPTION,
         "accuracy": float(accuracy),
-        "energy_consumed_wh": emissions_data.energy_consumed * 1000,
-        "emissions_gco2eq": emissions_data.emissions * 1000,
-        "emissions_data": clean_emissions_data(emissions_data),
         "api_route": ROUTE,
         "dataset_config": {
-            "dataset_name": request.dataset_name,
-            "test_size": request.test_size,
-            "test_seed": request.test_seed
-        }
+            "dataset_name": "QuotaClimat/frugalaichallenge-text-train",
+            "total_samples": len(df),
+        },
     }
-    
-    return results 
+
+    return results