from fastapi import APIRouter from datetime import datetime from datasets import load_dataset from sklearn.metrics import accuracy_score import random from .utils.evaluation import TextEvaluationRequest from .utils.emissions import tracker, clean_emissions_data, get_space_info #packages needed for inference from sentence_transformers import SentenceTransformer from xgboost import XGBClassifier import pickle import torch import os router = APIRouter() DESCRIPTION = "Embedding + Neural Network" ROUTE = "/text" @router.post(ROUTE, tags=["Text Task"], description=DESCRIPTION) async def evaluate_text(request: TextEvaluationRequest): """ Evaluate text classification for climate disinformation detection. Current Model: Random Baseline - Makes random predictions from the label space (0-7) - Used as a baseline for comparison """ # Get space info username, space_url = get_space_info() # Define the label mapping LABEL_MAPPING = { "0_not_relevant": 0, "1_not_happening": 1, "2_not_human": 2, "3_not_bad": 3, "4_solutions_harmful_unnecessary": 4, "5_science_unreliable": 5, "6_proponents_biased": 6, "7_fossil_fuels_needed": 7 } # Load and prepare the dataset dataset = load_dataset(request.dataset_name) # Convert string labels to integers dataset = dataset.map(lambda x: {"label": LABEL_MAPPING[x["label"]]}) # Split dataset train_test = dataset["train"].train_test_split(test_size=request.test_size, seed=request.test_seed) test_dataset = train_test["test"] # Start tracking emissions tracker.start() tracker.start_task("inference") #-------------------------------------------------------------------------------------------- # YOUR MODEL INFERENCE CODE HERE # Set the device to MPS (if available) device = torch.device("mps" if torch.backends.mps.is_available() else "cpu") print(f"Using device: {device}") model_name = "sentence-transformers/paraphrase-multilingual-mpnet-base-v2" # You can use other Sentence Transformers models as needed sentence_model = SentenceTransformer(model_name) # Convert each sentence into a vector representation (embedding) embeddings = sentence_model.encode(test_dataset['quote'], convert_to_tensor=True) # Make random predictions (placeholder for actual model inference) true_labels = test_dataset["label"] """ from torch import nn, optim class SimpleNN2(nn.Module): def __init__(self, input_dim, output_dim): super(SimpleNN2, self).__init__() self.fc1 = nn.Linear(input_dim, 128) # Reduce hidden units self.fc2 = nn.Linear(128, 64) # Further reduce units self.fc3 = nn.Linear(64, output_dim) self.relu = nn.ReLU() self.dropout = nn.Dropout(0.3) # Add dropout self.batch_norm1 = nn.BatchNorm1d(128) self.batch_norm2 = nn.BatchNorm1d(64) def forward(self, x): x = self.relu(self.batch_norm1(self.fc1(x))) x = self.dropout(x) # Apply dropout x = self.relu(self.batch_norm2(self.fc2(x))) x = self.dropout(x) # Apply dropout x = self.fc3(x) # Output raw logits return x """ current_file_path = os.path.abspath(__file__) current_dir = os.path.dirname(current_file_path) # model_nn = torch.load(os.path.join(current_dir,"model_nn.pth"), map_location=device) model_nn = torch.jit.load(os.path.join(current_dir,"model_nn_scripted.pth"), map_location=device) # Set the model to evaluation mode model_nn.eval() # Make predictions with torch.no_grad(): outputs = model_nn(embeddings) _, predicted = torch.max(outputs, 1) # Get the class with the highest score # Decode the predictions back to original labels using label_encoder predictions = predicted.cpu().numpy() #-------------------------------------------------------------------------------------------- # YOUR MODEL INFERENCE STOPS HERE #-------------------------------------------------------------------------------------------- # Stop tracking emissions emissions_data = tracker.stop_task() # Calculate accuracy accuracy = accuracy_score(true_labels, predictions) # Prepare results dictionary results = { "username": username, "space_url": space_url, "submission_timestamp": datetime.now().isoformat(), "model_description": DESCRIPTION, "accuracy": float(accuracy), "energy_consumed_wh": emissions_data.energy_consumed * 1000, "emissions_gco2eq": emissions_data.emissions * 1000, "emissions_data": clean_emissions_data(emissions_data), "api_route": ROUTE, "dataset_config": { "dataset_name": request.dataset_name, "test_size": request.test_size, "test_seed": request.test_seed } } return results