from fastapi import APIRouter from datetime import datetime from datasets import load_dataset from sklearn.metrics import accuracy_score from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification from transformers import Trainer, TrainingArguments import torch from .utils.evaluation import TextEvaluationRequest from .utils.emissions import tracker, clean_emissions_data, get_space_info router = APIRouter() DESCRIPTION = "DistilBERT Baseline" ROUTE = "/text" @router.post(ROUTE, tags=["Text Task"], description=DESCRIPTION) async def evaluate_text(request: TextEvaluationRequest): """ Evaluate text classification for climate disinformation detection. Current Model: DistilBERT - Fine-tunes and evaluates a DistilBERT model on the given dataset """ # Get space info username, space_url = get_space_info() # Define the label mapping LABEL_MAPPING = { "0_not_relevant": 0, "1_not_happening": 1, "2_not_human": 2, "3_not_bad": 3, "4_solutions_harmful_unnecessary": 4, "5_science_unreliable": 5, "6_proponents_biased": 6, "7_fossil_fuels_needed": 7 } # Load and prepare the dataset dataset = load_dataset(request.dataset_name) # Convert string labels to integers dataset = dataset.map(lambda x: {"label": LABEL_MAPPING[x["label"]]}) # Split dataset train_test = dataset["train"].train_test_split(test_size=request.test_size, seed=request.test_seed) train_dataset = train_test["train"] test_dataset = train_test["test"] # Tokenizer and model tokenizer = DistilBertTokenizerFast.from_pretrained("distilbert-base-uncased") model = DistilBertForSequenceClassification.from_pretrained("distilbert-base-uncased", num_labels=8) # Tokenize datasets def preprocess(examples): return tokenizer(examples["text"], truncation=True, padding=True, max_length=512) train_dataset = train_dataset.map(preprocess, batched=True) test_dataset = test_dataset.map(preprocess, batched=True) train_dataset.set_format(type="torch", columns=["input_ids", "attention_mask", "label"]) test_dataset.set_format(type="torch", columns=["input_ids", "attention_mask", "label"]) # Training arguments training_args = TrainingArguments( output_dir="./results", evaluation_strategy="epoch", learning_rate=5e-5, per_device_train_batch_size=16, per_device_eval_batch_size=16, num_train_epochs=3, weight_decay=0.01, logging_dir="./logs", logging_steps=10, ) trainer = Trainer( model=model, args=training_args, train_dataset=train_dataset, eval_dataset=test_dataset, tokenizer=tokenizer, ) # Start tracking emissions tracker.start() tracker.start_task("inference") # Train and evaluate the model trainer.train() # Perform inference predictions = trainer.predict(test_dataset).predictions predictions = torch.argmax(torch.tensor(predictions), axis=1).tolist() true_labels = test_dataset["label"] # Stop tracking emissions emissions_data = tracker.stop_task() # Calculate accuracy accuracy = accuracy_score(true_labels, predictions) # Prepare results dictionary results = { "username": username, "space_url": space_url, "submission_timestamp": datetime.now().isoformat(), "model_description": DESCRIPTION, "accuracy": float(accuracy), "energy_consumed_wh": emissions_data.energy_consumed * 1000, "emissions_gco2eq": emissions_data.emissions * 1000, "emissions_data": clean_emissions_data(emissions_data), "api_route": ROUTE, "dataset_config": { "dataset_name": request.dataset_name, "test_size": request.test_size, "test_seed": request.test_seed } } return results