from fastapi import APIRouter from datetime import datetime from datasets import load_dataset from sklearn.metrics import accuracy_score import random from .utils.evaluation import TextEvaluationRequest from .utils.emissions import tracker, clean_emissions_data, get_space_info router = APIRouter() DESCRIPTION = "First Baseline" ROUTE = "/text" @router.post(ROUTE, tags=["Text Task"], description=DESCRIPTION) async def evaluate_text(request: TextEvaluationRequest): """ Evaluate text classification for climate disinformation detection. Current Model: Random Baseline - Makes random predictions from the label space (0-7) - Used as a baseline for comparison """ # Get space info username, space_url = get_space_info() # Define the label mapping LABEL_MAPPING = { "0_not_relevant": 0, "1_not_happening": 1, "2_not_human": 2, "3_not_bad": 3, "4_solutions_harmful_unnecessary": 4, "5_science_unreliable": 5, "6_proponents_biased": 6, "7_fossil_fuels_needed": 7 } # Load and prepare the dataset dataset = load_dataset(request.dataset_name) # Convert string labels to integers dataset = dataset.map(lambda x: {"label": LABEL_MAPPING[x["label"]]}) # Split dataset train_test = dataset["train"] test_dataset = dataset["test"] # Start tracking emissions tracker.start() tracker.start_task("inference") #-------------------------------------------------------------------------------------------- # YOUR MODEL INFERENCE CODE HERE # Update the code below to replace the random baseline by your model inference within the inference pass where the energy consumption and emissions are tracked. #-------------------------------------------------------------------------------------------- class CovidTwitterBertClassifier(nn.Module): def __init__(self, n_classes): super().__init__() self.n_classes = n_classes self.bert = BertForPreTraining.from_pretrained('digitalepidemiologylab/covid-twitter-bert-v2') self.bert.cls.seq_relationship = nn.Linear(1024, n_classes) self.sigmoid = nn.Sigmoid() def forward(self, input_ids, token_type_ids, input_mask): outputs = self.bert(input_ids = input_ids, token_type_ids = token_type_ids, attention_mask = input_mask) logits = outputs[1] return logits model = CovidTwitterBertClassifier(8) model.to(device) model.load_state_dict(torch.load('model.pth')) model.eval() tokenizer = AutoTokenizer.from_pretrained('digitalepidemiologylab/covid-twitter-bert') test_texts = [t['quote'] for t in data_test] MAX_LEN = 128 #1024 # < m some tweets will be truncated tokenized_test = tokenizer(test_texts, max_length=MAX_LEN, padding='max_length', truncation=True) test_input_ids, test_token_type_ids, test_attention_mask = tokenized_test['input_ids'], tokenized_test['token_type_ids'], tokenized_test['attention_mask'] test_token_type_ids = torch.tensor(test_token_type_ids) test_input_ids = torch.tensor(test_input_ids) test_attention_mask = torch.tensor(test_attention_mask) batch_size = 8 # test_data = TensorDataset(test_input_ids, test_attention_mask, test_token_type_ids) test_sampler = SequentialSampler(test_data) test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=batch_size) predictions = [] for step, batch in enumerate(test_dataloader): # Add batch to GPU batch = tuple(t.to(device) for t in batch) b_input_ids, b_input_mask, b_token_type_ids = batch with torch.no_grad(): logits = model(b_input_ids, b_token_type_ids, b_input_mask) logits = logits.detach().cpu().numpy() predictions.extend(logits.argmax(1)) for l in ground_truth: labels_sep.append(l) true_labels = test_dataset["label"] # Make random predictions (placeholder for actual model inference) #true_labels = test_dataset["label"] #predictions = [random.randint(0, 7) for _ in range(len(true_labels))] #-------------------------------------------------------------------------------------------- # YOUR MODEL INFERENCE STOPS HERE #-------------------------------------------------------------------------------------------- # Stop tracking emissions emissions_data = tracker.stop_task() # Calculate accuracy accuracy = accuracy_score(true_labels, predictions) # Prepare results dictionary results = { "username": username, "space_url": space_url, "submission_timestamp": datetime.now().isoformat(), "model_description": DESCRIPTION, "accuracy": float(accuracy), "energy_consumed_wh": emissions_data.energy_consumed * 1000, "emissions_gco2eq": emissions_data.emissions * 1000, "emissions_data": clean_emissions_data(emissions_data), "api_route": ROUTE, "dataset_config": { "dataset_name": request.dataset_name, "test_size": request.test_size, "test_seed": request.test_seed } } return results