improve submission

requirements.txt

@@ -1,13 +1,12 @@
-fastapi>=0.68.0
-uvicorn>=0.15.0
-codecarbon>=2.3.1
-datasets>=2.14.0
-scikit-learn>=1.0.2
-pydantic>=1.10.0
-python-dotenv>=1.0.0
-gradio>=4.0.0
-requests>=2.31.0
-librosa==0.10.2.post1
-torch
-transformers
+fastapi==0.103.2
+uvicorn==0.23.2
+transformers==4.34.0
+torch==2.0.1
+datasets==2.14.5
+scikit-learn==1.3.1
+codecarbon==2.3.1
+python-dotenv==1.0.0
+requests==2.31.0
+numpy==1.24.3
+pydantic==2.4.2
 accelerate

submission_script.py

@@ -14,7 +14,6 @@ def evaluate_text_model(space_url: str, max_retries=3, retry_delay=5):
         "test_seed": 42,
     }
     
-    # Construct base URL and API endpoints
     if "localhost" in space_url:
         base_url = space_url
     else:
@@ -63,7 +62,11 @@ def evaluate_text_model(space_url: str, max_retries=3, retry_delay=5):
                 return response.json()
             else:
                 print(f"Error: Status {response.status_code}")
-                print(f"Response: {response.text}")
+                try:
+                    error_detail = response.json()
+                    print(f"Error detail: {error_detail}")
+                except:
+                    print(f"Response: {response.text}")
                 if attempt < max_retries - 1:
                     print(f"Waiting {retry_delay} seconds before retry...")
                     time.sleep(retry_delay)

tasks/text.py

@@ -1,3 +1,4 @@
+# tasks/text.py
 from fastapi import APIRouter
 from datetime import datetime
 from datasets import load_dataset
@@ -5,10 +6,15 @@ from sklearn.metrics import accuracy_score
 import torch
 from transformers import AutoTokenizer, AutoModelForSequenceClassification
 from torch.utils.data import Dataset, DataLoader
+import logging
 
 from .utils.evaluation import TextEvaluationRequest
 from .utils.emissions import tracker, clean_emissions_data, get_space_info
 
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
 router = APIRouter()
 
 DESCRIPTION = "Climate Guard Toxic Agent Model"
@@ -47,93 +53,108 @@ async def evaluate_text(request: TextEvaluationRequest):
     """
     Evaluate text classification for climate disinformation detection.
     """
-    username, space_url = get_space_info()
+    try:
+        logger.info("Starting evaluation")
+        username, space_url = get_space_info()
 
-    # Label mapping
-    LABEL_MAPPING = {
-        "0_not_relevant": 0,
-        "1_not_happening": 1,
-        "2_not_human": 2,
-        "3_not_bad": 3,
-        "4_solutions_harmful_unnecessary": 4,
-        "5_science_unreliable": 5,
-        "6_proponents_biased": 6,
-        "7_fossil_fuels_needed": 7
-    }
+        # Label mapping
+        LABEL_MAPPING = {
+            "0_not_relevant": 0,
+            "1_not_happening": 1,
+            "2_not_human": 2,
+            "3_not_bad": 3,
+            "4_solutions_harmful_unnecessary": 4,
+            "5_science_unreliable": 5,
+            "6_proponents_biased": 6,
+            "7_fossil_fuels_needed": 7
+        }
 
-    # Load dataset
-    dataset = load_dataset(request.dataset_name)
-    
-    # Convert string labels to integers
-    dataset = dataset.map(lambda x: {"label": LABEL_MAPPING[x["label"]]})
-    
-    # Get test dataset
-    test_dataset = dataset["test"]
-    
-    # Start tracking emissions
-    tracker.start()
-    
-    try:
-        # Load model and tokenizer
-        model_name = "Tonic/climate-guard-toxic-agent"
-        tokenizer = AutoTokenizer.from_pretrained(model_name)
-        model = AutoModelForSequenceClassification.from_pretrained(model_name)
-        
-        # Prepare dataset
-        test_data = TextDataset(
-            texts=test_dataset["text"],
-            labels=test_dataset["label"],
-            tokenizer=tokenizer
-        )
+        logger.info("Loading dataset")
+        # Load dataset
+        dataset = load_dataset(request.dataset_name)
         
-        test_loader = DataLoader(test_data, batch_size=16)
+        # Convert string labels to integers
+        dataset = dataset.map(lambda x: {"label": LABEL_MAPPING[x["label"]]})
         
-        # Model inference
-        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-        model = model.to(device)
-        model.eval()
+        # Get test dataset
+        test_dataset = dataset["test"]
         
-        predictions = []
-        ground_truth = []
+        logger.info("Starting emissions tracking")
+        # Start tracking emissions
+        tracker.start()
         
-        with torch.no_grad():
-            for batch in test_loader:
-                input_ids = batch['input_ids'].to(device)
-                attention_mask = batch['attention_mask'].to(device)
-                labels = batch['labels'].to(device)
-                
-                outputs = model(input_ids=input_ids, attention_mask=attention_mask)
-                _, predicted = torch.max(outputs.logits, 1)
-                
-                predictions.extend(predicted.cpu().numpy())
-                ground_truth.extend(labels.cpu().numpy())
-        
-        # Calculate accuracy
-        accuracy = accuracy_score(ground_truth, predictions)
-        
-        # Stop tracking emissions
-        emissions_data = tracker.stop()
-        
-        # Prepare results
-        results = {
-            "username": username,
-            "space_url": space_url,
-            "submission_timestamp": datetime.now().isoformat(),
-            "model_description": DESCRIPTION,
-            "accuracy": float(accuracy),
-            "energy_consumed_wh": emissions_data.energy_consumed * 1000,
-            "emissions_gco2eq": emissions_data.emissions * 1000,
-            "emissions_data": clean_emissions_data(emissions_data),
-            "api_route": ROUTE,
-            "dataset_config": {
-                "dataset_name": request.dataset_name,
-                "test_size": request.test_size,
-                "test_seed": request.test_seed
+        try:
+            # Load model and tokenizer
+            logger.info("Loading model and tokenizer")
+            model_name = "Tonic/climate-guard-toxic-agent"
+            tokenizer = AutoTokenizer.from_pretrained(model_name)
+            model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=len(LABEL_MAPPING))
+            
+            # Prepare dataset
+            logger.info("Preparing dataset")
+            test_data = TextDataset(
+                texts=test_dataset["text"],
+                labels=test_dataset["label"],
+                tokenizer=tokenizer
+            )
+            
+            test_loader = DataLoader(test_data, batch_size=16)
+            
+            # Model inference
+            device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+            logger.info(f"Using device: {device}")
+            model = model.to(device)
+            model.eval()
+            
+            predictions = []
+            ground_truth = []
+            
+            logger.info("Running inference")
+            with torch.no_grad():
+                for batch in test_loader:
+                    input_ids = batch['input_ids'].to(device)
+                    attention_mask = batch['attention_mask'].to(device)
+                    labels = batch['labels'].to(device)
+                    
+                    outputs = model(input_ids=input_ids, attention_mask=attention_mask)
+                    _, predicted = torch.max(outputs.logits, 1)
+                    
+                    predictions.extend(predicted.cpu().numpy())
+                    ground_truth.extend(labels.cpu().numpy())
+            
+            # Calculate accuracy
+            accuracy = accuracy_score(ground_truth, predictions)
+            logger.info(f"Accuracy: {accuracy}")
+            
+            # Stop tracking emissions
+            emissions_data = tracker.stop()
+            
+            # Prepare results
+            results = {
+                "username": username,
+                "space_url": space_url,
+                "submission_timestamp": datetime.now().isoformat(),
+                "model_description": DESCRIPTION,
+                "accuracy": float(accuracy),
+                "energy_consumed_wh": float(emissions_data.energy_consumed * 1000),
+                "emissions_gco2eq": float(emissions_data.emissions * 1000),
+                "emissions_data": clean_emissions_data(emissions_data),
+                "api_route": ROUTE,
+                "dataset_config": {
+                    "dataset_name": request.dataset_name,
+                    "test_size": request.test_size,
+                    "test_seed": request.test_seed
+                }
             }
-        }
-        
-        return results
-        
+            
+            logger.info("Evaluation completed successfully")
+            return results
+            
+        except Exception as e:
+            logger.error(f"Error during evaluation: {str(e)}")
+            tracker.stop()
+            raise e
+            
     except Exception as e:
-        tracker.stop()
-        raise e
+        logger.error(f"Error in evaluate_text: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))