tasks/image.py

from fastapi import APIRouter
from datetime import datetime
from datasets import load_dataset
import numpy as np
from sklearn.metrics import accuracy_score
import random
import os

from torch.utils.data import DataLoader

from ultralytics import YOLO
from .utils.evaluation import ImageEvaluationRequest
from .utils.emissions import tracker, clean_emissions_data, get_space_info

import os
import torch
import numpy as np
from PIL import Image
from transformers import MobileViTImageProcessor, MobileViTForSemanticSegmentation
import cv2
from tqdm import tqdm
from torch.utils.data import DataLoader

from dotenv import load_dotenv
load_dotenv()

router = APIRouter()

DESCRIPTION = "Mobile-ViT Smoke Detection"
ROUTE = "/image"

device = "cpu"
model_path = "mobilevit_segmentation_full_data.pth"
feature_extractor = MobileViTImageProcessor.from_pretrained("apple/deeplabv3-mobilevit-xx-small")
model = MobileViTForSemanticSegmentation.from_pretrained("apple/deeplabv3-mobilevit-xx-small").to(device)
model.load_state_dict(torch.load(model_path, map_location=torch.device('cpu')))
model.eval()

def preprocess(image):
    # Ensure input image is resized to a fixed size (512, 512)
    image = image.resize((512, 512))

    # Convert to NumPy and ensure BGR normalization
    image = np.array(image)[:, :, ::-1]  # Convert RGB to BGR
    image = np.array(image, dtype=np.float32) / 255.0

    # Return as a PIL Image for feature extractor compatibility
    return Image.fromarray((image * 255).astype(np.uint8))

def preprocess_batch(images):
    """
    Preprocess a batch of images for MobileViT inference.
    Resize to a fixed size (512, 512) and return as PIL Images.
    """
    preprocessed_images = []
    for image in images:
        resized_image = image.resize((512, 512))
        image_array = np.array(resized_image)[:, :, ::-1]  # Convert RGB to BGR
        image_float = np.array(image_array, dtype=np.float32) / 255.0
        processed_image = Image.fromarray((image_float * 255).astype(np.uint8))
        preprocessed_images.append(processed_image)
    return preprocessed_images

def get_bounding_boxes_from_mask(mask):
    """Extract bounding boxes from a binary mask."""
    pred_boxes = []
    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    for contour in contours:
        if len(contour) > 5:  # Ignore small/noisy contours
            x, y, w, h = cv2.boundingRect(contour)
            pred_boxes.append((x, y, x + w, y + h))
    return pred_boxes

def parse_boxes(annotation_string):
    """Parse multiple boxes from a single annotation string.
    Each box has 5 values: class_id, x_center, y_center, width, height"""
    values = [float(x) for x in annotation_string.strip().split()]
    boxes = []
    # Each box has 5 values
    for i in range(0, len(values), 5):
        if i + 5 <= len(values):
            # Skip class_id (first value) and take the next 4 values
            box = values[i+1:i+5]
            boxes.append(box)
    return boxes

def compute_iou(box1, box2):
    """Compute Intersection over Union (IoU) between two YOLO format boxes."""
    # Convert YOLO format (x_center, y_center, width, height) to corners
    def yolo_to_corners(box):
        x_center, y_center, width, height = box
        x1 = x_center - width/2
        y1 = y_center - height/2
        x2 = x_center + width/2
        y2 = y_center + height/2
        return np.array([x1, y1, x2, y2])
    
    box1_corners = yolo_to_corners(box1)
    box2_corners = yolo_to_corners(box2)
    
    # Calculate intersection
    x1 = max(box1_corners[0], box2_corners[0])
    y1 = max(box1_corners[1], box2_corners[1])
    x2 = min(box1_corners[2], box2_corners[2])
    y2 = min(box1_corners[3], box2_corners[3])
    
    intersection = max(0, x2 - x1) * max(0, y2 - y1)
    
    # Calculate union
    box1_area = (box1_corners[2] - box1_corners[0]) * (box1_corners[3] - box1_corners[1])
    box2_area = (box2_corners[2] - box2_corners[0]) * (box2_corners[3] - box2_corners[1])
    union = box1_area + box2_area - intersection
    
    return intersection / (union + 1e-6)

def compute_max_iou(true_boxes, pred_box):
    """Compute maximum IoU between a predicted box and all true boxes"""
    max_iou = 0
    for true_box in true_boxes:
        iou = compute_iou(true_box, pred_box)
        max_iou = max(max_iou, iou)
    return max_iou

@router.post(ROUTE, tags=["Image Task"],
             description=DESCRIPTION)
async def evaluate_image(request: ImageEvaluationRequest):
    """
    Evaluate image classification and object detection for forest fire smoke.
    
    Current Model: Random Baseline
    - Makes random predictions for both classification and bounding boxes
    - Used as a baseline for comparison
    
    Metrics:
    - Classification accuracy: Whether an image contains smoke or not
    - Object Detection accuracy: IoU (Intersection over Union) for smoke bounding boxes
    """
    # Get space info
    username, space_url = get_space_info()
    
    # Load and prepare the dataset
    dataset = load_dataset(request.dataset_name, token=os.getenv("HF_TOKEN"))

    # Split dataset
    train_test = dataset["train"].train_test_split(test_size=request.test_size, seed=request.test_seed)
    test_dataset = dataset["val"]#train_test["test"]

    # Start tracking emissions
    tracker.start()
    tracker.start_task("inference")
    
    #--------------------------------------------------------------------------------------------
    # YOUR MODEL INFERENCE CODE HERE
    # Update the code below to replace the random baseline with your model inference
    #--------------------------------------------------------------------------------------------   
    
    dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)

    predictions = []
    true_labels = []
    pred_boxes = []  
    true_boxes_list = []  

    for batch_idx, batch_examples in enumerate(dataloader):
        # Extract images and preprocess
        images = [example["image"] for example in batch_examples]
        annotations = [example.get("annotations", "").strip() for example in batch_examples]
        
        has_smoke_list = [len(annotation) > 0 for annotation in annotations]
        true_labels.extend([1 if has_smoke else 0 for has_smoke in has_smoke_list])

        # Preprocess images and extract features
        preprocessed_images = preprocess_batch(images)
        image_inputs = feature_extractor(images=preprocessed_images, return_tensors="pt", padding=True).pixel_values

        # Perform inference
        with torch.no_grad():
            outputs = model(pixel_values=image_inputs)
            logits = outputs.logits

        # Threshold and process the segmentation masks
        probabilities = torch.sigmoid(logits)
        batch_predicted_masks = (probabilities[:, 1, :, :] > 0.30).cpu().numpy().astype(np.uint8)

        for mask in batch_predicted_masks:
            mask_resized = cv2.resize(mask, (512, 512), interpolation=cv2.INTER_NEAREST)
            predicted_boxes = get_bounding_boxes_from_mask(mask_resized)
            pred_boxes.append(predicted_boxes)

            # Append smoke detection based on bounding boxes
            predictions.append(1 if len(predicted_boxes) > 0 else 0)
            print(f"Batch {batch_idx + 1}, Image Prediction: {1 if len(predicted_boxes) > 0 else 0}")
        
        # Parse true boxes for this batch
        for annotation in annotations:
            if len(annotation) > 0:
                true_boxes_list.append(parse_boxes(annotation))
            else:
                true_boxes_list.append([])
    
    # for example in test_dataset:
    #     # Extract image and annotations
    #     image = example["image"]

    #     original_shape = image.size        
    #     annotation = example.get("annotations", "").strip()
    #     has_smoke = len(annotation) > 0
    #     true_labels.append(1 if has_smoke else 0)
    
    #     if has_smoke:
    #         image_true_boxes = parse_boxes(annotation)
    #         if image_true_boxes:
    #             true_boxes_list.append(image_true_boxes)
    #         else:
    #             true_boxes_list.append([])
    #     else:
    #         true_boxes_list.append([])
    
    #     # Model Inference
    
    #     # Preprocess image
    #     image = preprocess(image)
        
    #     # Ensure correct feature extraction
    #     image_input = feature_extractor(images=image, return_tensors="pt").pixel_values
        
    #     # Perform inference
    #     with torch.no_grad():
    #         outputs = model(pixel_values=image_input)
    #         logits = outputs.logits
        
    #     # Threshold and process the segmentation mask
    #     probabilities = torch.sigmoid(logits)
    #     predicted_mask = (probabilities[0, 1] > 0.30).cpu().numpy().astype(np.uint8)
    #     predicted_mask_resized = cv2.resize(predicted_mask, (512,512), interpolation=cv2.INTER_NEAREST)
        
    #     # Extract bounding boxes
    #     predicted_boxes = get_bounding_boxes_from_mask(predicted_mask_resized)
    #     pred_boxes.append(predicted_boxes)
        
    #     # Smoke prediction based on bounding box presence
    #     predictions.append(1 if len(predicted_boxes) > 0 else 0)    
    #     print(f"Prediction : {1 if len(predicted_boxes) > 0 else 0}")
        

    # # Filter only valid box pairs
    # filtered_true_boxes_list = []
    # filtered_pred_boxes = []
    
    # for true_boxes, pred_boxes_entry in zip(true_boxes_list, pred_boxes):
    #     if true_boxes and pred_boxes_entry:
    #         filtered_true_boxes_list.append(true_boxes)
    #         filtered_pred_boxes.append(pred_boxes_entry)
    
    # true_boxes_list = filtered_true_boxes_list
    # pred_boxes = filtered_pred_boxes

    
    #--------------------------------------------------------------------------------------------
    # YOUR MODEL INFERENCE STOPS HERE
    #--------------------------------------------------------------------------------------------   
    
    # Stop tracking emissions
    emissions_data = tracker.stop_task()
    
    # Calculate classification accuracy
    classification_accuracy = accuracy_score(true_labels, predictions)
    
    # Calculate mean IoU for object detection (only for images with smoke)
    # For each image, we compute the max IoU between the predicted box and all true boxes
    ious = []
    for true_boxes, pred_box in zip(true_boxes_list, pred_boxes):
        max_iou = compute_max_iou(true_boxes, pred_box)
        ious.append(max_iou)
    
    mean_iou = float(np.mean(ious)) if ious else 0.0
    
    # Prepare results dictionary
    results = {
        "username": username,
        "space_url": space_url,
        "submission_timestamp": datetime.now().isoformat(),
        "model_description": DESCRIPTION,
        "classification_accuracy": float(classification_accuracy),
        "mean_iou": mean_iou,
        "energy_consumed_wh": emissions_data.energy_consumed * 1000,
        "emissions_gco2eq": emissions_data.emissions * 1000,
        "emissions_data": clean_emissions_data(emissions_data),
        "api_route": ROUTE,
        "dataset_config": {
            "dataset_name": request.dataset_name,
            "test_size": request.test_size,
            "test_seed": request.test_seed
        }
    }
    return results