Deploy Baseer Self-Driving API v1.0

Dockerfile

@@ -0,0 +1,41 @@
+# Read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# Baseer Self-Driving API - Hugging Face Space
+
+FROM python:3.9
+
+# إنشاء مستخدم غير root (متطلب Hugging Face)
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+# تعيين مجلد العمل
+WORKDIR /app
+
+# تثبيت متطلبات النظام كـ root
+USER root
+RUN apt-get update && apt-get install -y \
+    libglib2.0-0 \
+    libsm6 \
+    libxext6 \
+    libxrender-dev \
+    libgomp1 \
+    libgtk-3-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+# العودة لمستخدم user
+USER user
+
+# نسخ ملفات المتطلبات
+COPY --chown=user ./requirements.txt requirements.txt
+
+# تثبيت المتطلبات
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+# نسخ كود التطبيق
+COPY --chown=user . /app
+
+# تعيين متغيرات البيئة
+ENV PYTHONPATH=/app
+
+# تشغيل التطبيق على المنفذ 7860 (متطلب Hugging Face)
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Baseer Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

app.py

@@ -0,0 +1,419 @@
+# app.py - InterFuser Self-Driving API Server
+
+import uuid
+import base64
+import cv2
+import torch
+import numpy as np
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from torchvision import transforms
+from typing import List, Dict, Any, Optional
+import logging
+
+# استيراد من ملفاتنا المحلية
+from model_definition import InterfuserModel, load_and_prepare_model, create_model_config
+from simulation_modules import (
+    InterfuserController, ControllerConfig, Tracker, DisplayInterface, 
+    render, render_waypoints, render_self_car, WAYPOINT_SCALE_FACTOR, 
+    T1_FUTURE_TIME, T2_FUTURE_TIME
+)
+
+# إعداد التسجيل
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# ================== إعدادات عامة وتحميل النموذج ==================
+app = FastAPI(
+    title="Baseer Self-Driving API",
+    description="API للقيادة الذاتية باستخدام نموذج InterFuser",
+    version="1.0.0"
+)
+
+device = torch.device("cpu")
+logger.info(f"Using device: {device}")
+
+# تحميل النموذج باستخدام الدالة المحسنة
+try:
+    # إنشاء إعدادات النموذج باستخدام الإعدادات الصحيحة من التدريب
+    model_config = create_model_config(
+        model_path="model/best_model.pth"
+        # الإعدادات الصحيحة من التدريب ستطبق تلقائياً:
+        # embed_dim=256, rgb_backbone_name='r50', waypoints_pred_head='gru'
+        # with_lidar=False, with_right_left_sensors=False, with_center_sensor=False
+    )
+    
+    # تحميل النموذج مع الأوزان
+    model = load_and_prepare_model(model_config, device)
+    logger.info("✅ تم تحميل النموذج بنجاح")
+    
+except Exception as e:
+    logger.error(f"❌ خطأ في تحميل النموذج: {e}")
+    logger.info("🔄 محاولة إنشاء نموذج بأوزان عشوائية...")
+    try:
+        model = InterfuserModel()
+        model.to(device)
+        model.eval()
+        logger.warning("⚠️ تم إنشاء النموذج بأوزان عشوائية")
+    except Exception as e2:
+        logger.error(f"❌ فشل في إنشاء النموذج: {e2}")
+        model = None
+
+# تهيئة واجهة العرض
+display = DisplayInterface()
+
+# قاموس لتخزين جلسات المستخدمين
+SESSIONS: Dict[str, Dict] = {}
+
+# ================== هياكل بيانات Pydantic ==================
+class Measurements(BaseModel):
+    pos: List[float] = [0.0, 0.0]  # [x, y] position
+    theta: float = 0.0              # orientation angle
+    speed: float = 0.0              # current speed
+    steer: float = 0.0              # current steering
+    throttle: float = 0.0           # current throttle
+    brake: bool = False             # brake status
+    command: int = 4                # driving command (4 = FollowLane)
+    target_point: List[float] = [0.0, 0.0]  # target point [x, y]
+
+class ModelOutputs(BaseModel):
+    traffic: List[List[List[float]]]  # 20x20x7 grid
+    waypoints: List[List[float]]      # Nx2 waypoints
+    is_junction: float
+    traffic_light_state: float
+    stop_sign: float
+
+class ControlCommands(BaseModel):
+    steer: float
+    throttle: float
+    brake: bool
+
+class RunStepInput(BaseModel):
+    session_id: str
+    image_b64: str
+    measurements: Measurements
+
+class RunStepOutput(BaseModel):
+    model_outputs: ModelOutputs
+    control_commands: ControlCommands
+    dashboard_image_b64: str
+
+class SessionResponse(BaseModel):
+    session_id: str
+    message: str
+
+# ================== دوال المساعدة ==================
+def get_image_transform():
+    """إنشاء تحويلات الصورة كما في PDMDataset"""
+    return transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Resize((224, 224), antialias=True),
+        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+    ])
+
+# إنشاء كائن التحويل مرة واحدة
+image_transform = get_image_transform()
+
+def preprocess_input(frame_rgb: np.ndarray, measurements: Measurements, device: torch.device) -> Dict[str, torch.Tensor]:
+    """
+    تحاكي ما يفعله PDMDataset.__getitem__ لإنشاء دفعة (batch) واحدة.
+    """
+    # 1. معالجة الصورة الرئيسية
+    from PIL import Image
+    if isinstance(frame_rgb, np.ndarray):
+        frame_rgb = Image.fromarray(frame_rgb)
+    
+    image_tensor = image_transform(frame_rgb).unsqueeze(0).to(device)  # إضافة بُعد الدفعة
+
+    # 2. إنشاء مدخلات الكاميرات الأخرى عن طريق الاستنساخ
+    batch = {
+        'rgb': image_tensor,
+        'rgb_left': image_tensor.clone(),
+        'rgb_right': image_tensor.clone(),
+        'rgb_center': image_tensor.clone(),
+    }
+
+    # 3. إنشاء مدخل ليدار وهمي (أصفار)
+    batch['lidar'] = torch.zeros(1, 3, 224, 224, dtype=torch.float32).to(device)
+
+    # 4. تجميع القياسات بنفس ترتيب PDMDataset
+    m = measurements
+    measurements_tensor = torch.tensor([[
+        m.pos[0], m.pos[1], m.theta,
+        m.steer, m.throttle, float(m.brake),
+        m.speed, float(m.command)
+    ]], dtype=torch.float32).to(device)
+    batch['measurements'] = measurements_tensor
+
+    # 5. إنشاء نقطة هدف
+    batch['target_point'] = torch.tensor([m.target_point], dtype=torch.float32).to(device)
+    
+    # لا نحتاج إلى قيم ground truth (gt_*) أثناء التنبؤ
+    return batch
+
+def decode_base64_image(image_b64: str) -> np.ndarray:
+    """
+    فك تشفير صورة Base64
+    """
+    try:
+        image_bytes = base64.b64decode(image_b64)
+        nparr = np.frombuffer(image_bytes, np.uint8)
+        image = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
+        return image
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=f"Invalid image format: {str(e)}")
+
+def encode_image_to_base64(image: np.ndarray) -> str:
+    """
+    تشفير صورة إلى Base64
+    """
+    _, buffer = cv2.imencode('.jpg', image, [cv2.IMWRITE_JPEG_QUALITY, 85])
+    return base64.b64encode(buffer).decode('utf-8')
+
+# ================== نقاط نهاية الـ API ==================
+@app.get("/")
+async def root():
+    """
+    نقطة البداية للـ API
+    """
+    return {
+        "message": "InterFuser Self-Driving API",
+        "version": "1.0.0",
+        "status": "running",
+        "active_sessions": len(SESSIONS)
+    }
+
+@app.post("/start_session", response_model=SessionResponse)
+async def start_session():
+    """
+    بدء جلسة جديدة للمحاكاة
+    """
+    session_id = str(uuid.uuid4())
+    
+    # إنشاء جلسة جديدة
+    SESSIONS[session_id] = {
+        'tracker': Tracker(frequency=10),
+        'controller': InterfuserController(ControllerConfig()),
+        'frame_num': 0,
+        'created_at': np.datetime64('now'),
+        'last_activity': np.datetime64('now')
+    }
+    
+    logger.info(f"New session created: {session_id}")
+    
+    return SessionResponse(
+        session_id=session_id,
+        message="Session started successfully"
+    )
+
+@app.post("/run_step", response_model=RunStepOutput)
+async def run_step(data: RunStepInput):
+    """
+    تنفيذ خطوة محاكاة كاملة
+    """
+    # التحقق من وجود الجلسة
+    if data.session_id not in SESSIONS:
+        raise HTTPException(status_code=404, detail="Session not found")
+    
+    session = SESSIONS[data.session_id]
+    tracker = session['tracker']
+    controller = session['controller']
+    
+    # تحديث وقت النشاط
+    session['last_activity'] = np.datetime64('now')
+    
+    try:
+        # 1. فك تشفير الصورة
+        frame_bgr = decode_base64_image(data.image_b64)
+        frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
+        
+        # 2. معالجة المدخلات
+        inputs = preprocess_input(frame_rgb, data.measurements, device)
+        
+        # 3. تشغيل النموذج
+        if model is None:
+            raise HTTPException(status_code=500, detail="Model not loaded")
+        
+        with torch.no_grad():
+            traffic, waypoints, is_junction, traffic_light, stop_sign, _ = model(inputs)
+        
+        # 4. معالجة مخرجات النموذج
+        traffic_np = traffic.cpu().numpy()[0]  # أخذ أول عنصر من الـ batch
+        waypoints_np = waypoints.cpu().numpy()[0]
+        is_junction_prob = torch.sigmoid(is_junction)[0, 1].item()
+        traffic_light_prob = torch.sigmoid(traffic_light)[0, 0].item()
+        stop_sign_prob = torch.sigmoid(stop_sign)[0, 1].item()
+        
+        # 5. تحديث التتبع
+        # تحويل traffic grid إلى detections للتتبع
+        detections = []
+        h, w, c = traffic_np.shape
+        for y in range(h):
+            for x in range(w):
+                for ch in range(c):
+                    if traffic_np[y, x, ch] > 0.2:  # عتبة الكشف
+                        world_x = (x / w - 0.5) * 64  # تحويل إلى إحداثيات العالم
+                        world_y = (y / h - 0.5) * 64
+                        detections.append({
+                            'position': [world_x, world_y],
+                            'feature': traffic_np[y, x, ch]
+                        })
+        
+        updated_traffic = tracker.update_and_predict(detections, session['frame_num'])
+        
+        # 6. تشغيل المتحكم
+        steer, throttle, brake, metadata = controller.run_step(
+            current_speed=data.measurements.speed,
+            waypoints=waypoints_np,
+            junction=is_junction_prob,
+            traffic_light_state=traffic_light_prob,
+            stop_sign=stop_sign_prob,
+            meta_data={'frame': session['frame_num']}
+        )
+        
+        # 7. إنشاء خرائط العرض
+        surround_t0, counts_t0 = render(updated_traffic, t=0)
+        surround_t1, counts_t1 = render(updated_traffic, t=T1_FUTURE_TIME)
+        surround_t2, counts_t2 = render(updated_traffic, t=T2_FUTURE_TIME)
+        
+        # إضافة المسار المقترح
+        wp_map = render_waypoints(waypoints_np)
+        map_t0 = cv2.add(surround_t0, wp_map)
+        
+        # إضافة السيارة الذاتية
+        map_t0 = render_self_car(map_t0)
+        map_t1 = render_self_car(surround_t1)
+        map_t2 = render_self_car(surround_t2)
+        
+        # 8. إنشاء لوحة العرض النهائية
+        interface_data = {
+            'camera_view': frame_bgr,
+            'map_t0': map_t0,
+            'map_t1': map_t1,
+            'map_t2': map_t2,
+            'text_info': {
+                'Frame': f"Frame: {session['frame_num']}",
+                'Control': f"Steer: {steer:.2f}, Throttle: {throttle:.2f}, Brake: {brake}",
+                'Speed': f"Speed: {data.measurements.speed:.1f} km/h",
+                'Junction': f"Junction: {is_junction_prob:.2f}",
+                'Traffic Light': f"Red Light: {traffic_light_prob:.2f}",
+                'Stop Sign': f"Stop Sign: {stop_sign_prob:.2f}",
+                'Metadata': metadata
+            },
+            'object_counts': {
+                't0': counts_t0,
+                't1': counts_t1,
+                't2': counts_t2
+            }
+        }
+        
+        dashboard_image = display.run_interface(interface_data)
+        dashboard_b64 = encode_image_to_base64(dashboard_image)
+        
+        # 9. تجميع المخرجات النهائية
+        response = RunStepOutput(
+            model_outputs=ModelOutputs(
+                traffic=traffic_np.tolist(),
+                waypoints=waypoints_np.tolist(),
+                is_junction=is_junction_prob,
+                traffic_light_state=traffic_light_prob,
+                stop_sign=stop_sign_prob
+            ),
+            control_commands=ControlCommands(
+                steer=float(steer),
+                throttle=float(throttle),
+                brake=bool(brake)
+            ),
+            dashboard_image_b64=dashboard_b64
+        )
+        
+        # تحديث رقم الإطار
+        session['frame_num'] += 1
+        
+        logger.info(f"Step completed for session {data.session_id}, frame {session['frame_num']}")
+        
+        return response
+        
+    except Exception as e:
+        logger.error(f"Error in run_step: {str(e)}")
+        raise HTTPException(status_code=500, detail=f"Processing error: {str(e)}")
+
+@app.post("/end_session", response_model=SessionResponse)
+async def end_session(session_id: str):
+    """
+    إنهاء جلسة المحاكاة
+    """
+    if session_id not in SESSIONS:
+        raise HTTPException(status_code=404, detail="Session not found")
+    
+    # حذف الجلسة
+    del SESSIONS[session_id]
+    
+    logger.info(f"Session ended: {session_id}")
+    
+    return SessionResponse(
+        session_id=session_id,
+        message="Session ended successfully"
+    )
+
+@app.get("/sessions")
+async def list_sessions():
+    """
+    عرض قائمة الجلسات النشطة
+    """
+    active_sessions = []
+    current_time = np.datetime64('now')
+    
+    for session_id, session_data in SESSIONS.items():
+        time_diff = current_time - session_data['last_activity']
+        active_sessions.append({
+            'session_id': session_id,
+            'frame_count': session_data['frame_num'],
+            'created_at': str(session_data['created_at']),
+            'last_activity': str(session_data['last_activity']),
+            'inactive_minutes': float(time_diff / np.timedelta64(1, 'm'))
+        })
+    
+    return {
+        'total_sessions': len(active_sessions),
+        'sessions': active_sessions
+    }
+
+@app.delete("/sessions/cleanup")
+async def cleanup_inactive_sessions(max_inactive_minutes: int = 30):
+    """
+    تنظيف الجلسات غير النشطة
+    """
+    current_time = np.datetime64('now')
+    cleaned_sessions = []
+    
+    for session_id in list(SESSIONS.keys()):
+        session = SESSIONS[session_id]
+        time_diff = current_time - session['last_activity']
+        inactive_minutes = float(time_diff / np.timedelta64(1, 'm'))
+        
+        if inactive_minutes > max_inactive_minutes:
+            del SESSIONS[session_id]
+            cleaned_sessions.append(session_id)
+    
+    logger.info(f"Cleaned up {len(cleaned_sessions)} inactive sessions")
+    
+    return {
+        'message': f"Cleaned up {len(cleaned_sessions)} inactive sessions",
+        'cleaned_sessions': cleaned_sessions,
+        'remaining_sessions': len(SESSIONS)
+    }
+
+# ================== معالج الأخطاء ==================
+@app.exception_handler(Exception)
+async def global_exception_handler(request, exc):
+    logger.error(f"Global exception: {str(exc)}")
+    return {
+        "error": "Internal server error",
+        "detail": str(exc)
+    }
+
+# ================== تشغيل الخادم ==================
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=7860)

app_config.yaml

@@ -0,0 +1,17 @@
+title: Baseer Self-Driving API
+emoji: 🚗
+colorFrom: blue
+colorTo: green
+sdk: docker
+app_port: 7860
+pinned: false
+license: mit
+short_description: API للقيادة الذاتية باستخدام نموذج Baseer InterFuser
+tags:
+  - computer-vision
+  - autonomous-driving
+  - deep-learning
+  - fastapi
+  - pytorch
+  - interfuser
+  - graduation-project

health_check.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+# health_check.py - فحص صحة النظام قبل النشر
+
+import os
+import sys
+import torch
+import logging
+from pathlib import Path
+
+def check_python_version():
+    """فحص إصدار Python"""
+    version = sys.version_info
+    if version.major == 3 and version.minor >= 9:
+        print(f"✅ Python {version.major}.{version.minor}.{version.micro}")
+        return True
+    else:
+        print(f"❌ Python {version.major}.{version.minor}.{version.micro} - يتطلب Python 3.9+")
+        return False
+
+def check_pytorch():
+    """فحص PyTorch"""
+    try:
+        print(f"✅ PyTorch {torch.__version__}")
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        print(f"✅ Device: {device}")
+        return True
+    except Exception as e:
+        print(f"❌ PyTorch Error: {e}")
+        return False
+
+def check_required_files():
+    """فحص الملفات المطلوبة"""
+    required_files = [
+        "app.py",
+        "model_definition.py", 
+        "simulation_modules.py",
+        "requirements.txt",
+        "Dockerfile",
+        "app_config.yaml",
+        "model/best_model.pth"
+    ]
+    
+    missing_files = []
+    for file in required_files:
+        if Path(file).exists():
+            size = Path(file).stat().st_size
+            print(f"✅ {file} ({size:,} bytes)")
+        else:
+            print(f"❌ {file} - مفقود")
+            missing_files.append(file)
+    
+    return len(missing_files) == 0
+
+def check_model_loading():
+    """فحص تحميل النموذج"""
+    try:
+        from model_definition import InterfuserModel, create_model_config, load_and_prepare_model
+        
+        # إنشاء إعدادات النموذج
+        config = create_model_config("model/best_model.pth")
+        print("✅ تم إنشاء إعدادات النموذج")
+        
+        # تحميل النموذج
+        device = torch.device("cpu")
+        model = load_and_prepare_model(config, device)
+        print("✅ تم تحميل النموذج بنجاح")
+        
+        return True
+        
+    except Exception as e:
+        print(f"❌ خطأ في تحميل النموذج: {e}")
+        return False
+
+def check_api_imports():
+    """فحص استيراد مكونات الـ API"""
+    try:
+        from app import app
+        print("✅ تم استيراد FastAPI app")
+        
+        from simulation_modules import DisplayInterface, InterfuserController
+        print("✅ تم استيراد وحدات المحاكاة")
+        
+        return True
+        
+    except Exception as e:
+        print(f"❌ خطأ في استيراد الـ API: {e}")
+        return False
+
+def main():
+    """الفحص الشامل للنظام"""
+    print("🔍 فحص صحة نظام Baseer Self-Driving API")
+    print("=" * 50)
+    
+    checks = [
+        ("Python Version", check_python_version),
+        ("PyTorch", check_pytorch),
+        ("Required Files", check_required_files),
+        ("API Imports", check_api_imports),
+        ("Model Loading", check_model_loading),
+    ]
+    
+    passed = 0
+    total = len(checks)
+    
+    for name, check_func in checks:
+        print(f"\n🔍 {name}:")
+        try:
+            if check_func():
+                passed += 1
+            else:
+                print(f"❌ فشل في فحص {name}")
+        except Exception as e:
+            print(f"❌ خطأ في فحص {name}: {e}")
+    
+    print("\n" + "=" * 50)
+    print(f"📊 النتيجة النهائية: {passed}/{total} فحوصات نجحت")
+    
+    if passed == total:
+        print("🎉 النظام جاهز للنشر!")
+        return True
+    else:
+        print("⚠️ يجب إصلاح المشاكل قبل النشر")
+        return False
+
+if __name__ == "__main__":
+    success = main()
+    sys.exit(0 if success else 1)

model/README.md

@@ -0,0 +1,29 @@
+# InterFuser Model Directory
+
+## Required Files
+
+Place your trained InterFuser model files in this directory:
+
+1. **`interfuser_model.pth`** - The main model weights file
+2. **`config.json`** (optional) - Model configuration file
+
+## Model Format
+
+The model should be a PyTorch state dict saved with:
+```python
+torch.save(model.state_dict(), 'interfuser_model.pth')
+```
+
+## Loading in Code
+
+The model is loaded in `model_definition.py`:
+```python
+model = InterFuserModel()
+model.load_state_dict(torch.load('model/interfuser_model.pth', map_location='cpu'))
+```
+
+## Note
+
+- The current implementation uses a dummy model for testing
+- Replace with your actual trained InterFuser weights
+- Ensure the model architecture matches the one defined in `model_definition.py`

model_definition.py

@@ -0,0 +1,1318 @@
+# model_definition.py
+# ============================================================================
+# الاستيرادات الأساسية
+# ============================================================================
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+from torch.optim import AdamW
+from torch.optim.lr_scheduler import OneCycleLR
+from torch.utils.data import Dataset, DataLoader
+from torchvision import transforms
+from functools import partial
+from typing import Optional, List
+from torch import Tensor
+
+# مكتبات إضافية
+import os
+import json
+import logging
+import math
+import copy
+from pathlib import Path
+from collections import OrderedDict
+
+# مكتبات معالجة البيانات
+import numpy as np
+import cv2
+
+# مكتبات اختيارية (يمكن تعطيلها إذا لم تكن متوفرة)
+try:
+    import wandb
+    WANDB_AVAILABLE = True
+except ImportError:
+    WANDB_AVAILABLE = False
+    
+try:
+    from tqdm import tqdm
+except ImportError:
+    # إذا لم تكن tqdm متوفرة، استخدم دالة بديلة
+    def tqdm(iterable, *args, **kwargs):
+        return iterable
+
+# ============================================================================
+# دوال مساعدة
+# ============================================================================
+def to_2tuple(x):
+    """تحويل قيمة إلى tuple من عنصرين"""
+    if isinstance(x, (list, tuple)):
+        return tuple(x)
+    return (x, x)
+# ============================================================================
+# ============================================================================
+
+class HybridEmbed(nn.Module):
+    def __init__(
+        self,
+        backbone,
+        img_size=224,
+        patch_size=1,
+        feature_size=None,
+        in_chans=3,
+        embed_dim=768,
+    ):
+        super().__init__()
+        assert isinstance(backbone, nn.Module)
+        img_size = to_2tuple(img_size)
+        patch_size = to_2tuple(patch_size)
+        self.img_size = img_size
+        self.patch_size = patch_size
+        self.backbone = backbone
+        if feature_size is None:
+            with torch.no_grad():
+                training = backbone.training
+                if training:
+                    backbone.eval()
+                o = self.backbone(torch.zeros(1, in_chans, img_size[0], img_size[1]))
+                if isinstance(o, (list, tuple)):
+                    o = o[-1]  # last feature if backbone outputs list/tuple of features
+                feature_size = o.shape[-2:]
+                feature_dim = o.shape[1]
+                backbone.train(training)
+        else:
+            feature_size = to_2tuple(feature_size)
+            if hasattr(self.backbone, "feature_info"):
+                feature_dim = self.backbone.feature_info.channels()[-1]
+            else:
+                feature_dim = self.backbone.num_features
+
+        self.proj = nn.Conv2d(feature_dim, embed_dim, kernel_size=1, stride=1)
+
+    def forward(self, x):
+        x = self.backbone(x)
+        if isinstance(x, (list, tuple)):
+            x = x[-1]  # last feature if backbone outputs list/tuple of features
+        x = self.proj(x)
+        global_x = torch.mean(x, [2, 3], keepdim=False)[:, :, None]
+        return x, global_x
+
+
+class PositionEmbeddingSine(nn.Module):
+    """
+    This is a more standard version of the position embedding, very similar to the one
+    used by the Attention is all you need paper, generalized to work on images.
+    """
+
+    def __init__(
+        self, num_pos_feats=64, temperature=10000, normalize=False, scale=None
+    ):
+        super().__init__()
+        self.num_pos_feats = num_pos_feats
+        self.temperature = temperature
+        self.normalize = normalize
+        if scale is not None and normalize is False:
+            raise ValueError("normalize should be True if scale is passed")
+        if scale is None:
+            scale = 2 * math.pi
+        self.scale = scale
+
+    def forward(self, tensor):
+        x = tensor
+        bs, _, h, w = x.shape
+        not_mask = torch.ones((bs, h, w), device=x.device)
+        y_embed = not_mask.cumsum(1, dtype=torch.float32)
+        x_embed = not_mask.cumsum(2, dtype=torch.float32)
+        if self.normalize:
+            eps = 1e-6
+            y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale
+            x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale
+
+        dim_t = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device)
+        dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_pos_feats)
+
+        pos_x = x_embed[:, :, :, None] / dim_t
+        pos_y = y_embed[:, :, :, None] / dim_t
+        pos_x = torch.stack(
+            (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4
+        ).flatten(3)
+        pos_y = torch.stack(
+            (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4
+        ).flatten(3)
+        pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2)
+        return pos
+
+
+class TransformerEncoder(nn.Module):
+    def __init__(self, encoder_layer, num_layers, norm=None):
+        super().__init__()
+        self.layers = _get_clones(encoder_layer, num_layers)
+        self.num_layers = num_layers
+        self.norm = norm
+
+    def forward(
+        self,
+        src,
+        mask: Optional[Tensor] = None,
+        src_key_padding_mask: Optional[Tensor] = None,
+        pos: Optional[Tensor] = None,
+    ):
+        output = src
+
+        for layer in self.layers:
+            output = layer(
+                output,
+                src_mask=mask,
+                src_key_padding_mask=src_key_padding_mask,
+                pos=pos,
+            )
+
+        if self.norm is not None:
+            output = self.norm(output)
+
+        return output
+
+
+class SpatialSoftmax(nn.Module):
+    def __init__(self, height, width, channel, temperature=None, data_format="NCHW"):
+        super().__init__()
+
+        self.data_format = data_format
+        self.height = height
+        self.width = width
+        self.channel = channel
+
+        if temperature:
+            self.temperature = Parameter(torch.ones(1) * temperature)
+        else:
+            self.temperature = 1.0
+
+        pos_x, pos_y = np.meshgrid(
+            np.linspace(-1.0, 1.0, self.height), np.linspace(-1.0, 1.0, self.width)
+        )
+        pos_x = torch.from_numpy(pos_x.reshape(self.height * self.width)).float()
+        pos_y = torch.from_numpy(pos_y.reshape(self.height * self.width)).float()
+        self.register_buffer("pos_x", pos_x)
+        self.register_buffer("pos_y", pos_y)
+
+    def forward(self, feature):
+        # Output:
+        #   (N, C*2) x_0 y_0 ...
+
+        if self.data_format == "NHWC":
+            feature = (
+                feature.transpose(1, 3)
+                .tranpose(2, 3)
+                .view(-1, self.height * self.width)
+            )
+        else:
+            feature = feature.view(-1, self.height * self.width)
+
+        weight = F.softmax(feature / self.temperature, dim=-1)
+        expected_x = torch.sum(
+            torch.autograd.Variable(self.pos_x) * weight, dim=1, keepdim=True
+        )
+        expected_y = torch.sum(
+            torch.autograd.Variable(self.pos_y) * weight, dim=1, keepdim=True
+        )
+        expected_xy = torch.cat([expected_x, expected_y], 1)
+        feature_keypoints = expected_xy.view(-1, self.channel, 2)
+        feature_keypoints[:, :, 1] = (feature_keypoints[:, :, 1] - 1) * 12
+        feature_keypoints[:, :, 0] = feature_keypoints[:, :, 0] * 12
+        return feature_keypoints
+
+
+class MultiPath_Generator(nn.Module):
+    def __init__(self, in_channel, embed_dim, out_channel):
+        super().__init__()
+        self.spatial_softmax = SpatialSoftmax(100, 100, out_channel)
+        self.tconv0 = nn.Sequential(
+            nn.ConvTranspose2d(in_channel, 256, 4, 2, 1, bias=False),
+            nn.BatchNorm2d(256),
+            nn.ReLU(True),
+        )
+        self.tconv1 = nn.Sequential(
+            nn.ConvTranspose2d(256, 256, 4, 2, 1, bias=False),
+            nn.BatchNorm2d(256),
+            nn.ReLU(True),
+        )
+        self.tconv2 = nn.Sequential(
+            nn.ConvTranspose2d(256, 192, 4, 2, 1, bias=False),
+            nn.BatchNorm2d(192),
+            nn.ReLU(True),
+        )
+        self.tconv3 = nn.Sequential(
+            nn.ConvTranspose2d(192, 64, 4, 2, 1, bias=False),
+            nn.BatchNorm2d(64),
+            nn.ReLU(True),
+        )
+        self.tconv4_list = torch.nn.ModuleList(
+            [
+                nn.Sequential(
+                    nn.ConvTranspose2d(64, out_channel, 8, 2, 3, bias=False),
+                    nn.Tanh(),
+                )
+                for _ in range(6)
+            ]
+        )
+
+        self.upsample = nn.Upsample(size=(50, 50), mode="bilinear")
+
+    def forward(self, x, measurements):
+        mask = measurements[:, :6]
+        mask = mask.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).repeat(1, 1, 1, 100, 100)
+        velocity = measurements[:, 6:7].unsqueeze(-1).unsqueeze(-1)
+        velocity = velocity.repeat(1, 32, 2, 2)
+
+        n, d, c = x.shape
+        x = x.transpose(1, 2)
+        x = x.view(n, -1, 2, 2)
+        x = torch.cat([x, velocity], dim=1)
+        x = self.tconv0(x)
+        x = self.tconv1(x)
+        x = self.tconv2(x)
+        x = self.tconv3(x)
+        x = self.upsample(x)
+        xs = []
+        for i in range(6):
+            xt = self.tconv4_list[i](x)
+            xs.append(xt)
+        xs = torch.stack(xs, dim=1)
+        x = torch.sum(xs * mask, dim=1)
+        x = self.spatial_softmax(x)
+        return x
+
+
+class LinearWaypointsPredictor(nn.Module):
+    def __init__(self, input_dim, cumsum=True):
+        super().__init__()
+        self.cumsum = cumsum
+        self.rank_embed = nn.Parameter(torch.zeros(1, 10, input_dim))
+        self.head_fc1_list = nn.ModuleList([nn.Linear(input_dim, 64) for _ in range(6)])
+        self.head_relu = nn.ReLU(inplace=True)
+        self.head_fc2_list = nn.ModuleList([nn.Linear(64, 2) for _ in range(6)])
+
+    def forward(self, x, measurements):
+        # input shape: n 10 embed_dim
+        bs, n, dim = x.shape
+        x = x + self.rank_embed
+        x = x.reshape(-1, dim)
+
+        mask = measurements[:, :6]
+        mask = torch.unsqueeze(mask, -1).repeat(n, 1, 2)
+
+        rs = []
+        for i in range(6):
+            res = self.head_fc1_list[i](x)
+            res = self.head_relu(res)
+            res = self.head_fc2_list[i](res)
+            rs.append(res)
+        rs = torch.stack(rs, 1)
+        x = torch.sum(rs * mask, dim=1)
+
+        x = x.view(bs, n, 2)
+        if self.cumsum:
+            x = torch.cumsum(x, 1)
+        return x
+
+
+class GRUWaypointsPredictor(nn.Module):
+    def __init__(self, input_dim, waypoints=10):
+        super().__init__()
+        # self.gru = torch.nn.GRUCell(input_size=input_dim, hidden_size=64)
+        self.gru = torch.nn.GRU(input_size=input_dim, hidden_size=64, batch_first=True)
+        self.encoder = nn.Linear(2, 64)
+        self.decoder = nn.Linear(64, 2)
+        self.waypoints = waypoints
+
+    def forward(self, x, target_point):
+        bs = x.shape[0]
+        z = self.encoder(target_point).unsqueeze(0)
+        output, _ = self.gru(x, z)
+        output = output.reshape(bs * self.waypoints, -1)
+        output = self.decoder(output).reshape(bs, self.waypoints, 2)
+        output = torch.cumsum(output, 1)
+        return output
+
+class GRUWaypointsPredictorWithCommand(nn.Module):
+    def __init__(self, input_dim, waypoints=10):
+        super().__init__()
+        # self.gru = torch.nn.GRUCell(input_size=input_dim, hidden_size=64)
+        self.grus = nn.ModuleList([torch.nn.GRU(input_size=input_dim, hidden_size=64, batch_first=True) for _ in range(6)])
+        self.encoder = nn.Linear(2, 64)
+        self.decoders = nn.ModuleList([nn.Linear(64, 2) for _ in range(6)])
+        self.waypoints = waypoints
+
+    def forward(self, x, target_point, measurements):
+        bs, n, dim = x.shape
+        mask = measurements[:, :6, None, None]
+        mask = mask.repeat(1, 1, self.waypoints, 2)
+
+        z = self.encoder(target_point).unsqueeze(0)
+        outputs = []
+        for i in range(6):
+            output, _ = self.grus[i](x, z)
+            output = output.reshape(bs * self.waypoints, -1)
+            output = self.decoders[i](output).reshape(bs, self.waypoints, 2)
+            output = torch.cumsum(output, 1)
+            outputs.append(output)
+        outputs = torch.stack(outputs, 1)
+        output = torch.sum(outputs * mask, dim=1)
+        return output
+
+
+class TransformerDecoder(nn.Module):
+    def __init__(self, decoder_layer, num_layers, norm=None, return_intermediate=False):
+        super().__init__()
+        self.layers = _get_clones(decoder_layer, num_layers)
+        self.num_layers = num_layers
+        self.norm = norm
+        self.return_intermediate = return_intermediate
+
+    def forward(
+        self,
+        tgt,
+        memory,
+        tgt_mask: Optional[Tensor] = None,
+        memory_mask: Optional[Tensor] = None,
+        tgt_key_padding_mask: Optional[Tensor] = None,
+        memory_key_padding_mask: Optional[Tensor] = None,
+        pos: Optional[Tensor] = None,
+        query_pos: Optional[Tensor] = None,
+    ):
+        output = tgt
+
+        intermediate = []
+
+        for layer in self.layers:
+            output = layer(
+                output,
+                memory,
+                tgt_mask=tgt_mask,
+                memory_mask=memory_mask,
+                tgt_key_padding_mask=tgt_key_padding_mask,
+                memory_key_padding_mask=memory_key_padding_mask,
+                pos=pos,
+                query_pos=query_pos,
+            )
+            if self.return_intermediate:
+                intermediate.append(self.norm(output))
+
+        if self.norm is not None:
+            output = self.norm(output)
+            if self.return_intermediate:
+                intermediate.pop()
+                intermediate.append(output)
+
+        if self.return_intermediate:
+            return torch.stack(intermediate)
+
+        return output.unsqueeze(0)
+
+
+class TransformerEncoderLayer(nn.Module):
+    def __init__(
+        self,
+        d_model,
+        nhead,
+        dim_feedforward=2048,
+        dropout=0.1,
+        activation=nn.ReLU(),
+        normalize_before=False,
+    ):
+        super().__init__()
+        self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
+        # Implementation of Feedforward model
+        self.linear1 = nn.Linear(d_model, dim_feedforward)
+        self.dropout = nn.Dropout(dropout)
+        self.linear2 = nn.Linear(dim_feedforward, d_model)
+
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+        self.dropout1 = nn.Dropout(dropout)
+        self.dropout2 = nn.Dropout(dropout)
+
+        self.activation = activation()
+        self.normalize_before = normalize_before
+
+    def with_pos_embed(self, tensor, pos: Optional[Tensor]):
+        return tensor if pos is None else tensor + pos
+
+    def forward_post(
+        self,
+        src,
+        src_mask: Optional[Tensor] = None,
+        src_key_padding_mask: Optional[Tensor] = None,
+        pos: Optional[Tensor] = None,
+    ):
+        q = k = self.with_pos_embed(src, pos)
+        src2 = self.self_attn(
+            q, k, value=src, attn_mask=src_mask, key_padding_mask=src_key_padding_mask
+        )[0]
+        src = src + self.dropout1(src2)
+        src = self.norm1(src)
+        src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
+        src = src + self.dropout2(src2)
+        src = self.norm2(src)
+        return src
+
+    def forward_pre(
+        self,
+        src,
+        src_mask: Optional[Tensor] = None,
+        src_key_padding_mask: Optional[Tensor] = None,
+        pos: Optional[Tensor] = None,
+    ):
+        src2 = self.norm1(src)
+        q = k = self.with_pos_embed(src2, pos)
+        src2 = self.self_attn(
+            q, k, value=src2, attn_mask=src_mask, key_padding_mask=src_key_padding_mask
+        )[0]
+        src = src + self.dropout1(src2)
+        src2 = self.norm2(src)
+        src2 = self.linear2(self.dropout(self.activation(self.linear1(src2))))
+        src = src + self.dropout2(src2)
+        return src
+
+    def forward(
+        self,
+        src,
+        src_mask: Optional[Tensor] = None,
+        src_key_padding_mask: Optional[Tensor] = None,
+        pos: Optional[Tensor] = None,
+    ):
+        if self.normalize_before:
+            return self.forward_pre(src, src_mask, src_key_padding_mask, pos)
+        return self.forward_post(src, src_mask, src_key_padding_mask, pos)
+
+
+class TransformerDecoderLayer(nn.Module):
+    def __init__(
+        self,
+        d_model,
+        nhead,
+        dim_feedforward=2048,
+        dropout=0.1,
+        activation=nn.ReLU(),
+        normalize_before=False,
+    ):
+        super().__init__()
+        self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
+        self.multihead_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout)
+        # Implementation of Feedforward model
+        self.linear1 = nn.Linear(d_model, dim_feedforward)
+        self.dropout = nn.Dropout(dropout)
+        self.linear2 = nn.Linear(dim_feedforward, d_model)
+
+        self.norm1 = nn.LayerNorm(d_model)
+        self.norm2 = nn.LayerNorm(d_model)
+        self.norm3 = nn.LayerNorm(d_model)
+        self.dropout1 = nn.Dropout(dropout)
+        self.dropout2 = nn.Dropout(dropout)
+        self.dropout3 = nn.Dropout(dropout)
+
+        self.activation = activation()
+        self.normalize_before = normalize_before
+
+    def with_pos_embed(self, tensor, pos: Optional[Tensor]):
+        return tensor if pos is None else tensor + pos
+
+    def forward_post(
+        self,
+        tgt,
+        memory,
+        tgt_mask: Optional[Tensor] = None,
+        memory_mask: Optional[Tensor] = None,
+        tgt_key_padding_mask: Optional[Tensor] = None,
+        memory_key_padding_mask: Optional[Tensor] = None,
+        pos: Optional[Tensor] = None,
+        query_pos: Optional[Tensor] = None,
+    ):
+        q = k = self.with_pos_embed(tgt, query_pos)
+        tgt2 = self.self_attn(
+            q, k, value=tgt, attn_mask=tgt_mask, key_padding_mask=tgt_key_padding_mask
+        )[0]
+        tgt = tgt + self.dropout1(tgt2)
+        tgt = self.norm1(tgt)
+        tgt2 = self.multihead_attn(
+            query=self.with_pos_embed(tgt, query_pos),
+            key=self.with_pos_embed(memory, pos),
+            value=memory,
+            attn_mask=memory_mask,
+            key_padding_mask=memory_key_padding_mask,
+        )[0]
+        tgt = tgt + self.dropout2(tgt2)
+        tgt = self.norm2(tgt)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
+        tgt = tgt + self.dropout3(tgt2)
+        tgt = self.norm3(tgt)
+        return tgt
+
+    def forward_pre(
+        self,
+        tgt,
+        memory,
+        tgt_mask: Optional[Tensor] = None,
+        memory_mask: Optional[Tensor] = None,
+        tgt_key_padding_mask: Optional[Tensor] = None,
+        memory_key_padding_mask: Optional[Tensor] = None,
+        pos: Optional[Tensor] = None,
+        query_pos: Optional[Tensor] = None,
+    ):
+        tgt2 = self.norm1(tgt)
+        q = k = self.with_pos_embed(tgt2, query_pos)
+        tgt2 = self.self_attn(
+            q, k, value=tgt2, attn_mask=tgt_mask, key_padding_mask=tgt_key_padding_mask
+        )[0]
+        tgt = tgt + self.dropout1(tgt2)
+        tgt2 = self.norm2(tgt)
+        tgt2 = self.multihead_attn(
+            query=self.with_pos_embed(tgt2, query_pos),
+            key=self.with_pos_embed(memory, pos),
+            value=memory,
+            attn_mask=memory_mask,
+            key_padding_mask=memory_key_padding_mask,
+        )[0]
+        tgt = tgt + self.dropout2(tgt2)
+        tgt2 = self.norm3(tgt)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt2))))
+        tgt = tgt + self.dropout3(tgt2)
+        return tgt
+
+    def forward(
+        self,
+        tgt,
+        memory,
+        tgt_mask: Optional[Tensor] = None,
+        memory_mask: Optional[Tensor] = None,
+        tgt_key_padding_mask: Optional[Tensor] = None,
+        memory_key_padding_mask: Optional[Tensor] = None,
+        pos: Optional[Tensor] = None,
+        query_pos: Optional[Tensor] = None,
+    ):
+        if self.normalize_before:
+            return self.forward_pre(
+                tgt,
+                memory,
+                tgt_mask,
+                memory_mask,
+                tgt_key_padding_mask,
+                memory_key_padding_mask,
+                pos,
+                query_pos,
+            )
+        return self.forward_post(
+            tgt,
+            memory,
+            tgt_mask,
+            memory_mask,
+            tgt_key_padding_mask,
+            memory_key_padding_mask,
+            pos,
+            query_pos,
+        )
+
+
+def _get_clones(module, N):
+    return nn.ModuleList([copy.deepcopy(module) for i in range(N)])
+
+
+def _get_activation_fn(activation):
+    """Return an activation function given a string"""
+    if activation == "relu":
+        return F.relu
+    if activation == "gelu":
+        return F.gelu
+    if activation == "glu":
+        return F.glu
+    raise RuntimeError(f"activation should be relu/gelu, not {activation}.")
+
+
+def build_attn_mask(mask_type):
+    mask = torch.ones((151, 151), dtype=torch.bool).cuda()
+    if mask_type == "seperate_all":
+        mask[:50, :50] = False
+        mask[50:67, 50:67] = False
+        mask[67:84, 67:84] = False
+        mask[84:101, 84:101] = False
+        mask[101:151, 101:151] = False
+    elif mask_type == "seperate_view":
+        mask[:50, :50] = False
+        mask[50:67, 50:67] = False
+        mask[67:84, 67:84] = False
+        mask[84:101, 84:101] = False
+        mask[101:151, :] = False
+        mask[:, 101:151] = False
+    return mask
+# class InterfuserModel(nn.Module):
+
+class InterfuserModel(nn.Module):
+    def __init__(
+        self,
+        img_size=224,
+        multi_view_img_size=112,
+        patch_size=8,
+        in_chans=3,
+        embed_dim=768,
+        enc_depth=6,
+        dec_depth=6,
+        dim_feedforward=2048,
+        normalize_before=False,
+        rgb_backbone_name="r50",
+        lidar_backbone_name="r50",
+        num_heads=8,
+        norm_layer=None,
+        dropout=0.1,
+        end2end=False,
+        direct_concat=False,
+        separate_view_attention=False,
+        separate_all_attention=False,
+        act_layer=None,
+        weight_init="",
+        freeze_num=-1,
+        with_lidar=False,
+        with_right_left_sensors=False,
+        with_center_sensor=False,
+        traffic_pred_head_type="det",
+        waypoints_pred_head="heatmap",
+        reverse_pos=True,
+        use_different_backbone=False,
+        use_view_embed=False,
+        use_mmad_pretrain=None,
+    ):
+        super().__init__()
+        self.traffic_pred_head_type = traffic_pred_head_type
+        self.num_features = (
+            self.embed_dim
+        ) = embed_dim  # num_features for consistency with other models
+        norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
+        act_layer = act_layer or nn.GELU
+
+        self.reverse_pos = reverse_pos
+        self.waypoints_pred_head = waypoints_pred_head
+        self.with_lidar = with_lidar
+        self.with_right_left_sensors = with_right_left_sensors
+        self.with_center_sensor = with_center_sensor
+
+        self.direct_concat = direct_concat
+        self.separate_view_attention = separate_view_attention
+        self.separate_all_attention = separate_all_attention
+        self.end2end = end2end
+        self.use_view_embed = use_view_embed
+
+        if self.direct_concat:
+            in_chans = in_chans * 4
+            self.with_center_sensor = False
+            self.with_right_left_sensors = False
+
+        if self.separate_view_attention:
+            self.attn_mask = build_attn_mask("seperate_view")
+        elif self.separate_all_attention:
+            self.attn_mask = build_attn_mask("seperate_all")
+        else:
+            self.attn_mask = None
+
+        if use_different_backbone:
+            if rgb_backbone_name == "r50":
+                self.rgb_backbone = resnet50d(
+                    pretrained=True,
+                    in_chans=in_chans,
+                    features_only=True,
+                    out_indices=[4],
+                )
+            elif rgb_backbone_name == "r26":
+                self.rgb_backbone = resnet26d(
+                    pretrained=True,
+                    in_chans=in_chans,
+                    features_only=True,
+                    out_indices=[4],
+                )
+            elif rgb_backbone_name == "r18":
+                self.rgb_backbone = resnet18d(
+                    pretrained=True,
+                    in_chans=in_chans,
+                    features_only=True,
+                    out_indices=[4],
+                )
+            if lidar_backbone_name == "r50":
+                self.lidar_backbone = resnet50d(
+                    pretrained=False,
+                    in_chans=in_chans,
+                    features_only=True,
+                    out_indices=[4],
+                )
+            elif lidar_backbone_name == "r26":
+                self.lidar_backbone = resnet26d(
+                    pretrained=False,
+                    in_chans=in_chans,
+                    features_only=True,
+                    out_indices=[4],
+                )
+            elif lidar_backbone_name == "r18":
+                self.lidar_backbone = resnet18d(
+                    pretrained=False, in_chans=3, features_only=True, out_indices=[4]
+                )
+            rgb_embed_layer = partial(HybridEmbed, backbone=self.rgb_backbone)
+            lidar_embed_layer = partial(HybridEmbed, backbone=self.lidar_backbone)
+
+            if use_mmad_pretrain:
+                params = torch.load(use_mmad_pretrain)["state_dict"]
+                updated_params = OrderedDict()
+                for key in params:
+                    if "backbone" in key:
+                        updated_params[key.replace("backbone.", "")] = params[key]
+                self.rgb_backbone.load_state_dict(updated_params)
+
+            self.rgb_patch_embed = rgb_embed_layer(
+                img_size=img_size,
+                patch_size=patch_size,
+                in_chans=in_chans,
+                embed_dim=embed_dim,
+            )
+            self.lidar_patch_embed = lidar_embed_layer(
+                img_size=img_size,
+                patch_size=patch_size,
+                in_chans=3,
+                embed_dim=embed_dim,
+            )
+        else:
+            if rgb_backbone_name == "r50":
+                self.rgb_backbone = resnet50d(
+                    pretrained=True, in_chans=3, features_only=True, out_indices=[4]
+                )
+            elif rgb_backbone_name == "r101":
+                self.rgb_backbone = resnet101d(
+                    pretrained=True, in_chans=3, features_only=True, out_indices=[4]
+                )
+            elif rgb_backbone_name == "r26":
+                self.rgb_backbone = resnet26d(
+                    pretrained=True, in_chans=3, features_only=True, out_indices=[4]
+                )
+            elif rgb_backbone_name == "r18":
+                self.rgb_backbone = resnet18d(
+                    pretrained=True, in_chans=3, features_only=True, out_indices=[4]
+                )
+            embed_layer = partial(HybridEmbed, backbone=self.rgb_backbone)
+
+            self.rgb_patch_embed = embed_layer(
+                img_size=img_size,
+                patch_size=patch_size,
+                in_chans=in_chans,
+                embed_dim=embed_dim,
+            )
+            self.lidar_patch_embed = embed_layer(
+                img_size=img_size,
+                patch_size=patch_size,
+                in_chans=in_chans,
+                embed_dim=embed_dim,
+            )
+
+        self.global_embed = nn.Parameter(torch.zeros(1, embed_dim, 5))
+        self.view_embed = nn.Parameter(torch.zeros(1, embed_dim, 5, 1))
+
+        if self.end2end:
+            self.query_pos_embed = nn.Parameter(torch.zeros(1, embed_dim, 4))
+            self.query_embed = nn.Parameter(torch.zeros(4, 1, embed_dim))
+        elif self.waypoints_pred_head == "heatmap":
+            self.query_pos_embed = nn.Parameter(torch.zeros(1, embed_dim, 5))
+            self.query_embed = nn.Parameter(torch.zeros(400 + 5, 1, embed_dim))
+        else:
+            self.query_pos_embed = nn.Parameter(torch.zeros(1, embed_dim, 11))
+            self.query_embed = nn.Parameter(torch.zeros(400 + 11, 1, embed_dim))
+
+        if self.end2end:
+            self.waypoints_generator = GRUWaypointsPredictor(embed_dim, 4)
+        elif self.waypoints_pred_head == "heatmap":
+            self.waypoints_generator = MultiPath_Generator(
+                embed_dim + 32, embed_dim, 10
+            )
+        elif self.waypoints_pred_head == "gru":
+            self.waypoints_generator = GRUWaypointsPredictor(embed_dim)
+        elif self.waypoints_pred_head == "gru-command":
+            self.waypoints_generator = GRUWaypointsPredictorWithCommand(embed_dim)
+        elif self.waypoints_pred_head == "linear":
+            self.waypoints_generator = LinearWaypointsPredictor(embed_dim)
+        elif self.waypoints_pred_head == "linear-sum":
+            self.waypoints_generator = LinearWaypointsPredictor(embed_dim, cumsum=True)
+
+        self.junction_pred_head = nn.Linear(embed_dim, 2)
+        self.traffic_light_pred_head = nn.Linear(embed_dim, 2)
+        self.stop_sign_head = nn.Linear(embed_dim, 2)
+
+        if self.traffic_pred_head_type == "det":
+            self.traffic_pred_head = nn.Sequential(
+                *[
+                    nn.Linear(embed_dim + 32, 64),
+                    nn.ReLU(),
+                    nn.Linear(64, 7),
+                    # nn.Sigmoid(),
+                ]
+            )
+        elif self.traffic_pred_head_type == "seg":
+            self.traffic_pred_head = nn.Sequential(
+                *[nn.Linear(embed_dim, 64), nn.ReLU(), nn.Linear(64, 1), nn.Sigmoid()]
+            )
+
+        self.position_encoding = PositionEmbeddingSine(embed_dim // 2, normalize=True)
+
+        encoder_layer = TransformerEncoderLayer(
+            embed_dim, num_heads, dim_feedforward, dropout, act_layer, normalize_before
+        )
+        self.encoder = TransformerEncoder(encoder_layer, enc_depth, None)
+
+        decoder_layer = TransformerDecoderLayer(
+            embed_dim, num_heads, dim_feedforward, dropout, act_layer, normalize_before
+        )
+        decoder_norm = nn.LayerNorm(embed_dim)
+        self.decoder = TransformerDecoder(
+            decoder_layer, dec_depth, decoder_norm, return_intermediate=False
+        )
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        nn.init.uniform_(self.global_embed)
+        nn.init.uniform_(self.view_embed)
+        nn.init.uniform_(self.query_embed)
+        nn.init.uniform_(self.query_pos_embed)
+
+    def forward_features(
+        self,
+        front_image,
+        left_image,
+        right_image,
+        front_center_image,
+        lidar,
+        measurements,
+    ):
+        features = []
+
+        # Front view processing
+        front_image_token, front_image_token_global = self.rgb_patch_embed(front_image)
+        if self.use_view_embed:
+            front_image_token = (
+                front_image_token
+                + self.view_embed[:, :, 0:1, :]
+                + self.position_encoding(front_image_token)
+            )
+        else:
+            front_image_token = front_image_token + self.position_encoding(
+                front_image_token
+            )
+        front_image_token = front_image_token.flatten(2).permute(2, 0, 1)
+        front_image_token_global = (
+            front_image_token_global
+            + self.view_embed[:, :, 0, :]
+            + self.global_embed[:, :, 0:1]
+        )
+        front_image_token_global = front_image_token_global.permute(2, 0, 1)
+        features.extend([front_image_token, front_image_token_global])
+
+        if self.with_right_left_sensors:
+            # Left view processing
+            left_image_token, left_image_token_global = self.rgb_patch_embed(left_image)
+            if self.use_view_embed:
+                left_image_token = (
+                    left_image_token
+                    + self.view_embed[:, :, 1:2, :]
+                    + self.position_encoding(left_image_token)
+                )
+            else:
+                left_image_token = left_image_token + self.position_encoding(
+                    left_image_token
+                )
+            left_image_token = left_image_token.flatten(2).permute(2, 0, 1)
+            left_image_token_global = (
+                left_image_token_global
+                + self.view_embed[:, :, 1, :]
+                + self.global_embed[:, :, 1:2]
+            )
+            left_image_token_global = left_image_token_global.permute(2, 0, 1)
+
+            # Right view processing
+            right_image_token, right_image_token_global = self.rgb_patch_embed(
+                right_image
+            )
+            if self.use_view_embed:
+                right_image_token = (
+                    right_image_token
+                    + self.view_embed[:, :, 2:3, :]
+                    + self.position_encoding(right_image_token)
+                )
+            else:
+                right_image_token = right_image_token + self.position_encoding(
+                    right_image_token
+                )
+            right_image_token = right_image_token.flatten(2).permute(2, 0, 1)
+            right_image_token_global = (
+                right_image_token_global
+                + self.view_embed[:, :, 2, :]
+                + self.global_embed[:, :, 2:3]
+            )
+            right_image_token_global = right_image_token_global.permute(2, 0, 1)
+
+            features.extend(
+                [
+                    left_image_token,
+                    left_image_token_global,
+                    right_image_token,
+                    right_image_token_global,
+                ]
+            )
+
+        if self.with_center_sensor:
+            # Front center view processing
+            (
+                front_center_image_token,
+                front_center_image_token_global,
+            ) = self.rgb_patch_embed(front_center_image)
+            if self.use_view_embed:
+                front_center_image_token = (
+                    front_center_image_token
+                    + self.view_embed[:, :, 3:4, :]
+                    + self.position_encoding(front_center_image_token)
+                )
+            else:
+                front_center_image_token = (
+                    front_center_image_token
+                    + self.position_encoding(front_center_image_token)
+                )
+
+            front_center_image_token = front_center_image_token.flatten(2).permute(
+                2, 0, 1
+            )
+            front_center_image_token_global = (
+                front_center_image_token_global
+                + self.view_embed[:, :, 3, :]
+                + self.global_embed[:, :, 3:4]
+            )
+            front_center_image_token_global = front_center_image_token_global.permute(
+                2, 0, 1
+            )
+            features.extend([front_center_image_token, front_center_image_token_global])
+
+        if self.with_lidar:
+            lidar_token, lidar_token_global = self.lidar_patch_embed(lidar)
+            if self.use_view_embed:
+                lidar_token = (
+                    lidar_token
+                    + self.view_embed[:, :, 4:5, :]
+                    + self.position_encoding(lidar_token)
+                )
+            else:
+                lidar_token = lidar_token + self.position_encoding(lidar_token)
+            lidar_token = lidar_token.flatten(2).permute(2, 0, 1)
+            lidar_token_global = (
+                lidar_token_global
+                + self.view_embed[:, :, 4, :]
+                + self.global_embed[:, :, 4:5]
+            )
+            lidar_token_global = lidar_token_global.permute(2, 0, 1)
+            features.extend([lidar_token, lidar_token_global])
+
+        features = torch.cat(features, 0)
+        return features
+
+    def forward(self, x):
+        front_image = x["rgb"]
+        left_image = x["rgb_left"]
+        right_image = x["rgb_right"]
+        front_center_image = x["rgb_center"]
+        measurements = x["measurements"]
+        target_point = x["target_point"]
+        lidar = x["lidar"]
+
+        if self.direct_concat:
+            img_size = front_image.shape[-1]
+            left_image = torch.nn.functional.interpolate(
+                left_image, size=(img_size, img_size)
+            )
+            right_image = torch.nn.functional.interpolate(
+                right_image, size=(img_size, img_size)
+            )
+            front_center_image = torch.nn.functional.interpolate(
+                front_center_image, size=(img_size, img_size)
+            )
+            front_image = torch.cat(
+                [front_image, left_image, right_image, front_center_image], dim=1
+            )
+        features = self.forward_features(
+            front_image,
+            left_image,
+            right_image,
+            front_center_image,
+            lidar,
+            measurements,
+        )
+
+        bs = front_image.shape[0]
+
+        if self.end2end:
+            tgt = self.query_pos_embed.repeat(bs, 1, 1)
+        else:
+            tgt = self.position_encoding(
+                torch.ones((bs, 1, 20, 20), device=x["rgb"].device)
+            )
+            tgt = tgt.flatten(2)
+            tgt = torch.cat([tgt, self.query_pos_embed.repeat(bs, 1, 1)], 2)
+        tgt = tgt.permute(2, 0, 1)
+
+        memory = self.encoder(features, mask=self.attn_mask)
+        hs = self.decoder(self.query_embed.repeat(1, bs, 1), memory, query_pos=tgt)[0]
+
+        hs = hs.permute(1, 0, 2)  # Batchsize ,  N, C
+        if self.end2end:
+            waypoints = self.waypoints_generator(hs, target_point)
+            return waypoints
+
+        if self.waypoints_pred_head != "heatmap":
+            traffic_feature = hs[:, :400]
+            is_junction_feature = hs[:, 400]
+            traffic_light_state_feature = hs[:, 400]
+            stop_sign_feature = hs[:, 400]
+            waypoints_feature = hs[:, 401:411]
+        else:
+            traffic_feature = hs[:, :400]
+            is_junction_feature = hs[:, 400]
+            traffic_light_state_feature = hs[:, 400]
+            stop_sign_feature = hs[:, 400]
+            waypoints_feature = hs[:, 401:405]
+
+        if self.waypoints_pred_head == "heatmap":
+            waypoints = self.waypoints_generator(waypoints_feature, measurements)
+        elif self.waypoints_pred_head == "gru":
+            waypoints = self.waypoints_generator(waypoints_feature, target_point)
+        elif self.waypoints_pred_head == "gru-command":
+            waypoints = self.waypoints_generator(waypoints_feature, target_point, measurements)
+        elif self.waypoints_pred_head == "linear":
+            waypoints = self.waypoints_generator(waypoints_feature, measurements)
+        elif self.waypoints_pred_head == "linear-sum":
+            waypoints = self.waypoints_generator(waypoints_feature, measurements)
+
+        is_junction = self.junction_pred_head(is_junction_feature)
+        traffic_light_state = self.traffic_light_pred_head(traffic_light_state_feature)
+        stop_sign = self.stop_sign_head(stop_sign_feature)
+
+        velocity = measurements[:, 6:7].unsqueeze(-1)
+        velocity = velocity.repeat(1, 400, 32)
+        traffic_feature_with_vel = torch.cat([traffic_feature, velocity], dim=2)
+        traffic = self.traffic_pred_head(traffic_feature_with_vel)
+        return traffic, waypoints, is_junction, traffic_light_state, stop_sign, traffic_feature
+    def load_pretrained(self, model_path, strict=False):
+        """
+        تحميل الأ��زان المدربة مسبقاً - نسخة محسنة
+        
+        Args:
+            model_path (str): مسار ملف الأوزان
+            strict (bool): إذا كان True، يتطلب تطابق تام للمفاتيح
+        """
+        if not model_path or not Path(model_path).exists():
+            logging.warning(f"ملف الأوزان غير موجود: {model_path}")
+            logging.info("سيتم استخدام أوزان عشوائية")
+            return False
+            
+        try:
+            logging.info(f"محاولة تحميل الأوزان من: {model_path}")
+            
+            # تحميل الملف مع معالجة أنواع مختلفة من ملفات الحفظ
+            checkpoint = torch.load(model_path, map_location='cpu', weights_only=False)
+            
+            # استخراج state_dict من أنواع مختلفة من ملفات الحفظ
+            if isinstance(checkpoint, dict):
+                if 'model_state_dict' in checkpoint:
+                    state_dict = checkpoint['model_state_dict']
+                    logging.info("تم العثور على 'model_state_dict' في الملف")
+                elif 'state_dict' in checkpoint:
+                    state_dict = checkpoint['state_dict']
+                    logging.info("تم العثور على 'state_dict' في الملف")
+                elif 'model' in checkpoint:
+                    state_dict = checkpoint['model']
+                    logging.info("تم العثور على 'model' في الملف")
+                else:
+                    state_dict = checkpoint
+                    logging.info("استخدام الملف كـ state_dict مباشرة")
+            else:
+                state_dict = checkpoint
+                logging.info("استخدام الملف كـ state_dict مباشرة")
+            
+            # تنظيف أسماء المفاتيح (إزالة 'module.' إذا كانت موجودة)
+            clean_state_dict = OrderedDict()
+            for k, v in state_dict.items():
+                # إزالة 'module.' من بداية اسم المفتاح إذا كان موجوداً
+                clean_key = k[7:] if k.startswith('module.') else k
+                clean_state_dict[clean_key] = v
+            
+            # تحميل الأوزان
+            missing_keys, unexpected_keys = self.load_state_dict(clean_state_dict, strict=strict)
+            
+            # تقرير حالة التحميل
+            if missing_keys:
+                logging.warning(f"مفاتيح مفقودة ({len(missing_keys)}): {missing_keys[:5]}..." if len(missing_keys) > 5 else f"مفاتيح مفقودة: {missing_keys}")
+            
+            if unexpected_keys:
+                logging.warning(f"مفاتيح غير متوقعة ({len(unexpected_keys)}): {unexpected_keys[:5]}..." if len(unexpected_keys) > 5 else f"مفاتيح غير متوقعة: {unexpected_keys}")
+            
+            if not missing_keys and not unexpected_keys:
+                logging.info("✅ تم تحميل جميع الأوزان بنجاح تام")
+            elif not strict:
+                logging.info("✅ تم تحميل الأوزان بنجاح (مع تجاهل عدم التطابق)")
+            
+            return True
+            
+        except Exception as e:
+            logging.error(f"❌ خطأ في تحميل الأوزان: {str(e)}")
+            logging.info("سيتم استخدام أوزان عشوائية")
+            return False
+
+
+# ============================================================================
+# دوال مساعدة لتحميل النموذج
+# ============================================================================
+
+def load_and_prepare_model(config, device):
+    """
+    يقوم بإنشاء النموذج وتحميل الأوزان المدربة مسبقًا.
+    
+    Args:
+        config (dict): إعدادات النموذج والمسارات
+        device (torch.device): الجهاز المستهدف (CPU/GPU)
+    
+    Returns:
+        InterfuserModel: النموذج المحمل
+    """
+    try:
+        # إنشاء النموذج
+        model = InterfuserModel(**config.get('model_params', {})).to(device)
+        logging.info(f"تم إنشاء النموذج على الجهاز: {device}")
+        
+        # تحميل الأوزان إذا كان المسار محدد
+        checkpoint_path = config.get('paths', {}).get('pretrained_weights')
+        if checkpoint_path:
+            success = model.load_pretrained(checkpoint_path, strict=False)
+            if success:
+                logging.info("✅ تم تحميل النموذج والأوزان بنجاح")
+            else:
+                logging.warning("⚠️ تم إنشاء النموذج بأوزان عشوائية")
+        else:
+            logging.info("لم يتم تحديد مسار الأوزان، سيتم استخدام أوزان عشوائية")
+        
+        # وضع النموذج في وضع التقييم
+        model.eval()
+        
+        return model
+        
+    except Exception as e:
+        logging.error(f"خطأ في إنشاء النموذج: {str(e)}")
+        raise
+
+
+def create_model_config(model_path="model/best_model.pth", **model_params):
+    """
+    إنشاء إعدادات النموذج باستخدام الإعدادات الصحيحة من التدريب
+    
+    Args:
+        model_path (str): مسار ملف الأوزان
+        **model_params: معاملات النموذج الإضافية
+    
+    Returns:
+        dict: إعدادات النموذج
+    """
+    # الإعدادات الصحيحة من كونفيج التدريب الأصلي
+    training_config_params = {
+        "img_size": 224,
+        "embed_dim": 256,  # مهم: هذه القيمة من التدريب الأصلي
+        "enc_depth": 6,
+        "dec_depth": 6,
+        "rgb_backbone_name": 'r50',
+        "lidar_backbone_name": 'r18',
+        "waypoints_pred_head": 'gru',
+        "use_different_backbone": True,
+        "with_lidar": False,
+        "with_right_left_sensors": False,
+        "with_center_sensor": False,
+        
+        # إعدادات إضافية من الكونفيج الأصلي
+        "multi_view_img_size": 112,
+        "patch_size": 8,
+        "in_chans": 3,
+        "dim_feedforward": 2048,
+        "normalize_before": False,
+        "num_heads": 8,
+        "dropout": 0.1,
+        "end2end": False,
+        "direct_concat": False,
+        "separate_view_attention": False,
+        "separate_all_attention": False,
+        "freeze_num": -1,
+        "traffic_pred_head_type": "det",
+        "reverse_pos": True,
+        "use_view_embed": False,
+        "use_mmad_pretrain": None,
+    }
+    
+    # دمج المعاملات المخصصة مع الإعدادات من التدريب
+    training_config_params.update(model_params)
+    
+    config = {
+        'model_params': training_config_params,
+        'paths': {
+            'pretrained_weights': model_path
+        },
+        
+        # إضافة إعدادات الشبكة من التدريب
+        'grid_conf': {
+            'h': 20, 'w': 20,
+            'x_res': 1.0, 'y_res': 1.0,
+            'y_min': 0.0, 'y_max': 20.0,
+            'x_min': -10.0, 'x_max': 10.0,
+        },
+        
+        # معلومات إضافية عن التدريب
+        'training_info': {
+            'original_project': 'Interfuser_Finetuning',
+            'run_name': 'Finetune_Focus_on_Detection_v5',
+            'focus': 'traffic_detection_and_iou',
+            'backbone': 'ResNet50 + ResNet18',
+            'trained_on': 'PDM_Lite_Carla'
+        }
+    }
+    
+    return config
+
+
+def get_training_config():
+    """
+    إرجاع إعدادات التدريب الأصلية للمرجع
+    هذه الإعدادات توضح كيف تم تدريب النموذج
+    """
+    return {
+        'project_info': {
+            'project': 'Interfuser_Finetuning',
+            'entity': None,
+            'run_name': 'Finetune_Focus_on_Detection_v5'
+        },
+        'training': {
+            'epochs': 50,
+            'batch_size': 8,
+            'num_workers': 2,
+            'learning_rate': 1e-4,  # معدل تعلم منخفض للـ Fine-tuning
+            'weight_decay': 1e-2,
+            'patience': 15,
+            'clip_grad_norm': 1.0,
+        },
+        'loss_weights': {
+            'iou': 2.0,              # أولوية قصوى لدقة الصناديق
+            'traffic_map': 25.0,     # تركيز عالي على اكتشاف الكائنات
+            'waypoints': 1.0,        # مرجع أساسي
+            'junction': 0.25,        # مهام متقنة بالفعل
+            'traffic_light': 0.5,
+            'stop_sign': 0.25,
+        },
+        'data_split': {
+            'strategy': 'interleaved',
+            'segment_length': 100,
+            'validation_frequency': 10,
+        },
+        'transforms': {
+            'use_data_augmentation': False,  # معطل للتركيز على البيانات الأصلية
+        }
+    }

requirements.txt

@@ -0,0 +1,22 @@
+# مكتبات الخادم الأساسية
+fastapi==0.104.1
+uvicorn[standard]==0.24.0
+python-multipart==0.0.6
+pydantic==2.4.2
+
+# مكتبات التعلم العميق
+torch==2.1.0 --extra-index-url https://download.pytorch.org/whl/cpu
+torchvision==0.16.0 --extra-index-url https://download.pytorch.org/whl/cpu
+
+# مكتبات معالجة البيانات
+numpy==1.24.3
+opencv-python-headless==4.8.1.78
+Pillow==10.0.1
+
+# مكتبات إضافية للنموذج المحسن
+tqdm==4.66.1
+pathlib2==2.3.7
+
+# مكتبات اختيارية (يمكن تثبيتها حسب الحاجة)
+# wandb==0.15.12  # للمراقبة والتتبع
+# timm==0.9.7     # لنماذج الرؤية الحاسوبية

simulation_modules.py

@@ -0,0 +1,336 @@
+# simulation_modules.py
+
+import torch
+import numpy as np
+import cv2
+import math
+from collections import deque
+from typing import List, Tuple, Dict, Any, Optional
+
+# ================== Constants ==================
+WAYPOINT_SCALE_FACTOR = 5.0
+T1_FUTURE_TIME = 1.0
+T2_FUTURE_TIME = 2.0
+PIXELS_PER_METER = 8
+MAX_DISTANCE = 32
+IMG_SIZE = MAX_DISTANCE * PIXELS_PER_METER * 2
+EGO_CAR_X = IMG_SIZE // 2
+EGO_CAR_Y = IMG_SIZE - (4.0 * PIXELS_PER_METER)
+
+COLORS = {
+    'vehicle': [255, 0, 0],
+    'pedestrian': [0, 255, 0],
+    'cyclist': [0, 0, 255],
+    'waypoint': [255, 255, 0],
+    'ego_car': [255, 255, 255]
+}
+
+# ================== PID Controller ==================
+class PIDController:
+    def __init__(self, K_P=1.0, K_I=0.0, K_D=0.0, n=20):
+        self._K_P = K_P
+        self._K_I = K_I
+        self._K_D = K_D
+        self._window = deque([0 for _ in range(n)], maxlen=n)
+
+    def step(self, error):
+        self._window.append(error)
+        if len(self._window) >= 2:
+            integral = np.mean(self._window)
+            derivative = self._window[-1] - self._window[-2]
+        else:
+            integral = derivative = 0.0
+        return self._K_P * error + self._K_I * integral + self._K_D * derivative
+
+# ================== Helper Functions ==================
+def ensure_rgb(image):
+    if len(image.shape) == 2:
+        return cv2.cvtColor(image, cv2.COLOR_GRAY2BGR)
+    elif image.shape[2] == 1:
+        return cv2.cvtColor(image, cv2.COLOR_GRAY2BGR)
+    return image
+
+def add_rect(img, loc, ori, box, value, color):
+    center_x = int(loc[0] * PIXELS_PER_METER + MAX_DISTANCE * PIXELS_PER_METER)
+    center_y = int(loc[1] * PIXELS_PER_METER + MAX_DISTANCE * PIXELS_PER_METER)
+    size_px = (int(box[0] * PIXELS_PER_METER), int(box[1] * PIXELS_PER_METER))
+    angle_deg = -np.degrees(math.atan2(ori[1], ori[0]))
+    box_points = cv2.boxPoints(((center_x, center_y), size_px, angle_deg))
+    box_points = np.int32(box_points)
+    adjusted_color = [int(c * value) for c in color]
+    cv2.fillConvexPoly(img, box_points, adjusted_color)
+    return img
+
+def render(traffic_grid, t=0):
+    img = np.zeros((IMG_SIZE, IMG_SIZE, 3), dtype=np.uint8)
+    counts = {'vehicles': 0, 'pedestrians': 0, 'cyclists': 0}
+    
+    if isinstance(traffic_grid, torch.Tensor):
+        traffic_grid = traffic_grid.cpu().numpy()
+    
+    h, w, c = traffic_grid.shape
+    for y in range(h):
+        for x in range(w):
+            for ch in range(c):
+                if traffic_grid[y, x, ch] > 0.1:
+                    world_x = (x / w - 0.5) * MAX_DISTANCE * 2
+                    world_y = (y / h - 0.5) * MAX_DISTANCE * 2
+                    
+                    if ch < 3:
+                        color = COLORS['vehicle']
+                        counts['vehicles'] += 1
+                        box_size = [2.0, 4.0]
+                    elif ch < 5:
+                        color = COLORS['pedestrian']
+                        counts['pedestrians'] += 1
+                        box_size = [0.8, 0.8]
+                    else:
+                        color = COLORS['cyclist']
+                        counts['cyclists'] += 1
+                        box_size = [1.2, 2.0]
+                    
+                    img = add_rect(img, [world_x, world_y], [1.0, 0.0], 
+                                 box_size, traffic_grid[y, x, ch], color)
+    
+    return img, counts
+
+def render_waypoints(waypoints, scale_factor=WAYPOINT_SCALE_FACTOR):
+    img = np.zeros((IMG_SIZE, IMG_SIZE, 3), dtype=np.uint8)
+    
+    if isinstance(waypoints, torch.Tensor):
+        waypoints = waypoints.cpu().numpy()
+    
+    scaled_waypoints = waypoints * scale_factor
+    
+    for i, wp in enumerate(scaled_waypoints):
+        px = int(wp[0] * PIXELS_PER_METER + IMG_SIZE // 2)
+        py = int(wp[1] * PIXELS_PER_METER + IMG_SIZE // 2)
+        
+        if 0 <= px < IMG_SIZE and 0 <= py < IMG_SIZE:
+            radius = max(3, 8 - i)
+            cv2.circle(img, (px, py), radius, COLORS['waypoint'], -1)
+            
+            if i > 0:
+                prev_px = int(scaled_waypoints[i-1][0] * PIXELS_PER_METER + IMG_SIZE // 2)
+                prev_py = int(scaled_waypoints[i-1][1] * PIXELS_PER_METER + IMG_SIZE // 2)
+                if 0 <= prev_px < IMG_SIZE and 0 <= prev_py < IMG_SIZE:
+                    cv2.line(img, (prev_px, prev_py), (px, py), COLORS['waypoint'], 2)
+    
+    return img
+
+def render_self_car(img):
+    car_pos = [0, -4.0]
+    car_ori = [1.0, 0.0]
+    car_size = [2.0, 4.5]
+    return add_rect(img, car_pos, car_ori, car_size, 1.0, COLORS['ego_car'])
+
+# ================== Tracker Classes ==================
+class TrackedObject:
+    def __init__(self, obj_id: int):
+        self.id = obj_id
+        self.last_step = 0
+        self.last_pos = [0.0, 0.0]
+        self.historical_pos = []
+        self.historical_steps = []
+        self.velocity = [0.0, 0.0]
+        self.confidence = 1.0
+
+    def update(self, step: int, obj_info: List[float]):
+        self.last_step = step
+        self.last_pos = obj_info[:2]
+        self.historical_pos.append(obj_info[:2])
+        self.historical_steps.append(step)
+        
+        if len(self.historical_pos) >= 2:
+            dt = self.historical_steps[-1] - self.historical_steps[-2]
+            if dt > 0:
+                dx = self.historical_pos[-1][0] - self.historical_pos[-2][0]
+                dy = self.historical_pos[-1][1] - self.historical_pos[-2][1]
+                self.velocity = [dx/dt, dy/dt]
+
+    def predict_position(self, future_time: float) -> List[float]:
+        predicted_x = self.last_pos[0] + self.velocity[0] * future_time
+        predicted_y = self.last_pos[1] + self.velocity[1] * future_time
+        return [predicted_x, predicted_y]
+
+    def is_alive(self, current_step: int, max_age: int = 5) -> bool:
+        return (current_step - self.last_step) <= max_age
+
+class Tracker:
+    def __init__(self, frequency: int = 10):
+        self.tracks: List[TrackedObject] = []
+        self.frequency = frequency
+        self.next_id = 0
+        self.current_step = 0
+
+    def update_and_predict(self, detections: List[Dict], step: int) -> np.ndarray:
+        self.current_step = step
+        
+        for detection in detections:
+            pos = detection.get('position', [0, 0])
+            feature = detection.get('feature', 0.5)
+            
+            best_match = None
+            min_distance = float('inf')
+            
+            for track in self.tracks:
+                if track.is_alive(step):
+                    distance = np.linalg.norm(np.array(pos) - np.array(track.last_pos))
+                    if distance < min_distance and distance < 2.0:
+                        min_distance = distance
+                        best_match = track
+            
+            if best_match:
+                best_match.update(step, pos + [feature])
+            else:
+                new_track = TrackedObject(self.next_id)
+                new_track.update(step, pos + [feature])
+                self.tracks.append(new_track)
+                self.next_id += 1
+        
+        self.tracks = [t for t in self.tracks if t.is_alive(step)]
+        return self._generate_prediction_grid()
+
+    def _generate_prediction_grid(self) -> np.ndarray:
+        grid = np.zeros((20, 20, 7), dtype=np.float32)
+        
+        for track in self.tracks:
+            if track.is_alive(self.current_step):
+                current_pos = track.last_pos
+                future_pos_t1 = track.predict_position(T1_FUTURE_TIME)
+                future_pos_t2 = track.predict_position(T2_FUTURE_TIME)
+                
+                for pos in [current_pos, future_pos_t1, future_pos_t2]:
+                    grid_x = int((pos[0] / (MAX_DISTANCE * 2) + 0.5) * 20)
+                    grid_y = int((pos[1] / (MAX_DISTANCE * 2) + 0.5) * 20)
+                    
+                    if 0 <= grid_x < 20 and 0 <= grid_y < 20:
+                        channel = 0
+                        grid[grid_y, grid_x, channel] = max(grid[grid_y, grid_x, channel], track.confidence)
+        
+        return grid
+
+# ================== Controller Classes ==================
+class ControllerConfig:
+    def __init__(self):
+        self.turn_KP = 1.0
+        self.turn_KI = 0.1
+        self.turn_KD = 0.1
+        self.turn_n = 20
+        
+        self.speed_KP = 0.5
+        self.speed_KI = 0.05
+        self.speed_KD = 0.1
+        self.speed_n = 20
+        
+        self.max_speed = 6.0
+        self.max_throttle = 0.75
+        self.clip_delta = 0.25
+        
+        self.brake_speed = 0.4
+        self.brake_ratio = 1.1
+
+class InterfuserController:
+    def __init__(self, config: ControllerConfig):
+        self.config = config
+        self.turn_controller = PIDController(config.turn_KP, config.turn_KI, config.turn_KD, config.turn_n)
+        self.speed_controller = PIDController(config.speed_KP, config.speed_KI, config.speed_KD, config.speed_n)
+        self.last_steer = 0.0
+        self.last_throttle = 0.0
+        self.target_speed = 3.0
+
+    def run_step(self, current_speed: float, waypoints: np.ndarray, 
+                 junction: float, traffic_light_state: float, 
+                 stop_sign: float, meta_data: Dict) -> Tuple[float, float, bool, str]:
+        
+        if isinstance(waypoints, torch.Tensor):
+            waypoints = waypoints.cpu().numpy()
+        
+        if len(waypoints) > 1:
+            dx = waypoints[1][0] - waypoints[0][0]
+            dy = waypoints[1][1] - waypoints[0][1]
+            target_yaw = math.atan2(dy, dx)
+            steer = self.turn_controller.step(target_yaw)
+        else:
+            steer = 0.0
+        
+        steer = np.clip(steer, -1.0, 1.0)
+        
+        target_speed = self.target_speed
+        if junction > 0.5:
+            target_speed *= 0.7
+        if abs(steer) > 0.3:
+            target_speed *= 0.8
+        
+        speed_error = target_speed - current_speed
+        throttle = self.speed_controller.step(speed_error)
+        throttle = np.clip(throttle, 0.0, self.config.max_throttle)
+        
+        brake = False
+        if traffic_light_state > 0.5 or stop_sign > 0.5 or current_speed > self.config.max_speed:
+            brake = True
+            throttle = 0.0
+        
+        self.last_steer = steer
+        self.last_throttle = throttle
+        
+        metadata = f"Speed:{current_speed:.1f} Target:{target_speed:.1f} Junction:{junction:.2f}"
+        
+        return steer, throttle, brake, metadata
+
+# ================== Display Interface ==================
+class DisplayInterface:
+    def __init__(self, width: int = 1200, height: int = 600):
+        self._width = width
+        self._height = height
+        self.camera_width = width // 2
+        self.camera_height = height
+        self.map_width = width // 2
+        self.map_height = height // 3
+        
+    def run_interface(self, data: Dict[str, Any]) -> np.ndarray:
+        dashboard = np.zeros((self._height, self._width, 3), dtype=np.uint8)
+        
+        # Camera view
+        camera_view = data.get('camera_view')
+        if camera_view is not None:
+            camera_resized = cv2.resize(camera_view, (self.camera_width, self.camera_height))
+            dashboard[:, :self.camera_width] = camera_resized
+        
+        # Maps
+        map_start_x = self.camera_width
+        
+        map_t0 = data.get('map_t0')
+        if map_t0 is not None:
+            map_resized = cv2.resize(map_t0, (self.map_width, self.map_height))
+            dashboard[:self.map_height, map_start_x:] = map_resized
+            cv2.putText(dashboard, "Current (t=0)", (map_start_x + 10, 30), 
+                       cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 255), 2)
+        
+        map_t1 = data.get('map_t1')
+        if map_t1 is not None:
+            map_resized = cv2.resize(map_t1, (self.map_width, self.map_height))
+            y_start = self.map_height
+            dashboard[y_start:y_start + self.map_height, map_start_x:] = map_resized
+            cv2.putText(dashboard, f"Future (t={T1_FUTURE_TIME}s)", 
+                       (map_start_x + 10, y_start + 30), cv2.FONT_HERSHEY_SIMPLEX, 
+                       0.7, (255, 255, 255), 2)
+        
+        map_t2 = data.get('map_t2')
+        if map_t2 is not None:
+            map_resized = cv2.resize(map_t2, (self.map_width, self.map_height))
+            y_start = self.map_height * 2
+            dashboard[y_start:, map_start_x:] = map_resized
+            cv2.putText(dashboard, f"Future (t={T2_FUTURE_TIME}s)", 
+                       (map_start_x + 10, y_start + 30), cv2.FONT_HERSHEY_SIMPLEX, 
+                       0.7, (255, 255, 255), 2)
+        
+        # Text info
+        text_info = data.get('text_info', {})
+        y_offset = 50
+        for key, value in text_info.items():
+            cv2.putText(dashboard, value, (10, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 
+                       0.6, (0, 255, 0), 2)
+            y_offset += 30
+        
+        return dashboard