submission-fix (#1039)

- Use uv for dependencies and runtime (2620acfe2080a0b2b8b6139e9f1d1eba6ea3e0f7)
- Switch to uv in dev Dockerfile (b91f2b791918c02db1b7635d21b7dbde284938ba)
- Set logging level to WARNING (f6dd42cae1290960ee07f7caf9e8d20aa8eec6b1)
- Refactor model validation logic (0e60add1c9df5f43c2e3e3b920e5fcb0e58bd5f1)
- Improve model size calculation (9469eaec18aff6a0e88bc9361fb6bd6b637ac706)
- Replace Poetry with uv (d27998844b63b2eacb0fa32114cd7079c20d6639)
- Update backend to use uv (28fd56c9b310ceae547022f9d21d9375fa592e90)
- Fix weightsType casing (7f1a54e79105b3af55a31b7aa3160e222381e486)
- Minor changes (581b4b19243fa8975439b49ab07f8da8a88a1bb6)
- Correct text (deb47717036353ee8a8340b2ff28f6479580b72d)

Dockerfile

@@ -8,23 +8,22 @@ COPY frontend/ ./
 RUN npm run build
 
 # Build backend
-FROM python:3.9-slim
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim
+
 WORKDIR /app
 
 # Create non-root user
 RUN useradd -m -u 1000 user
 
-# Install poetry
-RUN pip install poetry
-
 # Create and configure cache directory
 RUN mkdir -p /app/.cache && \
     chown -R user:user /app
 
-# Copy and install backend dependencies
-COPY backend/pyproject.toml backend/poetry.lock* ./
-RUN poetry config virtualenvs.create false \
-    && poetry install --no-interaction --no-ansi --no-root --only main
+# Copy uv configuration files
+COPY backend/pyproject.toml backend/uv.lock ./
+
+# Install dependencies using uv
+RUN uv sync  --all-extras --frozen
 
 # Copy backend code
 COPY backend/ .
@@ -60,4 +59,4 @@ USER user
 EXPOSE 7860
 
 # Start both servers with wait-for
-CMD ["sh", "-c", "uvicorn app.asgi:app --host 0.0.0.0 --port 7861 & while ! nc -z localhost 7861; do sleep 1; done && cd frontend && npm run serve"]
+CMD ["sh", "-c", "uv run uvicorn app.asgi:app --host 0.0.0.0 --port 7861 & while ! nc -z localhost 7861; do sleep 1; done && cd frontend && npm run serve"]

backend/Dockerfile.dev

@@ -1,25 +1,35 @@
-FROM python:3.9-slim
+# Use a Python image with uv pre-installed
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim
 
+# Set the working directory
 WORKDIR /app
 
+# Enable bytecode compilation
+ENV UV_COMPILE_BYTECODE=1
+
+# Copy from the cache instead of linking since it's a mounted volume
+ENV UV_LINK_MODE=copy
+
+# Environment variables configuration for logs
+ENV PYTHONUNBUFFERED=1
+ENV LOG_LEVEL=INFO
+
 # Install required system dependencies
 RUN apt-get update && apt-get install -y \
     build-essential \
     && rm -rf /var/lib/apt/lists/*
 
-# Install poetry
-RUN pip install poetry
+# Copy uv configuration files
+COPY pyproject.toml uv.lock ./
 
-# Copy Poetry configuration files
-COPY pyproject.toml poetry.lock* ./
+# Install dependencies using uv
+RUN uv sync --frozen --no-install-project --no-dev
 
-# Install dependencies
-RUN poetry config virtualenvs.create false && \
-    poetry install --no-interaction --no-ansi --no-root
+# Place executables in the environment at the front of the path
+ENV PATH="/app/.venv/bin:$PATH"
 
-# Environment variables configuration for logs
-ENV PYTHONUNBUFFERED=1
-ENV LOG_LEVEL=INFO
+# Reset the entrypoint, don't invoke `uv`
+ENTRYPOINT []
 
 # In dev, mount volume directly
-CMD ["uvicorn", "app.asgi:app", "--host", "0.0.0.0", "--port", "7860", "--reload", "--log-level", "warning", "--no-access-log"]
+CMD ["uv" "run" "uvicorn", "app.asgi:app", "--host", "0.0.0.0", "--port", "7860", "--reload", "--log-level", "warning", "--no-access-log"]

backend/app/asgi.py

@@ -44,18 +44,18 @@ LOGGING_CONFIG = {
         },
         "uvicorn.access": {
             "handlers": ["default"],
-            "level": "INFO",
+            "level": "WARNING",
             "propagate": False,
         },
         "app": {
             "handlers": ["default"],
-            "level": "INFO",
+            "level": "WARNING",
             "propagate": False,
         }
     },
     "root": {
         "handlers": ["default"],
-        "level": "INFO",
+        "level": "WARNING",
     }
 }
 

backend/app/services/models.py

@@ -382,26 +382,6 @@ class ModelService(HuggingFaceService):
             if field not in model_data:
                 raise ValueError(f"Missing required field: {field}")
 
-        # Check if model already exists in the system
-        try:
-            logger.info(LogFormatter.subsection("CHECKING EXISTING SUBMISSIONS"))
-            existing_models = await self.get_models()
-            
-            # Check in all statuses (pending, evaluating, finished)
-            for status, models in existing_models.items():
-                for model in models:
-                    if model["name"] == model_data["model_id"]:
-                        error_msg = f"Model {model_data['model_id']} is already in the system with status: {status}"
-                        logger.error(LogFormatter.error("Submission rejected", error_msg))
-                        raise ValueError(error_msg)
-            
-            logger.info(LogFormatter.success("No existing submission found"))
-        except ValueError:
-            raise
-        except Exception as e:
-            logger.error(LogFormatter.error("Failed to check existing submissions", e))
-            raise
-
         # Get model info and validate it exists on HuggingFace
         try:
             logger.info(LogFormatter.subsection("MODEL VALIDATION"))
@@ -412,6 +392,7 @@ class ModelService(HuggingFaceService):
                 revision=model_data["revision"],
                 token=self.token
             )
+            
             if not model_info:
                 raise Exception(f"Model {model_data['model_id']} not found on HuggingFace Hub")
             
@@ -420,6 +401,29 @@ class ModelService(HuggingFaceService):
         except Exception as e:
             logger.error(LogFormatter.error("Model validation failed", e))
             raise
+        
+        # Update model revision with commit sha
+        model_data["revision"] = model_info.sha
+
+        # Check if model already exists in the system
+        try:
+            logger.info(LogFormatter.subsection("CHECKING EXISTING SUBMISSIONS"))
+            existing_models = await self.get_models()
+            
+            # Check in all statuses (pending, evaluating, finished)
+            for status, models in existing_models.items():
+                for model in models:
+                    if model["name"] == model_data["model_id"] and model["revision"] == model_data["revision"]:
+                        error_msg = f"Model {model_data['model_id']} revision {model_data["revision"]} is already in the system with status: {status}"
+                        logger.error(LogFormatter.error("Submission rejected", error_msg))
+                        raise ValueError(error_msg)
+            
+            logger.info(LogFormatter.success("No existing submission found"))
+        except ValueError:
+            raise
+        except Exception as e:
+            logger.error(LogFormatter.error("Failed to check existing submissions", e))
+            raise
 
         # Validate model card
         valid, error, model_card = await self.validator.check_model_card(
@@ -434,7 +438,8 @@ class ModelService(HuggingFaceService):
         model_size, error = await self.validator.get_model_size(
             model_info,
             model_data["precision"],
-            model_data["base_model"]
+            model_data["base_model"],
+            revision=model_data["revision"]
         )
         if model_size is None:
             logger.error(LogFormatter.error("Model size validation failed", error))
@@ -458,6 +463,11 @@ class ModelService(HuggingFaceService):
                 raise Exception(error)
             logger.info(LogFormatter.success("Chat template validation passed"))
 
+
+        architectures = model_info.config.get("architectures", "")     
+        if architectures:
+            architectures = ";".join(architectures)
+
         # Create eval entry
         eval_entry = {
             "model": model_data["model_id"],
@@ -465,7 +475,7 @@ class ModelService(HuggingFaceService):
             "revision": model_info.sha,
             "precision": model_data["precision"],
             "params": model_size,
-            "architectures": model_info.pipeline_tag if hasattr(model_info, 'pipeline_tag') else None,
+            "architectures": architectures,
             "weight_type": model_data["weight_type"],
             "status": "PENDING",
             "submitted_time": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),

backend/app/utils/model_validation.py

@@ -5,10 +5,12 @@ import re
 from typing import Tuple, Optional, Dict, Any
 import aiohttp
 from huggingface_hub import HfApi, ModelCard, hf_hub_download
+from huggingface_hub import hf_api
 from transformers import AutoConfig, AutoTokenizer
 from app.config.base import HF_TOKEN, API
 from app.utils.logging import LogFormatter
 
+
 logger = logging.getLogger(__name__)
 
 class ModelValidator:
@@ -54,78 +56,78 @@ class ModelValidator:
             logger.error(LogFormatter.error(error_msg, e))
             return False, str(e), None
             
-    async def get_safetensors_metadata(self, model_id: str, filename: str = "model.safetensors") -> Optional[Dict]:
+    async def get_safetensors_metadata(self, model_id: str, is_adapter: bool = False, revision: str = "main")  -> Optional[Dict]:
         """Get metadata from a safetensors file"""
         try:
-            url = f"{API['HUB']}/{model_id}/raw/main/{filename}"
-            async with aiohttp.ClientSession() as session:
-                async with session.get(url, headers=self.headers) as response:
-                    if response.status == 200:
-                        # Read only the first 32KB to get the metadata
-                        header = await response.content.read(32768)
-                        # Parse metadata length from the first 8 bytes
-                        metadata_len = int.from_bytes(header[:8], byteorder='little')
-                        metadata_bytes = header[8:8+metadata_len]
-                        return json.loads(metadata_bytes)
-            return None
+            if is_adapter:
+                metadata = await asyncio.to_thread(
+                    hf_api.parse_safetensors_file_metadata,
+                    model_id,
+                    "adapter_model.safetensors",
+                    token=self.token,
+                    revision=revision,
+                )
+            else:
+                metadata = await asyncio.to_thread(
+                    hf_api.get_safetensors_metadata,
+                    repo_id=model_id,
+                    token=self.token,
+                    revision=revision,
+                )
+            return metadata
+
         except Exception as e:
-            logger.warning(f"Failed to get safetensors metadata: {str(e)}")
+            logger.error(f"Failed to get safetensors metadata: {str(e)}")
             return None
-        
+
     async def get_model_size(
         self,
         model_info: Any,
         precision: str,
-        base_model: str
+        base_model: str,
+        revision: str
     ) -> Tuple[Optional[float], Optional[str]]:
         """Get model size in billions of parameters"""
         try:
             logger.info(LogFormatter.info(f"Checking model size for {model_info.modelId}"))
-            
+
             # Check if model is adapter
             is_adapter = any(s.rfilename == "adapter_config.json" for s in model_info.siblings if hasattr(s, 'rfilename'))
-            
+
             # Try to get size from safetensors first
             model_size = None
-            
+
             if is_adapter and base_model:
                 # For adapters, we need both adapter and base model sizes
-                adapter_meta = await self.get_safetensors_metadata(model_info.id, "adapter_model.safetensors")
-                base_meta = await self.get_safetensors_metadata(base_model)
-                
+                adapter_meta = await self.get_safetensors_metadata(model_info.id, is_adapter=True, revision=revision)
+                base_meta = await self.get_safetensors_metadata(base_model, revision="main")
+
                 if adapter_meta and base_meta:
-                    adapter_size = sum(int(v.split(',')[0]) for v in adapter_meta.get("tensor_metadata", {}).values())
-                    base_size = sum(int(v.split(',')[0]) for v in base_meta.get("tensor_metadata", {}).values())
+                    adapter_size = sum(adapter_meta.parameter_count.values())
+                    base_size = sum(base_meta.parameter_count.values())
                     model_size = (adapter_size + base_size) / (2 * 1e9)  # Convert to billions, assuming float16
             else:
                 # For regular models, just get the model size
-                meta = await self.get_safetensors_metadata(model_info.id)
+                meta = await self.get_safetensors_metadata(model_info.id, revision=revision)
                 if meta:
-                    total_params = sum(int(v.split(',')[0]) for v in meta.get("tensor_metadata", {}).values())
+                    total_params = sum(meta.parameter_count.values())
                     model_size = total_params / (2 * 1e9)  # Convert to billions, assuming float16
-            
+
             if model_size is None:
-                # Fallback: Try to get size from model name
-                size_pattern = re.compile(r"(\d+\.?\d*)b")  # Matches patterns like "7b", "13b", "1.1b"
-                size_match = re.search(size_pattern, model_info.id.lower())
-                
-                if size_match:
-                    size_str = size_match.group(1)
-                    model_size = float(size_str)
-                else:
-                    return None, "Could not determine model size from safetensors or model name"
-            
+                # If model size could not be determined, return an error
+                return None, "Model size could not be determined"
+
             # Adjust size for GPTQ models
             size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.id.lower()) else 1
             model_size = round(size_factor * model_size, 3)
-            
+
             logger.info(LogFormatter.success(f"Model size: {model_size}B parameters"))
             return model_size, None
-            
+
         except Exception as e:
-            error_msg = "Failed to get model size"
-            logger.error(LogFormatter.error(error_msg, e))
+            logger.error(LogFormatter.error(f"Error while determining model size: {e}"))
             return None, str(e)
+
             
     async def check_chat_template(
         self,

backend/pyproject.toml

@@ -1,30 +1,54 @@
-[tool.poetry]
+[project]
 name = "llm-leaderboard-backend"
 version = "0.1.0"
 description = "Backend for the Open LLM Leaderboard"
-authors = ["Your Name <your.email@example.com>"]
-
-[tool.poetry.dependencies]
-python = "^3.9"
-fastapi = "^0.104.1"
-uvicorn = {extras = ["standard"], version = "^0.24.0"}
-numpy = "1.24.3"
-pandas = "^2.0.0"
-datasets = "^2.0.0"
-pyarrow = "^14.0.1"
-python-multipart = "^0.0.6"
-huggingface-hub = "^0.19.0"
-transformers = "^4.35.0"
-safetensors = "^0.4.0"
-aiofiles = "^24.1.0"
-fastapi-cache2 = "^0.2.1"
-
-[tool.poetry.group.dev.dependencies]
-pytest = "^7.4.0"
-black = "^23.7.0"
-isort = "^5.12.0"
-flake8 = "^6.1.0"
+requires-python = "==3.12.1"
+
+dependencies = [
+    "fastapi >=0.115.6",
+    "uvicorn >=0.34.0",
+    "numpy >=2.2.0",
+    "pandas >=2.2.3",
+    "datasets >=3.2.0",
+    "pyarrow >=18.1.0",
+    "python-multipart >=0.0.20",
+    "huggingface-hub >=0.27.0",
+    "transformers >=4.47.0",
+    "safetensors >=0.4.5",
+    "aiofiles >=24.1.0",
+    "fastapi-cache2 >=0.2.1",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest >=8.3.4",
+    "black >=24.10.0",
+    "isort >=5.13.2",
+    "flake8 >=7.1.1",
+]
+
+[tool.ruff]
+line-length = 120
+target-version = "py312"
+include = ["*.py", "*.pyi", "**/pyproject.toml", "*.ipynb"]
+ignore=["I","EM","FBT","TRY003","S101","D101","D102","D103","D104","D105","G004","D107","FA102"]
+fixable=["ALL"]
+select=["ALL"]
+
+[tool.ruff.lint]
+select = ["E", "F"]
+fixable = ["ALL"]
+ignore = ["E501"] # line too long (black is taking care of this)
+
+[tool.isort]
+profile = "black"
+
+[tool.black]
+line-length = 119
 
 [build-system]
-requires = ["poetry-core>=1.0.0"]
-build-backend = "poetry.core.masonry.api" 
+requires = ["hatchling>=1.0.0"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["backend"]

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
   backend:
     build:
@@ -15,7 +13,7 @@ services:
       - ENVIRONMENT=${ENVIRONMENT:-development}
       - HF_TOKEN=${HF_TOKEN}
       - HF_HOME=${HF_HOME:-/.cache}
-    command: uvicorn app.asgi:app --host 0.0.0.0 --port 8000 --reload
+    command: uv run uvicorn app.asgi:app --host 0.0.0.0 --port 8000 --reload
 
   frontend:
     build:

frontend/src/pages/AddModelPage/components/ModelSubmissionForm/ModelSubmissionForm.js

@@ -113,7 +113,7 @@ const HELP_TEXTS = {
         Chat Template Support
       </Typography>
       <Typography variant="body2" sx={{ opacity: 0.9, lineHeight: 1.4 }}>
-        Activates automatically for chat models. Uses the standardized Hugging
+        Activates automatically for chat models. It uses the standardized Hugging
         Face chat template for consistent prompt formatting during evaluation.
         Required for models using RLHF, DPO, or instruction fine-tuning.
       </Typography>
@@ -137,7 +137,7 @@ function ModelSubmissionForm({ user, isAuthenticated }) {
     isChatModel: false,
     useChatTemplate: false,
     precision: "float16",
-    weightsType: "original",
+    weightsType: "Original",
     baseModel: "",
   });
   const [error, setError] = useState(null);
@@ -333,7 +333,7 @@ function ModelSubmissionForm({ user, isAuthenticated }) {
                   isChatModel: false,
                   useChatTemplate: false,
                   precision: "float16",
-                  weightsType: "original",
+                  weightsType: "Original",
                   baseModel: "",
                 });
               }}
@@ -482,7 +482,7 @@ function ModelSubmissionForm({ user, isAuthenticated }) {
                         onChange={handleChange}
                       />
                     }
-                    label="Uses Chat Template"
+                    label="Use Chat Template"
                   />
                   <InfoIconWithTooltip tooltip={HELP_TEXTS.chatTemplate} />
                 </Stack>
@@ -536,13 +536,13 @@ function ModelSubmissionForm({ user, isAuthenticated }) {
                 </FormControl>
               </Grid>
 
-              {formData.weightsType !== "original" && (
+              {formData.weightsType !== "Original" && (
                 <Grid item xs={12}>
                   <TextField
                     fullWidth
                     required={
-                      formData.weightsType === "delta" ||
-                      formData.weightsType === "adapter"
+                      formData.weightsType === "Delta" ||
+                      formData.weightsType === "Adapter"
                     }
                     name="baseModel"
                     label="Base Model"