Update main.py

main.py

@@ -2,11 +2,11 @@ from fastapi import FastAPI
 from pydantic import BaseModel
 from huggingface_hub import InferenceClient
 import uvicorn
-
+import asyncio
 
 app = FastAPI()
 
-client = InferenceClient("Qwen/Qwen2.5-Coder-32B-Instruct")
+client = InferenceClient(model="Qwen/Qwen2.5-7B")
 
 class Item(BaseModel):
     prompt: str
@@ -26,9 +26,7 @@ def format_prompt(message, history):
     return prompt
 
 def generate(item: Item):
-    temperature = float(item.temperature)
-    if temperature < 1e-2:
-        temperature = 1e-2
+    temperature = max(float(item.temperature), 1e-2)
     top_p = float(item.top_p)
 
     generate_kwargs = dict(
@@ -50,4 +48,16 @@ def generate(item: Item):
 
 @app.post("/generate/")
 async def generate_text(item: Item):
-    return {"response": generate(item)}
+    return {"response": generate(item)}
+
+@app.on_event("startup")
+async def preload_model():
+    # Check if the model is already loaded
+    status = client.get_model_status()
+    if not status.loaded:
+        # Trigger model loading by making a dummy request
+        dummy_prompt = "This is a dummy prompt to load the model."
+        client.text_generation(dummy_prompt, max_new_tokens=1)
+        # Optionally, wait until the model is loaded
+        while not client.get_model_status().loaded:
+            await asyncio.sleep(5)  # Wait for 5 seconds before checking again