Spaces:
Running
Running
File size: 1,708 Bytes
f1ac57a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 |
import os
import modal
vllm_image = (
modal.Image.debian_slim(python_version="3.10")
.pip_install(
"vllm",
"huggingface_hub[hf_transfer]==0.26.2",
"flashinfer-python==0.2.0.post2", # pinning, very unstable
extra_index_url="https://flashinfer.ai/whl/cu124/torch2.5",
)
.env({"HF_HUB_ENABLE_HF_TRANSFER": "1"}) # faster model transfers
)
MODELS_DIR = "/llamas"
MODEL_NAME = "microsoft/Phi-4-mini-instruct"
MODEL_REVISION = "c0fb9e74abda11b496b7907a9c6c9009a7a0488f"
hf_cache_vol = modal.Volume.from_name("huggingface-cache", create_if_missing=True)
vllm_cache_vol = modal.Volume.from_name("vllm-cache", create_if_missing=True)
app = modal.App("phi-4-mini-instruct-qa-vllm")
N_GPU = 1
MINUTES = 60 # seconds
VLLM_PORT = 8000
@app.function(
image=vllm_image,
# gpu=f"L40S:{N_GPU}",
gpu=f"A10G:{N_GPU}",
# how long should we stay up with no requests?
scaledown_window=5 * MINUTES,
volumes={
"/root/.cache/huggingface": hf_cache_vol,
"/root/.cache/vllm": vllm_cache_vol,
},
secrets=[modal.Secret.from_name("document-qa-api-key")]
)
@modal.concurrent(
max_inputs=5
) # how many requests can one replica handle? tune carefully!
@modal.web_server(port=VLLM_PORT, startup_timeout=5 * MINUTES)
def serve():
import subprocess
cmd = [
"vllm",
"serve",
"--uvicorn-log-level=info",
MODEL_NAME,
"--revision",
MODEL_REVISION,
"--max-model-len",
"32768",
"--host",
"0.0.0.0",
"--port",
str(VLLM_PORT),
"--api-key",
os.environ["API_KEY"],
]
subprocess.Popen(" ".join(cmd), shell=True) |