🚀 Initial commit of FastAPI LoRA chatbot

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.9
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY --chown=user . /app
+CMD ["bash", "start.sh"]

app/download_adapter.py

@@ -0,0 +1,41 @@
+import os
+import gdown
+import re
+
+# ✅ Google Drive folder with adapter versions like 'version 1', 'version 2', etc.
+DRIVE_FOLDER_URL = "https://drive.google.com/drive/folders/1S9xT92Zm9rZ4RSCxAe_DLld8vu78mqW4"
+LOCAL_DEST = "adapter"  # Where we'll copy the latest version
+
+def download_latest_adapter():
+    print("🔽 Downloading adapter folder from Google Drive...")
+
+    # Download everything from the Drive folder into temp dir
+    gdown.download_folder(url=DRIVE_FOLDER_URL, output="gdrive_tmp", quiet=False, use_cookies=False)
+
+    # Find all folders named "version X"
+    all_versions = sorted(
+        [d for d in os.listdir("gdrive_tmp") if re.match(r"version \d+", d)],
+        key=lambda x: int(x.split()[-1])
+    )
+
+    if not all_versions:
+        raise ValueError("❌ No version folders found in Google Drive folder.")
+
+    latest = all_versions[-1]
+    src = os.path.join("gdrive_tmp", latest)
+    print(f"✅ Latest adapter found: {latest}")
+
+    # Ensure destination exists
+    os.makedirs(LOCAL_DEST, exist_ok=True)
+
+    # Copy files to destination
+    for file in os.listdir(src):
+        src_file = os.path.join(src, file)
+        dest_file = os.path.join(LOCAL_DEST, file)
+        os.system(f"cp '{src_file}' '{dest_file}'")
+
+    print(f"✅ Adapter copied to: {LOCAL_DEST}")
+
+# ✅ Run automatically if script is executed directly
+if __name__ == "__main__":
+    download_latest_adapter()

app/main.py

@@ -0,0 +1,62 @@
+from fastapi import FastAPI, Request
+from pydantic import BaseModel
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+from peft import PeftModel
+import torch
+
+app = FastAPI()
+
+# ✅ Load tokenizer
+tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.2")
+tokenizer.pad_token = tokenizer.eos_token
+
+# ✅ Setup quantization config
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.float16
+)
+
+# ✅ Load base model
+model = AutoModelForCausalLM.from_pretrained(
+    "mistralai/Mistral-7B-Instruct-v0.2",
+    device_map="auto",
+    quantization_config=bnb_config
+)
+
+# ✅ Load LoRA adapter (ensure it's downloaded)
+ADAPTER_DIR = "./adapter/version 1"
+model = PeftModel.from_pretrained(model, ADAPTER_DIR)
+model.eval()
+
+# ✅ Build prompt from messages
+def build_prompt(messages):
+    prompt = ""
+    for msg in messages:
+        if msg["role"] == "user":
+            prompt += f"### User:\n{msg['content']}\n"
+        elif msg["role"] == "assistant":
+            prompt += f"### Assistant:\n{msg['content']}\n"
+    prompt += "### Assistant:\n"
+    return prompt
+
+# ✅ Input format
+class ChatRequest(BaseModel):
+    messages: list  # list of {"role": "user"/"assistant", "content": "..."}
+
+@app.post("/chat")
+async def chat(req: ChatRequest):
+    prompt = build_prompt(req.messages)
+    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+    output = model.generate(
+        **inputs,
+        max_new_tokens=256,
+        do_sample=True,
+        temperature=0.7,
+        top_p=0.95,
+        eos_token_id=tokenizer.eos_token_id,
+    )
+    response = tokenizer.decode(output[0], skip_special_tokens=True)
+    reply = response.split("### Assistant:")[-1].strip()
+    return {"response": reply}

requirements.txt

@@ -0,0 +1,9 @@
+transformers==4.40.1
+torch==2.1.2
+accelerate==0.29.3
+peft==0.10.0
+bitsandbytes==0.43.1
+uvicorn==0.29.0
+fastapi==0.110.2
+python-multipart==0.0.9
+gdown==5.1.0

start.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+echo "📦 Installing Python dependencies..."
+pip install -q --upgrade pip
+pip install -q torch transformers accelerate peft bitsandbytes gdown
+
+echo "✅ Dependencies installed."
+
+echo "📂 Downloading latest adapter from Google Drive..."
+python download_adapter.py
+
+echo "🤖 Launching AI chat server..."
+python main.py