app.py

from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer

class ModelInput(BaseModel):
    prompt: str
    max_new_tokens: int = 50

app = FastAPI()

# Since we're getting config errors with PEFT, let's load the fine-tuned model directly
model_path = "khurrameycon/SmolLM-135M-Instruct-qa_pairs_converted.json-25epochs"

try:
    # Load the model and tokenizer directly from your fine-tuned version
    model = AutoModelForCausalLM.from_pretrained(
        model_path,
        trust_remote_code=True,
        device_map="auto"
    )
    
    tokenizer = AutoTokenizer.from_pretrained(model_path)
    print("Model loaded successfully!")

except Exception as e:
    print(f"Error loading model: {e}")
    raise

def generate_response(model, tokenizer, instruction, max_new_tokens=128):
    """Generate a response from the model based on an instruction."""
    try:
        # Format the input
        messages = [{"role": "user", "content": instruction}]
        input_text = tokenizer.apply_chat_template(
            messages, tokenize=False, add_generation_prompt=True
        )
        
        # Generate
        inputs = tokenizer.encode(input_text, return_tensors="pt").to(model.device)
        outputs = model.generate(
            inputs,
            max_new_tokens=max_new_tokens,
            temperature=0.2,
            top_p=0.9,
            do_sample=True,
        )
        
        # Decode
        response = tokenizer.decode(outputs[0], skip_special_tokens=True)
        return response
    
    except Exception as e:
        raise ValueError(f"Error generating response: {e}")

@app.post("/generate")
async def generate_text(input: ModelInput):
    """API endpoint to generate text."""
    try:
        response = generate_response(
            model=model,
            tokenizer=tokenizer,
            instruction=input.prompt,
            max_new_tokens=input.max_new_tokens
        )
        return {"generated_text": response}
    
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/")
async def root():
    return {"message": "Welcome to the Model API!"}