app.py

import torch
import gradio as gr
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
import spaces

model_id = "google/flan-t5-base"
tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForSeq2SeqLM.from_pretrained(model_id)
model.to("cuda" if torch.cuda.is_available() else "cpu")

@spaces.GPU
def respond(message, history=[]):
    prompt = f"Answer the question: {message}"
    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
    output = model.generate(**inputs, max_new_tokens=100)
    response = tokenizer.decode(output[0], skip_special_tokens=True)
    history.append((message, response))
    return history

gr.ChatInterface(fn=respond).launch()