app.py

import gradio as gr
from huggingface_hub import InferenceClient
from optimum.intel import OVModelForCausalLM
from transformers import AutoTokenizer, pipeline

# Load the model and tokenizer
model_id = "hsuwill000/Qwen2.5-1.5B-Instruct-openvino-8bit"
model = OVModelForCausalLM.from_pretrained(model_id, device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(model_id)

# Create generation pipeline
pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)

def respond(message, history):
    # Combine current message with previous history
    input_text = message if not history else history[-1]["value"] + " " + message
    # Get model's response
    response = pipe(input_text, max_length=512, truncation=True, num_return_sequences=1)
    reply = response[0]['generated_text']
    
    # Return new message format
    print(f"Message: {message}")
    print(f"Reply: {reply}")
    return [{"role": "bot", "value": reply}]
    
# Set up Gradio chat interface
demo = gr.ChatInterface(fn=respond, title="Qwen2.5-3B-Instruct-openvino", description="Qwen2.5-3B-Instruct-openvino", type='chatbot')

if __name__ == "__main__":
    demo.launch()