app.py

import gradio as gr
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline

torch.random.manual_seed(0)

proc_model = AutoModelForCausalLM.from_pretrained(
    "microsoft/Phi-3-mini-4k-instruct",
    trust_remote_code=True, 
)

proc_model.to("cpu")
proc_model.eval()
proc_tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct")



proc_pipe = pipeline(
    "text-generation",
    model=proc_model,
    tokenizer=proc_tokenizer,
)

generation_args = {
    "max_new_tokens": 500,
    "return_full_text": False,
    "temperature": 0.0,
    "do_sample": False,
}

def generate_response(inputs):
    output = proc_pipe(inputs, **generation_args)
    return output[0]['generated_text']



# Create a Gradio interface
iface = gr.Interface(
    fn=generate_response, 
    inputs=[gr.Textbox(lines=5, placeholder="Enter your prompt here...")]
    outputs=gr.Textbox()
)

# Launch the interface
iface.launch()