File size: 2,702 Bytes
f1ac63f c095a09 f1ac63f 6bb29b1 f1ac63f c095a09 f1ac63f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 |
import torch
import gradio as gr
from transformers import AutoTokenizer, AutoModelForCausalLM
from peft import PeftModel, PeftConfig
import spaces
# Check if CUDA is available and set the device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using device: {device}")
# Load model and tokenizer
MODEL_PATH = "sagar007/phi3.5_finetune"
tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True)
tokenizer.pad_token = tokenizer.eos_token
base_model = AutoModelForCausalLM.from_pretrained(
"microsoft/Phi-3.5-mini-instruct",
torch_dtype=torch.float16 if device.type == "cuda" else torch.float32,
device_map="auto",
trust_remote_code=True
)
peft_config = PeftConfig.from_pretrained(MODEL_PATH)
model = PeftModel.from_pretrained(base_model, MODEL_PATH)
model.to(device)
model.eval()
@spaces.GPU(duration=60)
def generate_response(instruction, max_length=512):
prompt = f"Instruction: {instruction}\nResponse:"
inputs = tokenizer(prompt, return_tensors="pt").to(device)
with torch.no_grad():
outputs = model.generate(
**inputs,
max_length=max_length,
num_return_sequences=1,
temperature=0.7,
top_p=0.9,
do_sample=True
)
response = tokenizer.decode(outputs[0], skip_special_tokens=True)
return response.split("Response:")[1].strip()
def chatbot(message, history):
response = generate_response(message)
return response
demo = gr.ChatInterface(
chatbot,
title="Fine-tuned Phi-3.5 Chatbot",
description="This is a chatbot using a fine-tuned version of the Phi-2 model.",
theme="default",
examples=[
["Explain the concept of machine learning."],
["Write a short story about a robot learning to paint."],
["What are some effective ways to reduce stress?"],
["Summarize the key points of climate change in simple terms."],
["Create a step-by-step guide for making a perfect omelette."],
["Describe the differences between classical and quantum computing."],
["Write a motivational speech for a team starting a new project."],
["Explain the importance of biodiversity in ecosystems."],
["Compose a haiku about artificial intelligence."],
["List five tips for effective time management."],
["Describe the process of photosynthesis in layman's terms."],
["Write a dialogue between two characters discussing the future of space exploration."],
["Explain the concept of blockchain technology and its potential applications."]
cache_examples=True,
)
if __name__ == "__main__":
demo.launch() |