Spaces:
Paused
Paused
File size: 2,340 Bytes
f51a56a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 |
import shutil
import requests
import sys
from typing import Optional, List, Tuple
import json
from langchain_community.llms import HuggingFaceHub
##Loading the Model to answer questions
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
from peft import PeftModel, PeftConfig
peft_model_id = "Ubaidbhat/zephr_finance_finetuned"
config = PeftConfig.from_pretrained(peft_model_id)
print(config.base_model_name_or_path)
bnb_config = BitsAndBytesConfig(
load_in_4bit = True,
bnb_4bit_use_double_quant=True,
bnb_4bit_quant_type="nf4",
bnb_4bit_compute_dtype=torch.bfloat16
)
d_map = {"": torch.cuda.current_device()} if torch.cuda.is_available() else None
model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, quantization_config=bnb_config, device_map=d_map)
tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
model = PeftModel.from_pretrained(model, peft_model_id)
model = model.merge_and_unload()
##Creating base Model Chain
from langchain.llms import HuggingFacePipeline
from langchain.prompts import PromptTemplate
from transformers import pipeline
from langchain_core.output_parsers import StrOutputParser
from langchain.chains import LLMChain
text_generation_pipeline = pipeline(
model=model,
tokenizer=tokenizer,
task="text-generation",
temperature=0.2,
do_sample=True,
repetition_penalty=1.1,
return_full_text=True,
max_new_tokens=400,
pad_token_id=tokenizer.eos_token_id,
)
llm = HuggingFacePipeline(pipeline=text_generation_pipeline)
prompt_template = """
<|system|>
Answer the question based on your knowledge.
</s>
<|user|>
{question}
</s>
<|assistant|>
"""
prompt = PromptTemplate(
input_variables=["question"],
template=prompt_template,
)
llm_chain = prompt | llm | StrOutputParser()
def inference(question):
llmAnswer = llm_chain.invoke({"question": question})
llmAnswer = llmAnswer.rstrip()
return llmAnswer
import gradio as gr
from langchain_core.runnables import RunnablePassthrough
def predict(question):
return inference(question)
pred = gr.Interface(
fn=predict,
inputs=[
gr.Textbox(label="Question"),
],
outputs="text",
title="Finetuned Zephr Model in the Finance Domain."
)
pred.launch(share=True)
|