Spaces:
Runtime error
Runtime error
Commit
·
c5d1b72
1
Parent(s):
c002e8b
Update app.py
Browse files
app.py
CHANGED
|
@@ -5,7 +5,7 @@ import json
|
|
| 5 |
from langchain.document_loaders import DataFrameLoader
|
| 6 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 7 |
from langchain.llms import HuggingFaceHub
|
| 8 |
-
from langchain.embeddings import
|
| 9 |
from langchain.vectorstores import Chroma
|
| 10 |
from langchain.chains import RetrievalQA
|
| 11 |
|
|
@@ -45,11 +45,12 @@ def url_changes(url, pages_to_visit, urls_to_scrape, repo_id):
|
|
| 45 |
texts = text_splitter.split_documents(documents)
|
| 46 |
print(f"documents splitted into {len(texts)} chunks")
|
| 47 |
|
| 48 |
-
embeddings =
|
| 49 |
|
| 50 |
persist_directory = './vector_db'
|
| 51 |
db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory)
|
| 52 |
retriever = db.as_retriever()
|
|
|
|
| 53 |
llm = HuggingFaceHub(repo_id=repo_id, model_kwargs={"temperature":0.1, "max_new_tokens":250})
|
| 54 |
global qa
|
| 55 |
qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever, return_source_documents=True)
|
|
|
|
| 5 |
from langchain.document_loaders import DataFrameLoader
|
| 6 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 7 |
from langchain.llms import HuggingFaceHub
|
| 8 |
+
from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
|
| 9 |
from langchain.vectorstores import Chroma
|
| 10 |
from langchain.chains import RetrievalQA
|
| 11 |
|
|
|
|
| 45 |
texts = text_splitter.split_documents(documents)
|
| 46 |
print(f"documents splitted into {len(texts)} chunks")
|
| 47 |
|
| 48 |
+
embeddings = SentenceTransformerEmbeddings(model_name="jhgan/ko-sroberta-multitask")
|
| 49 |
|
| 50 |
persist_directory = './vector_db'
|
| 51 |
db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory)
|
| 52 |
retriever = db.as_retriever()
|
| 53 |
+
|
| 54 |
llm = HuggingFaceHub(repo_id=repo_id, model_kwargs={"temperature":0.1, "max_new_tokens":250})
|
| 55 |
global qa
|
| 56 |
qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever, return_source_documents=True)
|