timep12345 commited on
Commit
c5d1b72
·
1 Parent(s): c002e8b

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +3 -2
app.py CHANGED
@@ -5,7 +5,7 @@ import json
5
  from langchain.document_loaders import DataFrameLoader
6
  from langchain.text_splitter import RecursiveCharacterTextSplitter
7
  from langchain.llms import HuggingFaceHub
8
- from langchain.embeddings import HuggingFaceEmbeddings
9
  from langchain.vectorstores import Chroma
10
  from langchain.chains import RetrievalQA
11
 
@@ -45,11 +45,12 @@ def url_changes(url, pages_to_visit, urls_to_scrape, repo_id):
45
  texts = text_splitter.split_documents(documents)
46
  print(f"documents splitted into {len(texts)} chunks")
47
 
48
- embeddings = HuggingFaceEmbeddings(model_name="jhgan/ko-sroberta-multitask")
49
 
50
  persist_directory = './vector_db'
51
  db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory)
52
  retriever = db.as_retriever()
 
53
  llm = HuggingFaceHub(repo_id=repo_id, model_kwargs={"temperature":0.1, "max_new_tokens":250})
54
  global qa
55
  qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever, return_source_documents=True)
 
5
  from langchain.document_loaders import DataFrameLoader
6
  from langchain.text_splitter import RecursiveCharacterTextSplitter
7
  from langchain.llms import HuggingFaceHub
8
+ from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
9
  from langchain.vectorstores import Chroma
10
  from langchain.chains import RetrievalQA
11
 
 
45
  texts = text_splitter.split_documents(documents)
46
  print(f"documents splitted into {len(texts)} chunks")
47
 
48
+ embeddings = SentenceTransformerEmbeddings(model_name="jhgan/ko-sroberta-multitask")
49
 
50
  persist_directory = './vector_db'
51
  db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory)
52
  retriever = db.as_retriever()
53
+
54
  llm = HuggingFaceHub(repo_id=repo_id, model_kwargs={"temperature":0.1, "max_new_tokens":250})
55
  global qa
56
  qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever, return_source_documents=True)