Update app.py

app.py

@@ -3,9 +3,9 @@ import pandas as pd
 import json
 
 from langchain.document_loaders import DataFrameLoader
-from langchain.text_splitter import CharacterTextSplitter
+from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.llms import HuggingFaceHub
-from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
+from langchain.embeddings import HuggingFaceEmbeddings
 from langchain.vectorstores import Chroma
 from langchain.chains import RetrievalQA
 
@@ -34,13 +34,21 @@ def url_changes(url, pages_to_visit, urls_to_scrape, repo_id):
           result = json.loads(result)
 
           results_df = pd.concat([results_df, pd.DataFrame.from_records([result])])
-
-    loader = DataFrameLoader(results_df, page_content_column="text")
+    results_df.to_csv("./data.csv")
+    
+    df = pd.read_csv("./data.csv")
+    loader = DataFrameLoader(df, page_content_column="text")
     documents = loader.load()
-    text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+    print(f"{len(documents)} documents loaded") 
+
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
     texts = text_splitter.split_documents(documents)
-    embeddings = SentenceTransformerEmbeddings(model_name="jhgan/ko-sroberta-multitask")
-    db = Chroma.from_documents(texts, embeddings)
+    print(f"documents splitted into {len(texts)} chunks") 
+    
+    embeddings = HuggingFaceEmbeddings(model_name="jhgan/ko-sroberta-multitask")
+
+    persist_directory = './vector_db'
+    db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory)
     retriever = db.as_retriever()
     llm = HuggingFaceHub(repo_id=repo_id, model_kwargs={"temperature":0.1, "max_new_tokens":250})
     global qa