Rename and update db.py

setup_db.py → db.py

@@ -6,7 +6,6 @@ from langchain_chroma import Chroma
 
 
 file_path="./paul_graham_essays.csv"
-model_path = 'intfloat/multilingual-e5-large-instruct'
 db_persist_directory = './docs/chroma/'
 
 
@@ -26,14 +25,14 @@ def load_data():
     return data[1:]
 
 
-def split_docs(docs):
+def split_data(data, chunk_size, chunk_overlap):
     splitter = RecursiveCharacterTextSplitter(
-        chunk_size=1000,
-        chunk_overlap=200,
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
         separators=['\n\n', '\n', '(?<=\. )', ' ', '']
     )
 
-    return splitter.split_documents(docs)
+    return splitter.split_documents(data)
 
 
 def generate_embeddings():
@@ -49,8 +48,14 @@ def generate_embeddings():
     )
 
 
-def get_db():
-    splits = split_docs(load_data())
+def get_db(
+    chunk_size=1000,
+    chunk_overlap=200,
+    model_path = 'intfloat/multilingual-e5-large-instruct',
+):
+    data = load_data()
+    
+    splits = split_data(data, chunk_size, chunk_overlap)
 
     embedding = generate_embeddings()
     
@@ -58,4 +63,4 @@ def get_db():
         documents=splits,
         embedding=embedding,
         persist_directory=persist_directory,
-    )
+    )