Update utills.py
Browse files
utills.py
CHANGED
|
@@ -55,7 +55,7 @@ def load_txt_documents(data_path):
|
|
| 55 |
return documents
|
| 56 |
|
| 57 |
|
| 58 |
-
def split_docs(
|
| 59 |
try:
|
| 60 |
text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
|
| 61 |
chunk_size=chunk_size, chunk_overlap=chunk_overlap,
|
|
|
|
| 55 |
return documents
|
| 56 |
|
| 57 |
|
| 58 |
+
def split_docs(documents, chunk_size, chunk_overlap):
|
| 59 |
try:
|
| 60 |
text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
|
| 61 |
chunk_size=chunk_size, chunk_overlap=chunk_overlap,
|