Add the num_workers parameter to the load_data function to enable multiprocessing when loading documents. (#5)

- Add the num_workers parameter to the load_data function to enable multiprocessing when loading documents. (976a01e38acc5ceb3034dc92ce8be90b58a00342)


Co-authored-by: Gabriel Silva Rodrigues <[email protected]>

app.py

@@ -383,13 +383,14 @@ else:
 #    Caso contrário, apenas reutilizamos o que já existe.
 if "docstore" not in st.session_state:
     # Carregar documentos do diretório local
+    count_cores = os.cpu_count()
     file_extractor = {".csv": CustomPandasCSVReader()}
     documents = SimpleDirectoryReader(
         input_dir=documents_path,
         file_extractor=file_extractor,
         filename_as_id=True,
         recursive=True
-    ).load_data()
+    ).load_data(num_workers=count_cores)
 
     documents = clean_documents(documents)