update tiktoken requirement

embed_docs.py

@@ -3,6 +3,7 @@ import os
 
 from buster.documents_manager import DeepLakeDocumentsManager
 from buster.parsers import SphinxParser, get_all_documents
+from buster.utils import zip_contents
 
 from rtd_scraper.scrape_rtd import sanitize_url, run_spider
 
@@ -12,7 +13,7 @@ for name in logging.root.manager.loggerDict:
     logger.setLevel(logging.INFO)
 
 
-def embed_documents(homepage_url, save_directory, target_version=None):
+def crawl_docs(homepage_url, save_directory, target_version=None):
     # adds https:// and trailing slash
     homepage_url = sanitize_url(homepage_url)
 
@@ -21,6 +22,11 @@ def embed_documents(homepage_url, save_directory, target_version=None):
         homepage_url, save_directory=save_directory, target_version=target_version
     )
 
+
+def embed_documents(homepage_url, save_directory):
+    # # adds https:// and trailing slash
+    homepage_url = sanitize_url(homepage_url)
+
     # # Convert the .html pages into chunks using Buster's SphinxParser
     # root_dir is the folder containing the scraped content e.g. crawled_outputs/buster.readthedocs.io/
     root_dir = os.path.join(save_directory, homepage_url.split("https://")[1])
@@ -31,7 +37,7 @@ def embed_documents(homepage_url, save_directory, target_version=None):
         min_section_length=100,
         max_section_length=1000,
     )
-    df["source"] = "readthedocs"  # Add the source column
+    df["source"] = homepage_url  # Add the source column
 
     #  Initialize the DeepLake vector store
     vector_store_path = os.path.join(save_directory, "deeplake_store")
@@ -49,12 +55,17 @@ def embed_documents(homepage_url, save_directory, target_version=None):
         num_workers=32,
     )
 
+def crawl_and_embed_docs(homepage_url, save_directory, target_version=None):
+    # crawl_docs(homepage_url, save_directory, target_version)
+    # embed_documents(homepage_url, save_directory)
+    zip_contents(save_directory, output_path=".")
+
 
 if __name__ == "__main__":
-    homepage_url = "https://orion.readthedocs.io/"
-    target_version = "v0.2.7"
-    save_directory = "outputs/"
-    embed_documents(
+    homepage_url = "https://docs.mila.quebec/"
+    target_version = ""
+    save_directory = "outputs"
+    crawl_and_embed_docs(
         homepage_url=homepage_url,
         target_version=target_version,
         save_directory=save_directory,

requirements.txt

@@ -1,3 +1,3 @@
 buster-doctalk==1.0.28
 gradio==4.39.0
-scrapy
+tiktoken==0.7.0