added one boook
Browse files
.gitattributes
CHANGED
@@ -33,7 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
36 |
-
data/
|
37 |
-
data/
|
38 |
data/pci_dss_v4/index.faiss filter=lfs diff=lfs merge=lfs -text
|
39 |
data/pci_dss_v4/index.pkl filter=lfs diff=lfs merge=lfs -text
|
|
|
33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
36 |
+
data/ai_books/index.faiss filter=lfs diff=lfs merge=lfs -text
|
37 |
+
data/ai_books/index.pkl filter=lfs diff=lfs merge=lfs -text
|
38 |
data/pci_dss_v4/index.faiss filter=lfs diff=lfs merge=lfs -text
|
39 |
data/pci_dss_v4/index.pkl filter=lfs diff=lfs merge=lfs -text
|
data/{faiss_1024_512 → ai_books}/index.faiss
RENAMED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:27612af17ee2ee1c73a4dc91772122c112c1f14ae280649e3b7ca5afeea1e7b9
|
3 |
+
size 91047981
|
data/{faiss_1024_512 → ai_books}/index.pkl
RENAMED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:9df9fa6831ab9736f93a877822bdaf3e472dea2ba6701ea63598a4447bfab463
|
3 |
+
size 30845062
|
ingest.py
CHANGED
@@ -10,7 +10,7 @@ from langchain.vectorstores.base import VectorStore
|
|
10 |
from langchain.vectorstores.chroma import Chroma
|
11 |
from langchain.vectorstores.faiss import FAISS
|
12 |
|
13 |
-
from app_modules.
|
14 |
|
15 |
|
16 |
def load_documents(source_pdfs_path, urls) -> List:
|
@@ -54,8 +54,6 @@ def generate_index(
|
|
54 |
|
55 |
|
56 |
# Constants
|
57 |
-
init_settings()
|
58 |
-
|
59 |
device_type, hf_pipeline_device_type = get_device_types()
|
60 |
hf_embeddings_model_name = (
|
61 |
os.environ.get("HF_EMBEDDINGS_MODEL_NAME") or "hkunlp/instructor-xl"
|
|
|
10 |
from langchain.vectorstores.chroma import Chroma
|
11 |
from langchain.vectorstores.faiss import FAISS
|
12 |
|
13 |
+
from app_modules.init import *
|
14 |
|
15 |
|
16 |
def load_documents(source_pdfs_path, urls) -> List:
|
|
|
54 |
|
55 |
|
56 |
# Constants
|
|
|
|
|
57 |
device_type, hf_pipeline_device_type = get_device_types()
|
58 |
hf_embeddings_model_name = (
|
59 |
os.environ.get("HF_EMBEDDINGS_MODEL_NAME") or "hkunlp/instructor-xl"
|