vector store for windows

core-langchain-rag.py

@@ -214,7 +214,7 @@ def generate_qa_retriever(history: dict, question: str, llm_model:HuggingFaceEnd
     template = """
     You are a friendly insurance product advisor, your task is to help customers find the best products from Württembergische GmbH.\
     You help the user find the answers to all his questions. Answer in short and simple terms and offer to explain the product and terms to the user.\
-    Respond only using the provided context (delimited by <ctx></ctx>) and only in German or Englisch, depending on the question's language.
+    Respond only using the provided context (delimited by <ctx></ctx>) and only in German or English, depending on the question's language.
     Use the chat history (delimited by <hs></hs>) to help find the best product for the user:
     ------
     <ctx>

rag_app/load_vector_stores.py

@@ -10,6 +10,7 @@ from dotenv import load_dotenv
 import os
 import sys
 import logging
+from pathlib import Path
 
 # Load environment variables from a .env file
 config = load_dotenv(".env")
@@ -38,6 +39,7 @@ def get_faiss_vs():
 
     # Define the destination for the downloaded file
     VS_DESTINATION = FAISS_INDEX_PATH + ".zip"
+    
     try:
         # Download the pre-prepared vectorized index from the S3 bucket
         print("Downloading the pre-prepared vectorized index from S3...")
@@ -51,7 +53,32 @@ def get_faiss_vs():
         
     except Exception as e:
         print(f"Error during downloading or extracting from S3: {e}", file=sys.stderr)
-    #faissdb = FAISS.load_local(FAISS_INDEX_PATH, embeddings)
+    # faissdb = FAISS.load_local(FAISS_INDEX_PATH, embeddings)
+
+
+def get_faiss_vs_from_s3(s3_loc:str, 
+                         s3_vs_name:str,
+                         vs_dir:str='vectorstore') -> None:
+    """ Download the FAISS vector store from S3 bucket
+
+        Args:
+            s3_loc (str): Name of the S3 bucket
+            s3_vs_name (str): Name of the file to be downloaded
+            vs_dir (str): The name of the directory where the file is to be saved
+    """
+    # Initialize an S3 client with unsigned configuration for public access
+    s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED))
+    # Destination folder
+    vs_dir_path = Path("..") / vs_dir
+    assert vs_dir_path.is_dir(), "Cannot find vs_dir folder"
+    try:
+        vs_destination = Path("..") / vs_dir / "faiss-insurance-agent-500.zip"
+        s3.download_file(s3_loc, s3_vs_name, vs_destination)
+        # Extract the downloaded zip file
+        with zipfile.ZipFile(file=vs_destination, mode='r') as zip_ref:
+            zip_ref.extractall(path=vs_dir_path.as_posix())
+    except Exception as e:
+        print(f"Error during downloading or extracting from S3: {e}", file=sys.stderr)
 
 
 ## Chroma DB
@@ -67,4 +94,10 @@ def get_chroma_vs():
         chromadb = Chroma(persist_directory=CHROMA_DIRECTORY, embedding_function=embeddings)
         chromadb.get()
     except Exception as e:
-        print(f"Error during downloading or extracting from S3: {e}", file=sys.stderr)
+        print(f"Error during downloading or extracting from S3: {e}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    # get_faiss_vs_from_s3(s3_loc=S3_LOCATION, s3_vs_name=FAISS_VS_NAME)
+    pass
+    

rag_app/metadata.ipynb

@@ -0,0 +1,170 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "from langchain_community.vectorstores import FAISS\n",
+    "from dotenv import load_dotenv\n",
+    "import os\n",
+    "from langchain_huggingface import HuggingFaceEmbeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "load_dotenv()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFCEHUB_API_TOKEN')\n",
+    "EMBEDDING_MODEL = os.getenv(\"EMBEDDING_MODEL\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "folder_path = Path('..') / \"vectorstore/faiss-insurance-agent-500\"\n",
+    "faissdb = FAISS.load_local(folder_path=str(folder_path.resolve()),\n",
+    "                                embeddings=embeddings,\n",
+    "                                allow_dangerous_deserialization=True)    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Content: Die private Haftpflichtversicherung...\n",
+      "Metadata: {'source': 'https://www.wuerttembergische.de/versicherungen/stadt/wuppertal/', 'content_type': 'text/html; charset=UTF-8', 'title': 'Versicherung in Wuppertal', 'description': 'Ihre Versicherungsagentur in Wuppertal: Kommen Sie zur Württembergischen Versicherung und profitieren Sie von einer persönlichen Beratung und ausgezeichnetem Service. ', 'language': 'de'}\n",
+      "---\n",
+      "Content: Haftpflichtversicherung...\n",
+      "Metadata: {'source': 'https://www.wuerttembergische.de/wohnen/hausratversicherung/sengschaden/', 'content_type': 'text/html; charset=UTF-8', 'title': 'Sengschäden: So schützt Sie Ihre Hausrat- und Wohngebäudeversicherung', 'description': 'Deckt Ihre Hausratversicherung Sengschäden ab? Finden Sie heraus, wie Sie bei Schäden durch Glut oder Hitze ohne direktes Feuer geschützt sind.\\n', 'language': 'de'}\n",
+      "---\n",
+      "Content: Die Leistungen unserer privaten Haftpflichtversich...\n",
+      "Metadata: {'source': 'https://www.wuerttembergische.de/existenz/private-haftpflichtversicherung/drohnen-versichern/', 'content_type': 'text/html; charset=UTF-8', 'title': 'Drohnen über die private Haftpflicht versichern', 'description': 'Müssen Drohnen versichert sein? Welcher Tarif ist der beste? Erfahren Sie hier die wichtigsten Informationen rund ums Thema Drohne versichern.', 'language': 'de'}\n",
+      "---\n",
+      "Content: Das kann ohne private Haftpflichtversicherung pass...\n",
+      "Metadata: {'source': 'https://www.wuerttembergische.de/existenz/private-haftpflichtversicherung/pflicht/', 'content_type': 'text/html; charset=UTF-8', 'title': 'Ist die private Haftpflichtversicherung Pflicht oder freiwillig?', 'description': 'Ist eine Privathaftpflichtversicherung gesetzlich vorgeschrieben? Welche Haftpflichtversicherung Pflicht sind und welche freiwillig - das erfahren Sie hier.', 'language': 'de'}\n",
+      "---\n",
+      "Content: Private Haftpflicht: keine Pflichtversicherung\n",
+      "Fre...\n",
+      "Metadata: {'source': 'https://www.wuerttembergische.de/existenz/private-haftpflichtversicherung/pflicht/', 'content_type': 'text/html; charset=UTF-8', 'title': 'Ist die private Haftpflichtversicherung Pflicht oder freiwillig?', 'description': 'Ist eine Privathaftpflichtversicherung gesetzlich vorgeschrieben? Welche Haftpflichtversicherung Pflicht sind und welche freiwillig - das erfahren Sie hier.', 'language': 'de'}\n",
+      "---\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Perform a similarity search with an empty query to get random documents\n",
+    "documents = faissdb.similarity_search(\"Private Haftpflicht­versicherung\", k=5)\n",
+    "\n",
+    "for doc in documents:\n",
+    "    print(f\"Content: {doc.page_content[:50]}...\")  # Print first 50 chars of content\n",
+    "    print(f\"Metadata: {doc.metadata}\")\n",
+    "    print(\"---\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of entries in the database: 62496\n"
+     ]
+    }
+   ],
+   "source": [
+    "num_entries = len(faissdb.index_to_docstore_id)\n",
+    "print(f\"Number of entries in the database: {num_entries}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of entries in the database: 62496\n"
+     ]
+    }
+   ],
+   "source": [
+    "num_entries = faissdb.index.ntotal\n",
+    "print(f\"Number of entries in the database: {num_entries}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

rag_app/metadata_filtering.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+from langchain_community.vectorstores import FAISS
+from dotenv import load_dotenv
+import os
+from langchain_huggingface import HuggingFaceEmbeddings
+
+
+load_dotenv(".env")
+
+HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN')
+EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL")
+
+
+if __name__ == "__main__":
+
+    embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)
+
+    folder_path = Path('..') / "vectorstore/faiss-insurance-agent-500"
+
+    print(f'{Path(folder_path).exists() = }')        
+
+    faissdb = FAISS.load_local(folder_path=str(folder_path.resolve()),
+                                embeddings=embeddings,
+                                allow_dangerous_deserialization=True)
+    
+    documents = faissdb.get(list(range(5)))
+
+    for doc in documents:
+        print(f"Metadata: {doc.metadata}")