Merge pull request #1 from almutareb/starter_docsqa

.devcontainer/Dockerfile

@@ -0,0 +1,44 @@
+FROM mcr.microsoft.com/devcontainers/base:jammy
+# FROM mcr.microsoft.com/devcontainers/base:jammy 
+
+ARG DEBIAN_FRONTEND=noninteractive
+ARG USER=vscode
+
+RUN DEBIAN_FRONTEND=noninteractive \
+    && apt-get update \ 
+    && apt-get install -y build-essential --no-install-recommends make \
+        ca-certificates \
+        git \
+        libssl-dev \
+        zlib1g-dev \
+        libbz2-dev \
+        libreadline-dev \
+        libsqlite3-dev \
+        wget \
+        curl \
+        llvm \
+        libncurses5-dev \
+        xz-utils \
+        tk-dev \
+        libxml2-dev \
+        libxmlsec1-dev \
+        libffi-dev \
+        liblzma-dev
+
+# Python and poetry installation
+USER $USER
+ARG HOME="/home/$USER"
+ARG PYTHON_VERSION=3.11
+
+ENV PYENV_ROOT="${HOME}/.pyenv"
+ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:${HOME}/.local/bin:$PATH"
+
+RUN echo "done 0" \
+    && curl https://pyenv.run | bash \
+    && echo "done 1" \
+    && pyenv install ${PYTHON_VERSION} \
+    && echo "done 2" \
+    && pyenv global ${PYTHON_VERSION} \
+    && echo "done 3" \
+    && curl -sSL https://install.python-poetry.org | python3 - \
+    && poetry config virtualenvs.in-project true

.devcontainer/devcontainer.json

@@ -0,0 +1,25 @@
+{
+	"name": "poetry3-poetry-pyenv",
+	"build": {
+		"dockerfile": "Dockerfile"
+	},
+
+	// 👇 Features to add to the Dev Container. More info: https://containers.dev/implementors/features.
+	// "features": {},
+
+	// 👇 Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// 👇 Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "",
+
+	// 👇 Configure tool-specific properties.
+	"customizations": {
+        "vscode": {
+            "extensions":["ms-python.python", "njpwerner.autodocstring","ms-azuretools.vscode-docker", "qwtel.sqlite-viewer"]
+            }
+        }
+
+	// 👇 Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.github/dependabot.yml

@@ -0,0 +1,12 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for more information:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+# https://containers.dev/guide/dependabot
+
+version: 2
+updates:
+ - package-ecosystem: "devcontainers"
+   directory: "/"
+   schedule:
+     interval: weekly

.github/workflows/pylint.yml

@@ -0,0 +1,23 @@
+name: Pylint
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pylint
+    - name: Analysing the code with pylint
+      run: |
+        pylint $(git ls-files '*.py')

.gitignore

@@ -158,3 +158,6 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+*.zip
+*.pkl
+*.faiss

CONTRIBUTION.md

@@ -1,36 +1,28 @@
-# Contributing to The Project
+# Pull Request Template
 
-We welcome contributions from everyone. To make the process as smooth as possible, please follow the guidelines below. 
+## Description
+Please include a brief description of the changes introduced by this PR.
 
-## How to Contribute
+## Related Issue(s)
+- If this PR addresses a particular issue, please reference it here using GitHub's linking syntax, e.g., "Fixes #123".
+- If there's no related issue, briefly explain the motivation behind these changes.
 
-### Code Contributions
+## Changes Made
+Please provide a list of the changes made in this PR.
 
-1. **Creating a Branch**: For any code contribution, start by creating a branch from the main project. The branch should be named using the pattern `#issue-id short-name-to-describe-what-is-added`. This helps in linking the contributions directly to the issues they are meant to resolve.
+## Screenshots (if applicable)
+If the changes include UI updates or visual changes, please attach relevant screenshots here.
 
-2. **Pull Requests**: Once you've made your changes, create a pull request (PR). Please fill out the pull request template with all the required information. This includes a summary of your changes and why they are necessary.
+## Checklist
+- [ ] I have tested my changes locally and ensured that they work as expected.
+- [ ] I have updated the documentation (if applicable).
+- [ ] My code follows the project's coding conventions and style guidelines.
+- [ ] I have added appropriate test cases (if applicable).
+- [ ] I have reviewed my own code to ensure its quality.
 
-3. **Review Process**: Every code contribution requires a review. A minimum of one maintainer must review and approve your contribution before it can be merged. This ensures quality and adherence to the project's standards. Please ensure your code is well-documented through comments and, if necessary, additional documentation files.
+## Additional Notes
+Add any additional notes or context about this PR here.
 
-4. **Testing**: Your code must be tested to ensure it works as expected and does not introduce new issues.
-
-### Reporting Issues
-
-When reporting issues, please use the provided issue template. Your report should include:
-
-- A clear, descriptive title
-- A detailed description of the issue
-- Steps to reproduce the issue
-- Logs, if applicable
-- Screenshots, if applicable
-
-This information is crucial in diagnosing and fixing the issue you're experiencing.
-
-### Suggestions
-
-We're always looking for new ideas to improve our project. If you have a suggestion, please:
-
-- Clearly describe your suggestion, including the purpose and intended outcome.
-- Explain why you believe this change would be beneficial to the project.
-
-We appreciate your contributions and look forward to collaborating with you!
+## Reviewer(s)
+- @reviewer1
+- @reviewer2

README.md

@@ -1,32 +1,33 @@
-# mail_automation_wf
-<img width="424" alt="image" src="https://github.com/almutareb/mail_automation_wf/assets/104657679/9fc2d76a-03fd-42f3-904f-0f6386d37091">
-
-## Overview
-This project aims to automate the processing of incoming emails by performing various tasks, including fetching emails, recognizing entities, classifying and summarizing content, enriching data, and drafting responses. This README provides detailed information on the project's goals, agents, tasks, and overall workflow.
-
-### Table of Contents
-- Goals
-- Agents and Tasks
-  - Email Fetcher
-  - Attachment Downloader
-  - Attachment Processor
-  - Entity Recognizer
-  - Classifier
-  - Data Enricher
-  - Data Checker
-  - Historical Data Fetcher
-  - Email Writer
-  - Quality Checker
-  - Email Saver
-- Workflow
-- Installation
-- Configuration
-- Usage
-- Contributing
-- License
-
-
-
-
-
-  
+# Anatomy of Advanced Enterprise Rag Systems
+
+This repository accompanies the blog series "The Anatomy of Advanced Enterprise Rag Systems" and provides a hands-on learning experience for building sophisticated Rag systems. Dive deep into each component, from setup and evaluation to security and multi-agent interactions.
+
+Explore these key topics:
+
+- Test Setup and Evaluation Metrics: Learn how to assess the performance and effectiveness of your Rag system.
+- Data Preparation and Management: Discover techniques for organizing and optimizing your knowledge base.
+- User Input Processing: Understand how to handle diverse user queries and extract relevant information.
+- Retrieval System: Unleash the power of retrieving relevant passages from your knowledge base.
+- Information Processing and Generation: Craft accurate and informative responses using state-of-the-art techniques.
+- Feedback and Continuous Improvement: Enhance your Rag system over time using user feedback and data analysis.
+- Multi-agents and Agent-services: Explore advanced architectures for distributed and collaborative Rag systems.
+- Monitoring and Security: Ensure the robustness and trustworthiness of your Rag system with proper monitoring and security practices.
+
+What you'll find here:
+
+- Code examples: Implementations of key concepts from each topic, ready to use and adapt.
+- Data samples: Pre-prepared data sets for experimentation and testing.
+- Additional resources: Links to relevant articles, libraries, and tools to deepen your understanding.
+
+Getting started:
+
+- Clone this repository: git clone https://github.com/<username>/advanced-enterprise-rag-systems.git
+- Follow the instructions in each topic directory.
+
+Contributing:
+
+We welcome your contributions! Share your expertise, improve existing code examples, or add new ones. Submit a pull request to share your valuable additions.
+
+License:
+
+This project is licensed under the MIT License: LICENSE.

app_gui.py

@@ -0,0 +1,65 @@
+# Import Gradio for UI, along with other necessary libraries
+import gradio as gr
+# need to import the qa!
+
+# Function to add a new input to the chat history
+def add_text(history, text):
+    # Append the new text to the history with a placeholder for the response
+    history = history + [(text, None)]
+    return history, ""
+
+# Function representing the bot's response mechanism
+def bot(history):
+    # Obtain the response from the 'infer' function using the latest input
+    response = infer(history[-1][0], history)
+    sources = [doc.metadata.get("source") for doc in response['source_documents']]
+    src_list = '\n'.join(sources)
+    print_this = response['result'] + "\n\n\n Sources: \n\n\n" + src_list
+
+
+    history[-1][1] = print_this #response['answer']
+    # Update the history with the bot's response
+    #history[-1][1] = response['result']
+    return history
+
+# Function to infer the response using the RAG model
+def infer(question, history):
+    # Use the question and history to query the RAG model
+    result = qa({"query": question, "history": history, "question": question})
+    return result
+
+# CSS styling for the Gradio interface
+css = """
+#col-container {max-width: 700px; margin-left: auto; margin-right: auto;}
+"""
+
+# HTML content for the Gradio interface title
+title = """
+<div style="text-align:left;">
+    <p>Hello, I BotTina 2.0, your intelligent AI assistant. I can help you explore Wuerttembergische Versicherungs products.<br />
+</div>
+"""
+
+# Building the Gradio interface
+with gr.Blocks(theme=gr.themes.Soft()) as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.HTML(title)  # Add the HTML title to the interface
+        chatbot = gr.Chatbot([], elem_id="chatbot",
+                                     bubble_full_width=False,
+                                     avatar_images=(None, "https://dacodi-production.s3.amazonaws.com/store/87bc00b6727589462954f2e3ff6f531c.png"),
+                                     height=680,)  # Initialize the chatbot component
+        clear = gr.Button("Clear")  # Add a button to clear the chat
+
+        # Create a row for the question input
+        with gr.Row():
+            question = gr.Textbox(label="Question", placeholder="Type your question and hit Enter ")
+
+    # Define the action when the question is submitted
+    question.submit(add_text, [chatbot, question], [chatbot, question], queue=False).then(
+        bot, chatbot, chatbot
+    )
+    # Define the action for the clear button
+    clear.click(lambda: None, None, chatbot, queue=False)
+
+# Launch the Gradio demo interface
+demo.launch(share=False, debug=True)

core-langchain-rag.py

@@ -0,0 +1,315 @@
+# Importing necessary libraries
+import sys
+import os
+import time
+
+# # Importing RecursiveUrlLoader for web scraping and BeautifulSoup for HTML parsing
+# from langchain.document_loaders.recursive_url_loader import RecursiveUrlLoader
+# from bs4 import BeautifulSoup as Soup
+# import mimetypes
+
+# # List of URLs to scrape
+# urls = ["https://langchain-doc.readthedocs.io/en/latest"]
+
+# # Initialize an empty list to store the documents
+# docs = []
+
+# # Looping through each URL in the list - this could take some time!
+# stf = time.time()  # Start time for performance measurement
+# for url in urls:
+#     try:
+#         st = time.time()  # Start time for performance measurement
+#         # Create a RecursiveUrlLoader instance with a specified URL and depth
+#         # The extractor function uses BeautifulSoup to parse the HTML content and extract text
+#         loader = RecursiveUrlLoader(url=url, max_depth=5, extractor=lambda x: Soup(x, "html.parser").text)
+        
+#         # Load the documents from the URL and extend the docs list
+#         docs.extend(loader.load())
+
+#         et = time.time() - st  # Calculate time taken for splitting
+#         print(f'Time taken for downloading documents from {url}: {et} seconds.')
+#     except Exception as e:
+#         # Print an error message if there is an issue with loading or parsing the URL
+#         print(f"Failed to load or parse the URL {url}. Error: {e}", file=sys.stderr)
+# etf = time.time() - stf  # Calculate time taken for splitting
+# print(f'Total time taken for downloading {len(docs)} documents: {etf} seconds.')
+
+# # Import necessary modules for text splitting and vectorization
+# from langchain.text_splitter import RecursiveCharacterTextSplitter
+# import time
+# from langchain_community.vectorstores import FAISS
+# from langchain.vectorstores.utils import filter_complex_metadata
+# from langchain_community.embeddings import HuggingFaceEmbeddings
+
+# # Configure the text splitter
+# text_splitter = RecursiveCharacterTextSplitter(
+#     separators=["\n\n", "\n", "(?<=\. )", " ", ""],  # Define the separators for splitting text
+#     chunk_size=500,  # The size of each text chunk
+#     chunk_overlap=50,  # Overlap between chunks to ensure continuity
+#     length_function=len,  # Function to determine the length of each chunk
+# )
+
+# try:
+#     # Stage one: Splitting the documents into chunks for vectorization
+#     st = time.time()  # Start time for performance measurement
+#     print('Loading documents and creating chunks ...')
+#     # Split each document into chunks using the configured text splitter
+#     chunks = text_splitter.create_documents([doc.page_content for doc in docs], metadatas=[doc.metadata for doc in docs])
+#     et = time.time() - st  # Calculate time taken for splitting
+#     print(f"created "+chunks+" chunks")
+#     print(f'Time taken for document chunking: {et} seconds.')
+# except Exception as e:
+#     print(f"Error during document chunking: {e}", file=sys.stderr)
+
+# # Path for saving the FAISS index
+# FAISS_INDEX_PATH = "./vectorstore/lc-faiss-multi-mpnet-500"
+
+# try:
+#     # Stage two: Vectorization of the document chunks
+#     model_name = "sentence-transformers/multi-qa-mpnet-base-dot-v1"  # Model used for embedding
+
+#     # Initialize HuggingFace embeddings with the specified model
+#     embeddings = HuggingFaceEmbeddings(model_name=model_name)
+
+#     print(f'Loading chunks into vector store ...')
+#     st = time.time()  # Start time for performance measurement
+#     # Create a FAISS vector store from the document chunks and save it locally
+#     db = FAISS.from_documents(filter_complex_metadata(chunks), embeddings)
+#     db.save_local(FAISS_INDEX_PATH)
+#     et = time.time() - st  # Calculate time taken for vectorization
+#     print(f'Time taken for vectorization and saving: {et} seconds.')
+# except Exception as e:
+#     print(f"Error during vectorization or FAISS index saving: {e}", file=sys.stderr)
+
+# alternatively download a preparaed vectorized index from S3 and load the index into vectorstore
+# Import necessary libraries for AWS S3 interaction, file handling, and FAISS vector stores
+import boto3
+from botocore import UNSIGNED
+from botocore.client import Config
+import zipfile
+from langchain_community.vectorstores import FAISS
+from langchain_huggingface import HuggingFaceEmbeddings
+from dotenv import load_dotenv
+
+# Load environment variables from a .env file
+config = load_dotenv(".env")
+
+# Retrieve the Hugging Face API token from environment variables
+HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN')
+S3_LOCATION = os.getenv("S3_LOCATION")
+S3_FILE_NAME = os.getenv("FAISS_VS_NAME")
+FAISS_INDEX_PATH = os.getenv("FAISS_INDEX_PATH")
+
+# try:
+#     # Initialize an S3 client with unsigned configuration for public access
+#     s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED))
+
+#     # Define the FAISS index path and the destination for the downloaded file
+#     #FAISS_INDEX_PATH = './vectorstore/lc-faiss-multi-mpnet-500-markdown'
+#     VS_DESTINATION = FAISS_INDEX_PATH + ".zip"
+
+#     # Download the pre-prepared vectorized index from the S3 bucket
+#     print("Downloading the pre-prepared vectorized index from S3...")
+#     s3.download_file(S3_LOCATION, S3_FILE_NAME, VS_DESTINATION)
+
+#     # Extract the downloaded zip file
+#     with zipfile.ZipFile(VS_DESTINATION, 'r') as zip_ref:
+#         zip_ref.extractall('./vectorstore/')
+#     print("Download and extraction completed.")
+    
+# except Exception as e:
+#     print(f"Error during downloading or extracting from S3: {e}", file=sys.stderr)
+
+# Define the model name for embeddings
+model_name = os.getenv("EMBEDDING_MODEL")
+
+try:
+    # Initialize HuggingFace embeddings with the specified model
+    embeddings = HuggingFaceEmbeddings(model_name=model_name)
+
+    # Load the local FAISS index with the specified embeddings
+    db = FAISS.load_local(FAISS_INDEX_PATH, embeddings, allow_dangerous_deserialization=True)
+    print("FAISS index loaded successfully.")
+except Exception as e:
+    print(f"Error during FAISS index loading: {e}", file=sys.stderr)
+
+# Import necessary modules for environment variable management and HuggingFace integration
+from langchain_huggingface import HuggingFaceEndpoint
+
+# Initialize the vector store as a retriever for the RAG pipeline
+retriever = db.as_retriever()#search_type="mmr", search_kwargs={'k': 3, 'lambda_mult': 0.25})
+
+llm_model = os.getenv("LLM_MODEL")
+
+try:
+    # Load the model from the Hugging Face Hub
+    model_id = HuggingFaceEndpoint(repo_id=llm_model, 
+        temperature=0.1,         # Controls randomness in response generation (lower value means less random)
+        max_new_tokens=1024,     # Maximum number of new tokens to generate in responses
+        repetition_penalty=1.2,  # Penalty for repeating the same words (higher value increases penalty)
+        return_full_text=False   # If False, only the newly generated text is returned; if True, the input is included as well
+    )
+    print("Model loaded successfully from Hugging Face Hub.")
+except Exception as e:
+    print(f"Error loading model from Hugging Face Hub: {e}", file=sys.stderr)
+
+
+
+# Importing necessary modules for retrieval-based question answering and prompt handling
+from langchain.chains import RetrievalQA
+from langchain.chains import LLMChain
+from langchain_core.prompts import PromptTemplate
+from langchain.memory import ConversationBufferMemory
+from langchain_core.output_parsers import StrOutputParser
+
+# Declare a global variable 'qa' for the retrieval-based question answering system
+global qa
+
+# Define a prompt template for guiding the model's responses
+template = """
+You are a friendly insurance product advisor, your task is to help customers find the best products from Württembergische GmbH.\
+  You help the user find the answers to all his questions queries. Answer in short and simple terms and offer to explain the product and terms to the user.\
+    Use the following context (delimited by <ctx></ctx>) and the chat history (delimited by <hs></hs>) to help find the best product for the user:
+------
+<ctx>
+{context}
+</ctx>
+------
+<hs>
+{history}
+</hs>
+------
+{question}
+Answer:
+"""
+
+# Create a PromptTemplate object with specified input variables and the defined template
+prompt = PromptTemplate.from_template(
+    #input_variables=["history", "context", "question"],  # Variables to be included in the prompt
+    template=template,  # The prompt template as defined above
+)
+prompt.format(context="context", history="history", question="question")
+# Create a memory buffer to manage conversation history
+memory = ConversationBufferMemory(
+    memory_key="history",  # Key for storing the conversation history
+    input_key="question"  # Key for the input question
+)
+
+# Initialize the RetrievalQA object with the specified model, retriever, and additional configurations
+qa = RetrievalQA.from_chain_type(
+    llm=model_id,  # Language model loaded from Hugging Face Hub
+    retriever=retriever,  # The vector store retriever initialized earlier
+    return_source_documents=True,  # Option to return source documents along with responses
+    chain_type_kwargs={
+        "verbose": True,  # Enables verbose output for debugging and analysis
+        "memory": memory,  # Memory buffer for managing conversation history
+        "prompt": prompt  # Prompt template for guiding the model's responses
+    }
+)
+
+def generate_qa_retriever(history: dict, question: str, llm_model:HuggingFaceEndpoint = model_id) -> dict:
+    """ Generare a response to queries using the retriever"""
+
+    # Define a prompt template for guiding the model's responses
+    template = """
+    You are a friendly insurance product advisor, your task is to help customers find the best products from Württembergische GmbH.\
+    You help the user find the answers to all his questions. Answer in short and simple terms and offer to explain the product and terms to the user.\
+    Respond only using the provided context (delimited by <ctx></ctx>) and only in German or Englisch, depending on the question's language.
+    Use the chat history (delimited by <hs></hs>) to help find the best product for the user:
+    ------
+    <ctx>
+    {context}
+    </ctx>
+    ------
+    <hs>
+    {history}
+    </hs>
+    ------
+    {question}
+    Answer:
+    """
+
+    # Create a PromptTemplate object with specified input variables and the defined template
+    prompt = PromptTemplate.from_template(
+        template=template,  # The prompt template as defined above
+    )
+    prompt.format(context="context", history="history", question="question")
+    # Create a memory buffer to manage conversation history
+    memory = ConversationBufferMemory(
+        memory_key="history",  # Key for storing the conversation history
+        input_key="question"  # Key for the input question
+    )
+    
+    llm_chain = prompt | llm_model
+    result = llm_chain.invoke({"context": retriever, "history": history, "question": question})
+    print(result)
+    return result
+
+# Import Gradio for UI, along with other necessary libraries
+import gradio as gr
+
+# Function to add a new input to the chat history
+def add_text(history, text):
+    # Append the new text to the history with a placeholder for the response
+    history = history + [(text, None)]
+    return history, ""
+
+# Function representing the bot's response mechanism
+def bot(history):
+    # Obtain the response from the 'infer' function using the latest input
+    response = infer(history[-1][0], history)
+    sources = [doc.metadata.get("source") for doc in response['source_documents']]
+    src_list = '\n'.join(sources)
+    print_this = response['result'] + "\n\n\n Sources: \n\n\n" + src_list
+
+
+    #history[-1][1] = response #print_this #response['answer']
+    history[-1][1] = print_this #response['answer']
+    # Update the history with the bot's response
+    #history[-1][1] = response['result']
+    return history
+
+# Function to infer the response using the RAG model
+def infer(question, history):
+    # Use the question and history to query the RAG model
+    #result = generate_qa_retriever(history, question)
+    result = qa({"query": question, "history": history, "question": question})
+    print(*result)
+    return result
+
+# CSS styling for the Gradio interface
+css = """
+#col-container {max-width: 700px; margin-left: auto; margin-right: auto;}
+"""
+
+# HTML content for the Gradio interface title
+title = """
+<div style="text-align:left;">
+    <p>Hello, I BotTina 2.0, your intelligent AI assistant. I can help you explore Wuerttembergische Versicherungs products.<br />
+</div>
+"""
+
+# Building the Gradio interface
+with gr.Blocks(theme=gr.themes.Soft()) as demo:
+    with gr.Column(elem_id="col-container"):
+        gr.HTML(title)  # Add the HTML title to the interface
+        chatbot = gr.Chatbot([], elem_id="chatbot",
+                                    label="BotTina 2.0",
+                                     bubble_full_width=False,
+                                     avatar_images=(None, "https://dacodi-production.s3.amazonaws.com/store/87bc00b6727589462954f2e3ff6f531c.png"),
+                                     height=680,)  # Initialize the chatbot component
+        clear = gr.Button("Clear")  # Add a button to clear the chat
+
+        # Create a row for the question input
+        with gr.Row():
+            question = gr.Textbox(label="Question", placeholder="Type your question and hit Enter ")
+
+    # Define the action when the question is submitted
+    question.submit(add_text, [chatbot, question], [chatbot, question], queue=False).then(
+        bot, chatbot, chatbot
+    )
+    # Define the action for the clear button
+    clear.click(lambda: None, None, chatbot, queue=False)
+
+# Launch the Gradio demo interface
+demo.launch(debug=True)

docs/template.md

@@ -0,0 +1,16 @@
+# Template
+
+1. **Architecture of advanced RAG**
+2. **Test setup and Evaluation metrics**
+3. **Data preparation (vectorization & chunking)**
+4. **Search indexing**
+5. **Query transformation**
+6. **Chat logic and query routing**
+7. **Multi agents and agent-services**
+8. **Monitoring responses and adding security**
+
+## Additional Resources
+
+[Enterprise Rag](https://www.rungalileo.io/blog/mastering-rag-how-to-architect-an-enterprise-rag-systemhttps://www.rungalileo.io/blog/mastering-rag-how-to-architect-an-enterprise-rag-system)
+
+[Advanced RAG](https://medium.com/towards-artificial-intelligence/advanced-rag-techniques-an-illustrated-overview-04d193d8fec6)

docs/workflow-advanced-rag.drawio

@@ -0,0 +1,83 @@
+<mxfile host="app.diagrams.net" modified="2024-02-02T11:21:08.029Z" agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0" etag="EvpGiXuqtWkE4FAqL8_g" version="22.1.21" type="github">
+  <diagram id="C5RBs43oDa-KdzZeNtuy" name="Page-1">
+    <mxGraphModel dx="1434" dy="774" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
+      <root>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-0" />
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-1" parent="WIyWlLk6GJQsqaUBKTNV-0" />
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-1" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-3" target="eFb6EC-VP60E3mpf6WAh-0">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-3" value="User Authentication and Input" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#d5e8d4;strokeColor=#82b366;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
+          <mxGeometry x="24" y="80" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-12" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-7" target="WIyWlLk6GJQsqaUBKTNV-11">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-7" value="Query Processing" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#d5e8d4;strokeColor=#82b366;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
+          <mxGeometry x="180" y="160" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-13" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-11" target="eFb6EC-VP60E3mpf6WAh-3">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="WIyWlLk6GJQsqaUBKTNV-11" value="Data Preparation and Management" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#ffe6cc;strokeColor=#d79b00;" parent="WIyWlLk6GJQsqaUBKTNV-1" vertex="1">
+          <mxGeometry x="330" y="160" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-11" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="eFb6EC-VP60E3mpf6WAh-0" target="WIyWlLk6GJQsqaUBKTNV-7">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-0" value="Input Guardrails" style="whiteSpace=wrap;html=1;rounded=1;glass=0;strokeWidth=1;shadow=0;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="24" y="160" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-14" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="eFb6EC-VP60E3mpf6WAh-3" target="eFb6EC-VP60E3mpf6WAh-5">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-3" value="Retrieval System" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="480" y="160" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-15" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="eFb6EC-VP60E3mpf6WAh-5" target="eFb6EC-VP60E3mpf6WAh-9">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-18" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0;exitDx=0;exitDy=0;entryX=0.75;entryY=1;entryDx=0;entryDy=0;dashed=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="eFb6EC-VP60E3mpf6WAh-5" target="eFb6EC-VP60E3mpf6WAh-6">
+          <mxGeometry relative="1" as="geometry">
+            <Array as="points">
+              <mxPoint x="630" y="130" />
+              <mxPoint x="450" y="130" />
+            </Array>
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-5" value="Information Processing and Augmentation" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="630" y="160" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-6" value="Observability and Feedback" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="360" y="40" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-21" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.98;exitY=0.02;exitDx=0;exitDy=0;exitPerimeter=0;entryX=0.575;entryY=1;entryDx=0;entryDy=0;dashed=1;entryPerimeter=0;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="eFb6EC-VP60E3mpf6WAh-8" target="eFb6EC-VP60E3mpf6WAh-3">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-8" value="Caching" style="strokeWidth=2;html=1;shape=mxgraph.flowchart.multi-document;whiteSpace=wrap;fillColor=#ffe6cc;strokeColor=#d79b00;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="376" y="240" width="88" height="60" as="geometry" />
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-16" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0;exitDx=0;exitDy=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;dashed=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="eFb6EC-VP60E3mpf6WAh-9" target="eFb6EC-VP60E3mpf6WAh-6">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-9" value="Output and Response" style="rounded=1;whiteSpace=wrap;html=1;fontSize=12;glass=0;strokeWidth=1;shadow=0;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="WIyWlLk6GJQsqaUBKTNV-1">
+          <mxGeometry x="630" y="80" width="120" height="40" as="geometry" />
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-17" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0;exitDx=0;exitDy=0;entryX=0.325;entryY=0.975;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="eFb6EC-VP60E3mpf6WAh-0" target="eFb6EC-VP60E3mpf6WAh-6">
+          <mxGeometry relative="1" as="geometry">
+            <Array as="points">
+              <mxPoint x="144" y="130" />
+              <mxPoint x="399" y="130" />
+            </Array>
+          </mxGeometry>
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-19" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.75;exitY=1;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="WIyWlLk6GJQsqaUBKTNV-7" target="eFb6EC-VP60E3mpf6WAh-8">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+        <mxCell id="eFb6EC-VP60E3mpf6WAh-20" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;entryPerimeter=0;dashed=1;" edge="1" parent="WIyWlLk6GJQsqaUBKTNV-1" source="eFb6EC-VP60E3mpf6WAh-9" target="eFb6EC-VP60E3mpf6WAh-8">
+          <mxGeometry relative="1" as="geometry" />
+        </mxCell>
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>

example.env

@@ -0,0 +1,15 @@
+# API Keys for services
+HUGGINGFACEHUB_API_TOKEN=""
+GOOGLE_CSE_ID=""
+GOOGLE_API_KEY=""
+
+# AWS S3 object storage 
+S3_LOCATION=""
+S3_FILE_NAME=""
+
+# Local vectorstore storage
+FAISS_INDEX_PATH = ""
+
+# llm and embedding models
+embedding_model=""
+llm_model=""

mail_automation_draft.drawio

@@ -1,140 +0,0 @@
-<mxfile host="app.diagrams.net" modified="2024-05-21T13:40:30.482Z" agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0" etag="dPw-sfjJEinoEt61k41I" version="24.4.4" type="github">
-  <diagram name="Page-1" id="JIRIjCkc2eHKuGLTyx-l">
-    <mxGraphModel dx="1434" dy="-1564" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
-      <root>
-        <mxCell id="0" />
-        <mxCell id="1" parent="0" />
-        <mxCell id="0ipz06AKfEBxF18oT8nj-1" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;" parent="1" source="0ipz06AKfEBxF18oT8nj-19" target="0ipz06AKfEBxF18oT8nj-6" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-2" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-3" target="0ipz06AKfEBxF18oT8nj-31" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-3" value="Ticket Creation" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="217.5" y="2640" width="120" height="80" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-4" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-6" target="0ipz06AKfEBxF18oT8nj-8" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-5" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-6" target="0ipz06AKfEBxF18oT8nj-21" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-6" value="Fetch Emails" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="210.5" y="2480" width="120" height="80" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-7" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-8" target="0ipz06AKfEBxF18oT8nj-10" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-8" value="Entity Recognition" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="410.5" y="2480" width="120" height="80" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-9" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-10" target="0ipz06AKfEBxF18oT8nj-14" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-10" value="Classification &amp;amp; Summarization" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="630" y="2480" width="120" height="80" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-11" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=1;exitDx=0;exitDy=0;" parent="1" source="0ipz06AKfEBxF18oT8nj-17" target="0ipz06AKfEBxF18oT8nj-15" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-12" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-14" target="0ipz06AKfEBxF18oT8nj-17" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-13" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-14" target="0ipz06AKfEBxF18oT8nj-24" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-14" value="Data Enrichment" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="630" y="2640" width="120" height="80" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-15" value="Draft Response" style="ellipse;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" parent="1" vertex="1">
-          <mxGeometry x="410.5" y="2800" width="120" height="80" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-16" value="" style="curved=1;endArrow=classic;html=1;rounded=0;exitX=0.75;exitY=1;exitDx=0;exitDy=0;entryX=0.25;entryY=1;entryDx=0;entryDy=0;" parent="1" source="0ipz06AKfEBxF18oT8nj-19" target="0ipz06AKfEBxF18oT8nj-19" edge="1">
-          <mxGeometry width="50" height="50" relative="1" as="geometry">
-            <mxPoint x="200" y="2730" as="sourcePoint" />
-            <mxPoint x="80" y="2580" as="targetPoint" />
-            <Array as="points">
-              <mxPoint x="130" y="2600" />
-              <mxPoint x="90" y="2610" />
-              <mxPoint x="60" y="2620" />
-              <mxPoint x="53" y="2570" />
-            </Array>
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-17" value="Data Complete" style="rhombus;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="430.5" y="2640" width="80" height="80" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-18" value="No" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;" parent="1" vertex="1">
-          <mxGeometry x="460.5" y="2748" width="40" height="30" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-19" value="Periodically Check Emails" style="shape=process;whiteSpace=wrap;html=1;backgroundOutline=1;" parent="1" vertex="1">
-          <mxGeometry x="30" y="2490" width="120" height="60" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-20" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-21" target="0ipz06AKfEBxF18oT8nj-22" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-21" value="Download Attachements" style="whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="210.5" y="2360" width="120" height="60" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-22" value="Extract Text and Images from PDF" style="whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="399.5" y="2360" width="120" height="60" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-23" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.596;exitY=1.033;exitDx=0;exitDy=0;exitPerimeter=0;" parent="1" source="0ipz06AKfEBxF18oT8nj-22" target="0ipz06AKfEBxF18oT8nj-8" edge="1">
-          <mxGeometry relative="1" as="geometry">
-            <mxPoint x="281" y="2490" as="sourcePoint" />
-            <mxPoint x="281" y="2430" as="targetPoint" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-24" value="CRM, SQL, Product KB" style="ellipse;whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="650" y="2800" width="80" height="80" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-25" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;entryX=1;entryY=0.5;entryDx=0;entryDy=0;" parent="1" source="0ipz06AKfEBxF18oT8nj-17" target="0ipz06AKfEBxF18oT8nj-3" edge="1">
-          <mxGeometry relative="1" as="geometry">
-            <mxPoint x="620.5" y="2690" as="sourcePoint" />
-            <mxPoint x="520.5" y="2690" as="targetPoint" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-26" value="Yes" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" parent="0ipz06AKfEBxF18oT8nj-25" vertex="1" connectable="0">
-          <mxGeometry x="-0.1183" y="-5" relative="1" as="geometry">
-            <mxPoint as="offset" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-27" value="" style="endArrow=classic;html=1;rounded=0;exitX=0.5;exitY=1;exitDx=0;exitDy=0;entryX=1;entryY=0;entryDx=0;entryDy=0;" parent="1" source="0ipz06AKfEBxF18oT8nj-15" target="0ipz06AKfEBxF18oT8nj-8" edge="1">
-          <mxGeometry width="50" height="50" relative="1" as="geometry">
-            <mxPoint x="520" y="2980" as="sourcePoint" />
-            <mxPoint x="570" y="2930" as="targetPoint" />
-            <Array as="points">
-              <mxPoint x="470" y="2930" />
-              <mxPoint x="770" y="2930" />
-              <mxPoint x="770" y="2690" />
-              <mxPoint x="770" y="2450" />
-              <mxPoint x="513" y="2450" />
-            </Array>
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-28" value="Mail / WhatsApp /..." style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" parent="0ipz06AKfEBxF18oT8nj-27" vertex="1" connectable="0">
-          <mxGeometry x="-0.6717" y="3" relative="1" as="geometry">
-            <mxPoint as="offset" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-29" value="" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;" parent="1" source="0ipz06AKfEBxF18oT8nj-31" target="0ipz06AKfEBxF18oT8nj-32" edge="1">
-          <mxGeometry relative="1" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-30" value="slack / discord / teams" style="edgeLabel;html=1;align=center;verticalAlign=middle;resizable=0;points=[];" parent="0ipz06AKfEBxF18oT8nj-29" vertex="1" connectable="0">
-          <mxGeometry x="-0.3" y="2" relative="1" as="geometry">
-            <mxPoint as="offset" />
-          </mxGeometry>
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-31" value="Fetch Relevant historical data &amp;amp; BP" style="whiteSpace=wrap;html=1;" parent="1" vertex="1">
-          <mxGeometry x="217.5" y="2810" width="120" height="60" as="geometry" />
-        </mxCell>
-        <mxCell id="0ipz06AKfEBxF18oT8nj-32" value="Recommendations to Agent" style="whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" parent="1" vertex="1">
-          <mxGeometry x="217.5" y="2950" width="120" height="60" as="geometry" />
-        </mxCell>
-        <mxCell id="1TZoFbZY8AwqjJc08LRQ-1" value="email, customer name, contract id, date, category, summary, issue category,T&amp;amp;C, Process descriptions, customer history, issue summary" style="text;html=1;align=center;verticalAlign=middle;resizable=0;points=[];autosize=1;strokeColor=none;fillColor=none;" vertex="1" parent="1">
-          <mxGeometry x="265" y="3028" width="770" height="30" as="geometry" />
-        </mxCell>
-      </root>
-    </mxGraphModel>
-  </diagram>
-</mxfile>

rag_app/create_embedding.py

@@ -0,0 +1,50 @@
+# embeddings functions
+#from langchain_community.vectorstores import FAISS
+#from langchain_community.document_loaders import ReadTheDocsLoader
+#from langchain_community.vectorstores.utils import filter_complex_metadata
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_huggingface import HuggingFaceEmbeddings
+import time
+from langchain_core.documents import Document
+
+
+def create_embeddings(
+        docs: list[Document], 
+        chunk_size:int = 500, 
+        chunk_overlap:int = 50,
+        embedding_model: str = "sentence-transformers/multi-qa-mpnet-base-dot-v1", 
+        ):
+    """given a sequence of `Document` objects this fucntion will
+    generate embeddings for it.
+    
+    ## argument
+    :params docs (list[Document]) -> list of `list[Document]`
+    :params chunk_size (int) -> chunk size in which documents are chunks, defaults to 500
+    :params chunk_overlap (int) -> the amount of token that will be overlapped between chunks, defaults to 50
+    :params embedding_model (str) -> the huggingspace model that will embed the documents 
+    ## Return
+    Tuple of embedding and chunks
+    """
+    
+    
+    text_splitter = RecursiveCharacterTextSplitter(
+        separators=["\n\n", "\n", "(?<=\. )", " ", ""],
+        chunk_size = chunk_size,
+        chunk_overlap  = chunk_overlap,
+        length_function = len,
+    )
+
+    # Stage one: read all the docs, split them into chunks.
+    st = time.time()
+    print('Loading documents and creating chunks ...')
+
+    # Split each document into chunks using the configured text splitter
+    chunks = text_splitter.create_documents([doc.page_content for doc in docs], metadatas=[doc.metadata for doc in docs])
+    et = time.time() - st
+    print(f'Time taken to chunk {len(docs)} documents: {et} seconds.')
+
+    #Stage two: embed the docs.
+    embeddings = HuggingFaceEmbeddings(model_name=embedding_model)
+    print(f"created a total of {len(chunks)} chunks")
+
+    return embeddings,chunks

rag_app/generate_summary.py

@@ -0,0 +1,69 @@
+from langchain_huggingface import HuggingFaceEndpoint
+from langchain_core.prompts import PromptTemplate
+from langchain_core.output_parsers import StrOutputParser
+import json
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+
+HF_API_TOKEN = os.getenv('HUGGINGFACE_API_TOKEN')
+model_id=os.getenv('LLM_MODEL')    
+
+LLM = HuggingFaceEndpoint(
+repo_id=model_id, 
+temperature=0.1, 
+max_new_tokens=512,
+repetition_penalty=1.2,
+return_full_text=False,
+huggingfacehub_api_token=HF_API_TOKEN) 
+
+def generate_keywords(document:dict,
+                     llm_model:HuggingFaceEndpoint = LLM) -> str:
+    """ Generate a meaningful list of meta keywords for the provided document or chunk"""
+    
+    template = (
+        """
+        You are a SEO expert bot. Your task is to craft a meaningful list of 5 keywords to organize documents. 
+        The keywords should help us in searching and retrieving the documents later.
+
+        You will only respond with the clear, concise and meaningful 5 of keywords separated by comma. 
+          
+        <<<
+        Document: {document}
+        >>>
+
+        Keywords:
+        """
+    )
+
+    prompt = PromptTemplate.from_template(template=template)
+    
+    chain = prompt | llm_model | StrOutputParser()
+    result = chain.invoke({'document': document})
+    return result.strip()
+
+def generate_description(document:dict,
+                     llm_model:HuggingFaceEndpoint = LLM) -> str:
+    """ Generate a meaningful document description based on document content """
+    
+    template = (
+        """
+        You are a SEO expert bot. Your task is to craft a meaningful summary to descripe and organize documents. 
+        The description should be a meaningful summary of the document's content and help us in searching and retrieving the documents later.
+
+        You will only respond with the clear, concise and meaningful description. 
+          
+        <<<
+        Document: {document}
+        >>>
+
+        Description: 
+        """
+    )
+
+    prompt = PromptTemplate.from_template(template=template)
+    
+    chain = prompt | llm_model | StrOutputParser()
+    result = chain.invoke({'document': document})
+    return result.strip()

rag_app/get_db_retriever.py

@@ -0,0 +1,29 @@
+# retriever and qa_chain function
+
+# HF libraries
+from langchain.llms import HuggingFaceHub
+from langchain.embeddings import HuggingFaceHubEmbeddings
+# vectorestore
+from langchain.vectorstores import FAISS
+# retrieval chain
+from langchain.chains import RetrievalQA
+# prompt template
+from langchain.prompts import PromptTemplate
+from langchain.memory import ConversationBufferMemory
+
+
+def get_db_retriever(vector_db:str=None):
+    model_name = "sentence-transformers/multi-qa-mpnet-base-dot-v1"
+    embeddings = HuggingFaceHubEmbeddings(repo_id=model_name)
+
+    #db = Chroma(persist_directory="./vectorstore/lc-chroma-multi-mpnet-500", embedding_function=embeddings)
+    #db.get()
+    if not vector_db:
+        FAISS_INDEX_PATH='./vectorstore/py-faiss-multi-mpnet-500'
+    else:
+        FAISS_INDEX_PATH=vector_db
+    db = FAISS.load_local(FAISS_INDEX_PATH, embeddings)
+
+    retriever = db.as_retriever()
+
+    return retriever

rag_app/handle_vector_store.py

@@ -0,0 +1,82 @@
+# vectorization functions
+from langchain_community.vectorstores import FAISS
+from langchain_community.vectorstores import Chroma
+from langchain_community.document_loaders import ReadTheDocsLoader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.retrievers import BM25Retriever
+from rag_app.create_embedding import create_embeddings
+from rag_app.generate_summary import generate_description, generate_keywords
+import time
+import os
+from dotenv import load_dotenv
+
+def build_vector_store(
+        docs: list, 
+        db_path: str, 
+        embedding_model: str, 
+        new_db:bool=False, 
+        chunk_size:int=500, 
+        chunk_overlap:int=50,
+        ):
+    """
+
+    """
+
+    if db_path is None:
+        FAISS_INDEX_PATH = os.getenv("FAISS_INDEX_PATH")
+    else:
+        FAISS_INDEX_PATH = db_path
+
+    embeddings,chunks = create_embeddings(docs, chunk_size, chunk_overlap, embedding_model)
+    for chunk in chunks:
+        keywords=generate_keywords(chunk)
+        description=generate_description(chunk)
+        chunk.metadata['chunk_keywords']=keywords
+        chunk.metadata['chunk_description']=description
+
+    #load chunks into vector store
+    print(f'Loading chunks into faiss vector store ...')
+    st = time.time()
+    if new_db:
+        db_faiss = FAISS.from_documents(chunks, embeddings)
+        bm25_retriever = BM25Retriever.from_documents(chunks)
+    else:
+        db_faiss = FAISS.add_documents(chunks, embeddings)
+        bm25_retriever = BM25Retriever.add_documents(chunks)
+    db_faiss.save_local(FAISS_INDEX_PATH)
+    et = time.time() - st
+    print(f'Time taken: {et} seconds.')
+
+    print(f'Loading chunks into chroma vector store ...')
+    st = time.time()
+    persist_directory='./vectorstore/chroma-insurance-agent-1500'
+    db_chroma = Chroma.from_documents(chunks, embeddings, persist_directory=persist_directory)
+    et = time.time() - st
+    print(f'Time taken: {et} seconds.')
+    result = f"built vectore store at {FAISS_INDEX_PATH}"
+    return result
+
+
+# # Path for saving the FAISS index
+# FAISS_INDEX_PATH = "./vectorstore/lc-faiss-multi-mpnet-500"
+
+# try:
+#     # Stage two: Vectorization of the document chunks
+#     model_name = "sentence-transformers/multi-qa-mpnet-base-dot-v1"  # Model used for embedding
+
+#     # Initialize HuggingFace embeddings with the specified model
+#     embeddings = HuggingFaceEmbeddings(model_name=model_name)
+
+#     print(f'Loading chunks into vector store ...')
+#     st = time.time()  # Start time for performance measurement
+#     # Create a FAISS vector store from the document chunks and save it locally
+#     db = FAISS.from_documents(filter_complex_metadata(chunks), embeddings)
+#     db.save_local(FAISS_INDEX_PATH)
+#     et = time.time() - st  # Calculate time taken for vectorization
+#     print(f'Time taken for vectorization and saving: {et} seconds.')
+# except Exception as e:
+#     print(f"Error during vectorization or FAISS index saving: {e}", file=sys.stderr)
+
+# alternatively download a preparaed vectorized index from S3 and load the index into vectorstore
+# Import necessary libraries for AWS S3 interaction, file handling, and FAISS vector stores

rag_app/load_data_from_urls.py

@@ -0,0 +1,50 @@
+# documents loader function
+from langchain_community.document_loaders import RecursiveUrlLoader
+from bs4 import BeautifulSoup as Soup
+from validators import url as url_validator
+from langchain_core.documents import Document
+import time
+import logging
+import sys
+
+logger = logging.getLogger(__name__)
+
+def load_docs_from_urls(
+        urls: list = ["https://docs.python.org/3/"], 
+        max_depth: int = 5,
+        ) -> list[Document]:
+    """
+    Load documents from a list of URLs.
+
+    ## Args:
+        urls (list, optional): A list of URLs to load documents from. Defaults to ["https://docs.python.org/3/"].
+        max_depth (int, optional): Maximum depth to recursively load documents from each URL. Defaults to 5.
+
+    ## Returns:
+        list: A list of documents loaded from the given URLs.
+        
+    ## Raises:
+        ValueError: If any URL in the provided list is invalid.
+    """
+    stf = time.time()  # Start time for performance measurement
+    docs = []
+    for url in urls:
+        st = time.time()  # Start time for outer performance measurement
+        if not url_validator(url):
+            raise ValueError(f"Invalid URL: {url}")
+        try:
+            st = time.time()  # Start time for inner performance measurement
+            loader = RecursiveUrlLoader(url=url, max_depth=max_depth, extractor=lambda x: Soup(x, "html.parser").text)
+            docs.extend(loader.load())
+
+            et = time.time() - st  # Calculate time taken for splitting
+            logMessage=f'Time taken for downloading documents from {url}: {et} seconds.'
+            logger.info(logMessage)
+            print(logMessage)
+        except Exception as e:
+            logMessage=f"Failed to load or parse the URL {url}. Error: {e}"
+            logger.error(logMessage)
+            print(logMessage, file=sys.stderr)
+    etf = time.time() - stf  # Calculate time taken for scrapping all URLs
+    print(f'Total time taken for downloading {len(docs)} documents: {etf} seconds.')
+    return docs

rag_app/load_vector_stores.py

@@ -0,0 +1,70 @@
+# preprocessed vectorstore retrieval
+import boto3
+from botocore import UNSIGNED
+from botocore.client import Config
+import zipfile
+from langchain_community.vectorstores import FAISS
+from langchain_community.vectorstores import Chroma
+from langchain_huggingface import HuggingFaceEmbeddings
+from dotenv import load_dotenv
+import os
+import sys
+import logging
+
+# Load environment variables from a .env file
+config = load_dotenv(".env")
+
+# Retrieve the Hugging Face API token from environment variables
+HUGGINGFACEHUB_API_TOKEN = os.getenv('HUGGINGFACEHUB_API_TOKEN')
+S3_LOCATION = os.getenv("S3_LOCATION")
+FAISS_VS_NAME = os.getenv("FAISS_VS_NAME")
+FAISS_INDEX_PATH = os.getenv("FAISS_INDEX_PATH")
+CHROMA_DIRECTORY = os.getenv("CHROMA_DIRECTORY")
+CHROMA_VS_NAME = os.getenv("CHROMA_VS_NAME")
+EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL")
+
+model_name = EMBEDDING_MODEL
+#model_kwargs = {"device": "cuda"}
+
+embeddings = HuggingFaceEmbeddings(
+    model_name=model_name,
+#    model_kwargs=model_kwargs
+    )
+
+## FAISS
+def get_faiss_vs():
+    # Initialize an S3 client with unsigned configuration for public access
+    s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED))
+
+    # Define the destination for the downloaded file
+    VS_DESTINATION = FAISS_INDEX_PATH + ".zip"
+    try:
+        # Download the pre-prepared vectorized index from the S3 bucket
+        print("Downloading the pre-prepared vectorized index from S3...")
+        s3.download_file(S3_LOCATION, FAISS_VS_NAME, VS_DESTINATION)
+
+        # Extract the downloaded zip file
+        with zipfile.ZipFile(VS_DESTINATION, 'r') as zip_ref:
+            zip_ref.extractall('./vectorstore/')
+        print("Download and extraction completed.")
+        return FAISS.load_local(FAISS_INDEX_PATH, embeddings)
+        
+    except Exception as e:
+        print(f"Error during downloading or extracting from S3: {e}", file=sys.stderr)
+    #faissdb = FAISS.load_local(FAISS_INDEX_PATH, embeddings)
+
+
+## Chroma DB
+def get_chroma_vs():
+    # Initialize an S3 client with unsigned configuration for public access
+    s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED))
+
+    VS_DESTINATION = CHROMA_DIRECTORY+".zip"
+    try:
+        s3.download_file(S3_LOCATION, CHROMA_VS_NAME, VS_DESTINATION)
+        with zipfile.ZipFile(VS_DESTINATION, 'r') as zip_ref:
+            zip_ref.extractall('./vectorstore/')
+        chromadb = Chroma(persist_directory=CHROMA_DIRECTORY, embedding_function=embeddings)
+        chromadb.get()
+    except Exception as e:
+        print(f"Error during downloading or extracting from S3: {e}", file=sys.stderr)

requirements.txt

@@ -0,0 +1,13 @@
+ langchain 
+ langchain-community 
+ langchain-HuggingFace
+ langchain-text-splitters
+ beautifulsoup4 
+ faiss-cpu 
+ chromadb 
+ validators 
+ sentence_transformers 
+ typing-extensions
+ unstructured 
+ gradio
+ boto3

test_this.py

@@ -0,0 +1,17 @@
+from rag_app.load_data_from_urls import load_docs_from_urls
+from rag_app.create_embedding import create_embeddings
+from rag_app.generate_summary import generate_description, generate_keywords
+from rag_app.handle_vector_store import build_vector_store
+
+docs = load_docs_from_urls(["https://www.wuerttembergische.de/"],5)
+
+for doc in docs:
+    keywords=generate_keywords(doc)
+    description=generate_description(doc)
+    doc.metadata['keywords']=keywords
+    doc.metadata['description']=description
+
+build_vector_store(docs, './vectorstore/faiss-insurance-agent-1500','sentence-transformers/multi-qa-mpnet-base-dot-v1',True,1500,150)
+
+
+#print(create_embeddings(docs))

vectorstore/placeholder.txt

@@ -0,0 +1 @@
+This file keeps the folder from being deleted for now