Update app.py

app.py

@@ -31,29 +31,77 @@ def expand_query(query):
     return refined_query if refined_query else query
 
 # 🔹 Hybrid Search (TF-IDF + Semantic Retrieval)
+# def hybrid_search(query, user_groups, index_name="briefmeta", min_score=0.01, fetch_k=50):
+#     vector_store = PineconeVectorStore(index_name=index_name, embedding=embeddings)
+#     semantic_results = vector_store.max_marginal_relevance_search(query, k=10, fetch_k=fetch_k)
+
+#     all_texts = [doc.page_content for doc in semantic_results]
+#     vectorizer = TfidfVectorizer(stop_words="english")
+#     tfidf_matrix = vectorizer.fit_transform(all_texts)
+#     query_tfidf = vectorizer.transform([query])
+#     keyword_scores = cosine_similarity(query_tfidf, tfidf_matrix).flatten()
+
+#     combined_results, seen_ids = [], set()
+#     for i, doc in enumerate(semantic_results):
+#         doc_id, doc_groups = doc.metadata.get("id"), doc.metadata.get("groups", [])
+#         semantic_score = float(doc.metadata.get("score", 0))
+#         keyword_score = float(keyword_scores[i])
+#         final_score = 0.65 * semantic_score + 0.35 * keyword_score  # Hybrid score
+
+#         if doc_id not in seen_ids and any(group in user_groups for group in doc_groups) and final_score > min_score:
+#             seen_ids.add(doc_id)
+#             doc.metadata["final_score"] = final_score
+#             combined_results.append(doc)
+
+#     combined_results.sort(key=lambda x: x.metadata["final_score"], reverse=True)
+#     return [
+#         {
+#             "doc_id": doc.metadata.get("doc_id", "N/A"),
+#             "chunk_id": doc.metadata.get("id", "N/A"),
+#             "title": doc.metadata.get("source", "N/A"),
+#             "text": doc.page_content,
+#             "page_number": str(doc.metadata.get("page_number", "N/A")),
+#             "score": str(doc.metadata.get("final_score", "N/A")),
+#         }
+#         for doc in combined_results
+    ]
+
 def hybrid_search(query, user_groups, index_name="briefmeta", min_score=0.01, fetch_k=50):
     vector_store = PineconeVectorStore(index_name=index_name, embedding=embeddings)
+
+    # **Filter chunks by user groups before retrieval**
+    filtered_results = vector_store.similarity_search(
+        query="",  # Empty query just to fetch all documents
+        k=fetch_k,
+        filter={"groups": {"$in": user_groups}},  # Filter for user-specific chunks
+    )
+
+    # **Perform Semantic Search on Filtered Results**
     semantic_results = vector_store.max_marginal_relevance_search(query, k=10, fetch_k=fetch_k)
 
+    # **TF-IDF Keyword Search**
     all_texts = [doc.page_content for doc in semantic_results]
     vectorizer = TfidfVectorizer(stop_words="english")
     tfidf_matrix = vectorizer.fit_transform(all_texts)
     query_tfidf = vectorizer.transform([query])
     keyword_scores = cosine_similarity(query_tfidf, tfidf_matrix).flatten()
 
+    # **Hybrid Score Calculation**
     combined_results, seen_ids = [], set()
     for i, doc in enumerate(semantic_results):
-        doc_id, doc_groups = doc.metadata.get("id"), doc.metadata.get("groups", [])
+        doc_id = doc.metadata.get("id")
         semantic_score = float(doc.metadata.get("score", 0))
         keyword_score = float(keyword_scores[i])
         final_score = 0.65 * semantic_score + 0.35 * keyword_score  # Hybrid score
 
-        if doc_id not in seen_ids and any(group in user_groups for group in doc_groups) and final_score > min_score:
+        if doc_id not in seen_ids and final_score > min_score:
             seen_ids.add(doc_id)
             doc.metadata["final_score"] = final_score
             combined_results.append(doc)
 
+    # **Sort Results by Final Score**
     combined_results.sort(key=lambda x: x.metadata["final_score"], reverse=True)
+
     return [
         {
             "doc_id": doc.metadata.get("doc_id", "N/A"),
@@ -66,6 +114,7 @@ def hybrid_search(query, user_groups, index_name="briefmeta", min_score=0.01, fe
         for doc in combined_results
     ]
 
+
 # 🔹 Metadata-Weighted Reranking
 def rerank(query, context):
     reranker = pc.inference.rerank(
@@ -75,9 +124,9 @@ def rerank(query, context):
     final_reranked = []
     for entry in reranker.data:
         doc, score = entry["document"], float(entry["score"])
-        citation_boost = 1.2 if "high_citations" in doc.get("tags", []) else 1.0
-        recency_boost = 1.1 if "recent_upload" in doc.get("tags", []) else 1.0
-        final_score = score * citation_boost * recency_boost
+        # citation_boost = 1.2 if "high_citations" in doc.get("tags", []) else 1.0
+        # recency_boost = 1.1 if "recent_upload" in doc.get("tags", []) else 1.0
+        # final_score = score * citation_boost * recency_boost
         doc["final_score"] = final_score
         final_reranked.append(doc)