Update app.py

app.py

@@ -66,53 +66,63 @@ def expand_query(query):
 #         for doc in combined_results
     # ]
 
-def hybrid_search(query, user_groups, index_name="briefmeta", min_score=0, fetch_k=50):
+def hybrid_search(query, user_groups, index_name="briefmeta", min_score=0.01, fetch_k=50):
     vector_store = PineconeVectorStore(index_name=index_name, embedding=embeddings)
 
-    # **Filter chunks by user groups before retrieval**
-    filtered_results = vector_store.similarity_search(
-        query="",  # Empty query just to fetch all documents
-        k=fetch_k,
-        filter={"groups": {"$in": user_groups}},  # Filter for user-specific chunks
-    )
-
-    # **Perform Semantic Search on Filtered Results**
-    semantic_results = vector_store.max_marginal_relevance_search(query, k=10, fetch_k=fetch_k)
-
-    # **TF-IDF Keyword Search**
-    all_texts = [doc.page_content for doc in semantic_results]
-    vectorizer = TfidfVectorizer(stop_words="english")
-    tfidf_matrix = vectorizer.fit_transform(all_texts)
-    query_tfidf = vectorizer.transform([query])
-    keyword_scores = cosine_similarity(query_tfidf, tfidf_matrix).flatten()
-
-    # **Hybrid Score Calculation**
-    combined_results, seen_ids = [], set()
-    for i, doc in enumerate(semantic_results):
-        doc_id = doc.metadata.get("id")
-        semantic_score = float(doc.metadata.get("score", 0))
-        keyword_score = float(keyword_scores[i])
-        final_score = 0.65 * semantic_score + 0.35 * keyword_score  # Hybrid score
-
-        if doc_id not in seen_ids and final_score > min_score:
-            seen_ids.add(doc_id)
-            doc.metadata["final_score"] = final_score
-            combined_results.append(doc)
-
-    # **Sort Results by Final Score**
-    combined_results.sort(key=lambda x: x.metadata["final_score"], reverse=True)
-
-    return [
-        {
-            "doc_id": doc.metadata.get("doc_id", "N/A"),
-            "chunk_id": doc.metadata.get("id", "N/A"),
-            "title": doc.metadata.get("source", "N/A"),
-            "text": doc.page_content,
-            "page_number": str(doc.metadata.get("page_number", "N/A")),
-            "score": str(doc.metadata.get("final_score", "N/A")),
-        }
-        for doc in combined_results
-    ]
+    try:
+    
+        filtered_results = vector_store.similarity_search(
+            query="",  # Empty query just to fetch all documents
+            k=fetch_k,
+            filter={"groups": {"$in": user_groups}},  # Filter for user-specific chunks
+        )
+    
+        if not filtered_results:
+            print("No results:")
+            return []
+        else:
+            print(filtered_results)
+            
+        # **2️⃣ Perform Semantic Search on the Filtered Set**
+        #semantic_results = vector_store.max_marginal_relevance_search(query, k=10, fetch_k=fetch_k)
+    
+        # **3️⃣ TF-IDF Keyword Search on Filtered Set**
+        all_texts = [doc.page_content for doc in filtered_results]  # Use filtered docs
+        vectorizer = TfidfVectorizer(stop_words="english")
+        tfidf_matrix = vectorizer.fit_transform(all_texts)
+        query_tfidf = vectorizer.transform([query])
+        keyword_scores = cosine_similarity(query_tfidf, tfidf_matrix).flatten()
+    
+        # **4️⃣ Hybrid Score Calculation**
+        combined_results, seen_ids = [], set()
+        for i, doc in enumerate(filtered_results):  # Iterate over filtered results
+            doc_id = doc.metadata.get("id")
+            semantic_score = float(doc.metadata.get("score", 0))
+            keyword_score = float(keyword_scores[i])
+            final_score = 0.65 * semantic_score + 0.35 * keyword_score  # Hybrid score
+    
+            if doc_id not in seen_ids and final_score > min_score:
+                seen_ids.add(doc_id)
+                doc.metadata["final_score"] = final_score
+                combined_results.append(doc)
+    
+        # **5️⃣ Sort Results by Final Score**
+        combined_results.sort(key=lambda x: x.metadata["final_score"], reverse=True)
+    
+        return [
+            {
+                "doc_id": doc.metadata.get("doc_id", "N/A"),
+                "chunk_id": doc.metadata.get("id", "N/A"),
+                "title": doc.metadata.get("source", "N/A"),
+                "text": doc.page_content,
+                "page_number": str(doc.metadata.get("page_number", "N/A")),
+                "score": str(doc.metadata.get("final_score", "N/A")),
+            }
+            for doc in combined_results
+        ]
+    except Exception as e:
+        print(e)
+        return
 
 # 🔹 Metadata-Weighted Reranking
 def rerank(query, context):