Update app.py

app.py

@@ -26,103 +26,45 @@ embeddings = VoyageAIEmbeddings(voyage_api_key=voyage_api_key, model="voyage-law
 # 🔹 Query Expansion using GPT-4
 def expand_query(query):
     llm = ChatOpenAI(model="gpt-4", openai_api_key=openai.api_key, temperature=0.3)
-    prompt = f"Rewrite this vague query for searching a document into a more specific one:\nQuery: {query}\nSpecific Query: If it is just 'docs'/'documents' do not rewrite"
+    prompt = f"Rewrite this vague query into a more specific one:\nQuery: {query}\nSpecific Query:"
     refined_query = llm([HumanMessage(content=prompt)]).content.strip()
     return refined_query if refined_query else query
 
 # 🔹 Hybrid Search (TF-IDF + Semantic Retrieval)
-# def hybrid_search(query, user_groups, index_name="briefmeta", min_score=0.01, fetch_k=50):
-#     vector_store = PineconeVectorStore(index_name=index_name, embedding=embeddings)
-#     semantic_results = vector_store.max_marginal_relevance_search(query, k=10, fetch_k=fetch_k)
-
-#     all_texts = [doc.page_content for doc in semantic_results]
-#     vectorizer = TfidfVectorizer(stop_words="english")
-#     tfidf_matrix = vectorizer.fit_transform(all_texts)
-#     query_tfidf = vectorizer.transform([query])
-#     keyword_scores = cosine_similarity(query_tfidf, tfidf_matrix).flatten()
-
-#     combined_results, seen_ids = [], set()
-#     for i, doc in enumerate(semantic_results):
-#         doc_id, doc_groups = doc.metadata.get("id"), doc.metadata.get("groups", [])
-#         semantic_score = float(doc.metadata.get("score", 0))
-#         keyword_score = float(keyword_scores[i])
-#         final_score = 0.65 * semantic_score + 0.35 * keyword_score  # Hybrid score
-
-#         if doc_id not in seen_ids and any(group in user_groups for group in doc_groups) and final_score > min_score:
-#             seen_ids.add(doc_id)
-#             doc.metadata["final_score"] = final_score
-#             combined_results.append(doc)
-
-#     combined_results.sort(key=lambda x: x.metadata["final_score"], reverse=True)
-#     return [
-#         {
-#             "doc_id": doc.metadata.get("doc_id", "N/A"),
-#             "chunk_id": doc.metadata.get("id", "N/A"),
-#             "title": doc.metadata.get("source", "N/A"),
-#             "text": doc.page_content,
-#             "page_number": str(doc.metadata.get("page_number", "N/A")),
-#             "score": str(doc.metadata.get("final_score", "N/A")),
-#         }
-#         for doc in combined_results
-    # ]
-
 def hybrid_search(query, user_groups, index_name="briefmeta", min_score=0.01, fetch_k=50):
     vector_store = PineconeVectorStore(index_name=index_name, embedding=embeddings)
-
-    try:
-    
-        filtered_results = vector_store.similarity_search(
-            query="",  # Empty query just to fetch all documents
-            k=fetch_k,
-            filter={"groups": {"$in": user_groups}},  # Filter for user-specific chunks
-        )
-    
-        if not filtered_results:
-            print("No results:")
-            return []
-        else:
-            print(filtered_results)
-            
-        # **2️⃣ Perform Semantic Search on the Filtered Set**
-        #semantic_results = vector_store.max_marginal_relevance_search(query, k=10, fetch_k=fetch_k)
-    
-        # **3️⃣ TF-IDF Keyword Search on Filtered Set**
-        all_texts = [doc.page_content for doc in filtered_results]  # Use filtered docs
-        vectorizer = TfidfVectorizer(stop_words="english")
-        tfidf_matrix = vectorizer.fit_transform(all_texts)
-        query_tfidf = vectorizer.transform([query])
-        keyword_scores = cosine_similarity(query_tfidf, tfidf_matrix).flatten()
-    
-        # **4️⃣ Hybrid Score Calculation**
-        combined_results, seen_ids = [], set()
-        for i, doc in enumerate(filtered_results):  # Iterate over filtered results
-            doc_id = doc.metadata.get("id")
-            semantic_score = float(doc.metadata.get("score", 0))
-            keyword_score = float(keyword_scores[i])
-            final_score = 0.65 * semantic_score + 0.35 * keyword_score  # Hybrid score
-    
-            if doc_id not in seen_ids and final_score > min_score:
-                seen_ids.add(doc_id)
-                doc.metadata["final_score"] = final_score
-                combined_results.append(doc)
-    
-        # **5️⃣ Sort Results by Final Score**
-        combined_results.sort(key=lambda x: x.metadata["final_score"], reverse=True)
-    
-        return [
-            {
-                "doc_id": doc.metadata.get("doc_id", "N/A"),
-                "chunk_id": doc.metadata.get("id", "N/A"),
-                "title": doc.metadata.get("source", "N/A"),
-                "text": doc.page_content,
-                "page_number": str(doc.metadata.get("page_number", "N/A")),
-                "score": str(doc.metadata.get("final_score", "N/A")),
-            }
-            for doc in combined_results
-        ]
-    except Exception as e:
-        print(e)
-        return
+    semantic_results = vector_store.max_marginal_relevance_search(query, k=10, fetch_k=fetch_k)
+
+    all_texts = [doc.page_content for doc in semantic_results]
+    vectorizer = TfidfVectorizer(stop_words="english")
+    tfidf_matrix = vectorizer.fit_transform(all_texts)
+    query_tfidf = vectorizer.transform([query])
+    keyword_scores = cosine_similarity(query_tfidf, tfidf_matrix).flatten()
+
+    combined_results, seen_ids = [], set()
+    for i, doc in enumerate(semantic_results):
+        doc_id, doc_groups = doc.metadata.get("id"), doc.metadata.get("groups", [])
+        semantic_score = float(doc.metadata.get("score", 0))
+        keyword_score = float(keyword_scores[i])
+        final_score = 0.7 * semantic_score + 0.3 * keyword_score  # Hybrid score
+
+        if doc_id not in seen_ids and any(group in user_groups for group in doc_groups) and final_score > min_score:
+            seen_ids.add(doc_id)
+            doc.metadata["final_score"] = final_score
+            combined_results.append(doc)
+
+    combined_results.sort(key=lambda x: x.metadata["final_score"], reverse=True)
+    return [
+        {
+            "doc_id": doc.metadata.get("doc_id", "N/A"),
+            "chunk_id": doc.metadata.get("id", "N/A"),
+            "title": doc.metadata.get("source", "N/A"),
+            "text": doc.page_content,
+            "page_number": str(doc.metadata.get("page_number", "N/A")),
+            "score": str(doc.metadata.get("final_score", "N/A")),
+        }
+        for doc in combined_results
+    ]
 
 # 🔹 Metadata-Weighted Reranking
 def rerank(query, context):
@@ -133,9 +75,9 @@ def rerank(query, context):
     final_reranked = []
     for entry in reranker.data:
         doc, score = entry["document"], float(entry["score"])
-        # citation_boost = 1.2 if "high_citations" in doc.get("tags", []) else 1.0
-        # recency_boost = 1.1 if "recent_upload" in doc.get("tags", []) else 1.0
-        # final_score = score * citation_boost * recency_boost
+        citation_boost = 1.2 if "high_citations" in doc.get("tags", []) else 1.0
+        recency_boost = 1.1 if "recent_upload" in doc.get("tags", []) else 1.0
+        final_score = score * citation_boost * recency_boost
         doc["final_score"] = final_score
         final_reranked.append(doc)
 
@@ -143,24 +85,23 @@ def rerank(query, context):
     return final_reranked
 
 # 🔹 Intelligent Search Summary Generator
-def generate_search_summary(search_results, document_titles, query):
+def generate_search_summary(search_results, query):
     if not search_results:
         return "No relevant documents found. Try refining your query."
 
-    num_results = len(document_titles)
+    num_results = len(search_results)
     doc_titles = [doc.get("title", "Unknown Document") for doc in search_results]
     doc_pages = [doc.get("page_number", "N/A") for doc in search_results]
     relevance_scores = [float(doc.get("score", 0)) for doc in search_results]
 
     summary_prompt = f"""
-    Generate a concise 1-3 sentence summary for the document search results found:
+    Generate a concise 1-3 sentence summary:
     - User Query: "{query}"
     - Matching Documents: {num_results} found
     - Titles: {", ".join(set(doc_titles))}
     - Pages Referenced: {", ".join(set(doc_pages))}
     - Relevance Scores (0-1): {relevance_scores}
     Provide a clear, user-friendly summary with an action suggestion.
-    If scores are low but the documents are from the same title no need to comment on the scores.
     """
 
     llm = ChatOpenAI(model="gpt-4", openai_api_key=openai.api_key, temperature=0.5)
@@ -202,7 +143,7 @@ def complete_workflow(query, user_groups, index_name="briefmeta"):
 
         document_titles = list({os.path.basename(doc["title"]) for doc in context_data})
         formatted_titles = " " + "\n".join(document_titles)
-        intelligent_search_summary = generate_search_summary(context_data, document_titles, refined_query)
+        intelligent_search_summary = generate_search_summary(context_data, refined_query)
 
         results = {
             "results": [