Update app.py

app.py

@@ -1,6 +1,6 @@
 import streamlit as st
 import pandas as pd
-from appStore.prep_data import process_giz_worldwide
+from appStore.prep_data import process_giz_worldwide, remove_duplicates
 from appStore.prep_utils import create_documents, get_client
 from appStore.embed import hybrid_embed_chunks
 from appStore.search import hybrid_search
@@ -125,12 +125,16 @@ if button:
     filtered_semantic = filter_results(semantic_all, country_filter, end_year_range)
     filtered_lexical = filter_results(lexical_all, country_filter, end_year_range)
 
+    filtered_semantic_no_dupe = remove_duplicates(filtered_semantic)
+    filtered_lexical_no_dupe = remove_duplicates(filtered_lexical)
+
+
     # 3) Now we take the top 10 *after* filtering
     # Check user preference
     if show_exact_matches:
         st.write(f"Showing **Top 10 Lexical Search results** for query: {var}")
         # Show the top 10 from filtered_lexical
-        for res in filtered_lexical[:10]:
+        for res in filtered_lexical_no_dupe[:10]:
             project_name = res.payload['metadata'].get('project_name', 'Project Link')
             url = res.payload['metadata'].get('url', '#')
             st.markdown(f"#### [{project_name}]({url})")
@@ -172,7 +176,7 @@ if button:
     else:
         st.write(f"Showing **Top 10 Semantic Search results** for query: {var}")
         # Show the top 10 from filtered_semantic
-        for res in filtered_semantic[:10]:
+        for res in filtered_semantic_no_dupe[:10]:
             project_name = res.payload['metadata'].get('project_name', 'Project Link')
             url = res.payload['metadata'].get('url', '#')
             st.markdown(f"#### [{project_name}]({url})")