Update app.py

app.py

@@ -85,132 +85,130 @@ with col2:
 # Checkbox to control whether to show only exact matches
 show_exact_matches = st.checkbox("Show only exact matches", value=False)
 
-button = st.button("Search")
+def filter_results(results, country_filter, end_year_range):
+    filtered = []
+    for r in results:
+        metadata = r.payload.get('metadata', {})
+        countries = metadata.get('countries', "[]")
+        end_year_val = float(metadata.get('end_year', 0))
+
+        # Convert countries to a list
+        try:
+            c_list = json.loads(countries.replace("'", '"'))
+            c_list = [code.upper() for code in c_list if len(code) == 2]
+        except json.JSONDecodeError:
+            c_list = []
+
+        # Translate selected country name to iso2
+        selected_iso_code = country_name_mapping.get(country_filter, None)
+
+        # Filtering
+        if (
+            (country_filter == "All/Not allocated" or selected_iso_code in c_list)
+            and (end_year_range[0] <= end_year_val <= end_year_range[1])
+        ):
+            filtered.append(r)
+    return filtered
+
+
 if button:
-    results = hybrid_search(client, var, collection_name)
+    # 1) Use a bigger limit so we get more than 10 results
+    #    We'll filter them first, then slice the top 10 from the filtered set.
+    results = hybrid_search(client, var, collection_name, limit=100)  # e.g., 100 or 200
 
-    def filter_results(results, country_filter, end_year_range):
-        filtered = []
-        for res in results:
-            metadata = res.payload.get('metadata', {})
-            countries = metadata.get('countries', "[]")
-            end_year = float(metadata.get('end_year', 0))
-    
-            # Process countries string to a list
-            try:
-                country_list = json.loads(countries.replace("'", '"'))
-                # Normalize to uppercase and filter only 2-digit ISO codes
-                country_list = [code.upper() for code in country_list if len(code) == 2]
-            except json.JSONDecodeError:
-                country_list = []
-    
-            # Translate selected country name back to 2-digit ISO code
-            selected_iso_code = country_name_mapping.get(country_filter, None)
-    
-            # Apply country and year filters
-            if (country_filter == "All/Not allocated" or selected_iso_code in country_list) and (end_year_range[0] <= end_year <= end_year_range[1]):
-                filtered.append(res)
-        return filtered
+    # results is a tuple: (semantic_results, lexical_results)
+    semantic_all = results[0]
+    lexical_all = results[1]
 
+    # 2) Filter the entire sets
+    filtered_semantic = filter_results(semantic_all, country_filter, end_year_range)
+    filtered_lexical = filter_results(lexical_all, country_filter, end_year_range)
 
-    # Check user preference for exact matches
+    # 3) Now we take the top 10 *after* filtering
+    # Check user preference
     if show_exact_matches:
         st.write(f"Showing **Top 10 Lexical Search results** for query: {var}")
-        lexical_results = results[1]  # Lexical results are in index 1
-        filtered_lexical_results = filter_results(lexical_results, country_filter, end_year_range)
-        for res in filtered_lexical_results[:10]:
+        # Show the top 10 from filtered_lexical
+        for res in filtered_lexical[:10]:
             project_name = res.payload['metadata'].get('project_name', 'Project Link')
             url = res.payload['metadata'].get('url', '#')
             st.markdown(f"#### [{project_name}]({url})")
-            # ------- Display first 4 lines + expander -------
+
+            # Snippet logic (80 words)
             full_text = res.payload['page_content']
-            # Split the text by whitespace
             words = full_text.split()
-            # For instance, show only the first 40 words
-            preview_word_count = 120
-            # Create the short preview and the remainder
+            preview_word_count = 80
             preview_text = " ".join(words[:preview_word_count])
             remainder_text = " ".join(words[preview_word_count:])
-            # Always display the preview_text
             st.write(preview_text + ("..." if remainder_text else ""))
-            # ------ Extract top 5 keywords and display ------
+
+            # Keywords
             top_keywords = extract_top_keywords(full_text, top_n=5)
             if top_keywords:
-                st.markdown(f"_{' · '.join(top_keywords)}_")  # italic + ' · ' separator
-            # ------- Additional info below the text -------
+                st.markdown(f"_{' · '.join(top_keywords)}_")
+
+            # Metadata
             metadata = res.payload.get('metadata', {})
             countries = metadata.get('countries', "[]")
-            client = metadata.get('client', 'Unknown Client')
+            client_name = metadata.get('client', 'Unknown Client')
             start_year = metadata.get('start_year', None)
-            end_year = metadata.get('end_year', None)
+            end_year_ = metadata.get('end_year', None)
 
-            # Process countries
             try:
-                country_list = json.loads(countries.replace("'", '"'))
-                # Normalize to uppercase and map to country names
-                country_names = [get_country_name(code.upper(), region_df) for code in country_list if len(code) == 2]
-                country_names = country_names if country_names else country_list  # Fallback if no names found
+                c_list = json.loads(countries.replace("'", '"'))
+                country_names = [get_country_name(code.upper(), region_df) for code in c_list if len(code) == 2]
             except json.JSONDecodeError:
-                country_names = countries
+                country_names = []
 
-            # Format start and end year
-            start_year = f"{int(round(float(start_year)))}" if start_year else "Unknown"
-            end_year = f"{int(round(float(end_year)))}" if end_year else "Unknown"
+            start_year_str = f"{int(round(float(start_year)))}" if start_year else "Unknown"
+            end_year_str = f"{int(round(float(end_year_)))}" if end_year_ else "Unknown"
 
-            # Generate additional text with Markdown for bold formatting
-            additional_text = f"**{', '.join(country_names)}**, commissioned by **{client}**, **{start_year}-{end_year}**"
+            additional_text = (
+                f"**{', '.join(country_names)}**, commissioned by **{client_name}**, **{start_year_str}-{end_year_str}**"
+            )
             st.markdown(additional_text)
-
-
             st.divider()
     else:
         st.write(f"Showing **Top 10 Semantic Search results** for query: {var}")
-        semantic_results = results[0]  # Semantic results are in index 0
-        filtered_semantic_results = filter_results(semantic_results, country_filter, end_year_range)
-        for res in filtered_semantic_results[:10]:
+        # Show the top 10 from filtered_semantic
+        for res in filtered_semantic[:10]:
             project_name = res.payload['metadata'].get('project_name', 'Project Link')
             url = res.payload['metadata'].get('url', '#')
             st.markdown(f"#### [{project_name}]({url})")
-            # ------- Display first 4 lines + expander -------
+
+            # Snippet logic
             full_text = res.payload['page_content']
-            # Split the text by whitespace
             words = full_text.split()
-            # For instance, show only the first 40 words
-            preview_word_count = 40
-            # Create the short preview and the remainder
+            preview_word_count = 80
             preview_text = " ".join(words[:preview_word_count])
             remainder_text = " ".join(words[preview_word_count:])
-            # Always display the preview_text
             st.write(preview_text + ("..." if remainder_text else ""))
-            # ------ Extract top 5 keywords and display ------
+
+            # Keywords
             top_keywords = extract_top_keywords(full_text, top_n=5)
             if top_keywords:
-                st.markdown(f"_{' · '.join(top_keywords)}_")  # italic + ' · ' separator
-            
-            # Additional text below the content
+                st.markdown(f"_{' · '.join(top_keywords)}_")
+
+            # Metadata
             metadata = res.payload.get('metadata', {})
             countries = metadata.get('countries', "[]")
-            client = metadata.get('client', 'Unknown Client')
+            client_name = metadata.get('client', 'Unknown Client')
             start_year = metadata.get('start_year', None)
-            end_year = metadata.get('end_year', None)
+            end_year_ = metadata.get('end_year', None)
 
-            # Process countries
             try:
-                country_list = json.loads(countries.replace("'", '"'))
-                country_names = [get_country_name(code.upper(), region_df) for code in country_list if len(code) == 2]
-                country_names = country_names if country_names else country_list
+                c_list = json.loads(countries.replace("'", '"'))
+                country_names = [get_country_name(code.upper(), region_df) for code in c_list if len(code) == 2]
             except json.JSONDecodeError:
-                country_names = countries
+                country_names = []
 
-            # Format start and end year
-            start_year = f"{int(round(float(start_year)))}" if start_year else "Unknown"
-            end_year = f"{int(round(float(end_year)))}" if end_year else "Unknown"
+            start_year_str = f"{int(round(float(start_year)))}" if start_year else "Unknown"
+            end_year_str = f"{int(round(float(end_year_)))}" if end_year_ else "Unknown"
 
-            # Generate additional text with Markdown for bold formatting
-            additional_text = f"**{', '.join(country_names)}**, commissioned by **{client}**, **{start_year}-{end_year}**"
+            additional_text = (
+                f"**{', '.join(country_names)}**, commissioned by **{client_name}**, **{start_year_str}-{end_year_str}**"
+            )
             st.markdown(additional_text)
-
-
             st.divider()