Update app.py

app.py

@@ -153,14 +153,14 @@ if button:
 
     # 2) Filter out content < 20 chars (as intermediate fix to problem that e.g. super short paragraphs with few chars get high similarity score)
     semantic_all = [
-        r for r in semantic_all if len(r.payload["page_content"]) >= 20
+        r for r in semantic_all if len(r.payload["page_content"]) >= 70
     ]
     lexical_all = [
-        r for r in lexical_all if len(r.payload["page_content"]) >= 20
+        r for r in lexical_all if len(r.payload["page_content"]) >= 70
     ]
     
     # 2) Apply a threshold to SEMANTIC results (score >= 0.3)
-    semantic_thresholded = [r for r in semantic_all if r.score >= 0.3]
+    semantic_thresholded = [r for r in semantic_all if r.score >= 0.4]
     
     # 2) Filter the entire sets
     filtered_semantic = filter_results(semantic_thresholded, country_filter, region_filter, end_year_range)
@@ -173,47 +173,78 @@ if button:
     # 3) Now we take the top 10 *after* filtering
     # Check user preference
     if show_exact_matches:
+        # 1) Display heading
         st.write(f"Showing **Top 10 Lexical Search results** for query: {var}")
-        # Show the top 10 from filtered_lexical
-        for res in filtered_lexical_no_dupe[:10]:
-            project_name = res.payload['metadata'].get('project_name', 'Project Link')
-            url = res.payload['metadata'].get('url', '#')
-            st.markdown(f"#### [{project_name}]({url})")
-
-            # Snippet logic (80 words)
-            full_text = res.payload['page_content']
-            words = full_text.split()
-            preview_word_count = 80
-            preview_text = " ".join(words[:preview_word_count])
-            remainder_text = " ".join(words[preview_word_count:])
-            st.write(preview_text + ("..." if remainder_text else ""))
-
-            # Keywords
-            top_keywords = extract_top_keywords(full_text, top_n=5)
-            if top_keywords:
-                st.markdown(f"_{' · '.join(top_keywords)}_")
-
-            # Metadata
-            metadata = res.payload.get('metadata', {})
-            countries = metadata.get('countries', "[]")
-            client_name = metadata.get('client', 'Unknown Client')
-            start_year = metadata.get('start_year', None)
-            end_year_ = metadata.get('end_year', None)
-
-            try:
-                c_list = json.loads(countries.replace("'", '"'))
-                country_names = [get_country_name(code.upper(), region_df) for code in c_list if len(code) == 2]
-            except json.JSONDecodeError:
-                country_names = []
-
-            start_year_str = f"{int(round(float(start_year)))}" if start_year else "Unknown"
-            end_year_str = f"{int(round(float(end_year_)))}" if end_year_ else "Unknown"
-
-            additional_text = (
-                f"**{', '.join(country_names)}**, commissioned by **{client_name}**, **{start_year_str}-{end_year_str}**"
-            )
-            st.markdown(additional_text)
-            st.divider()
+    
+        # 2) Do a simple substring check (case-insensitive)
+        #    We'll create a new list lexical_substring_filtered
+        query_substring = var.strip().lower()
+        lexical_substring_filtered = []
+        for r in lexical_all:
+            # page_content in lowercase
+            page_text_lower = r.payload["page_content"].lower()
+            # Keep this result only if the query substring is found
+            if query_substring in page_text_lower:
+                lexical_substring_filtered.append(r)
+    
+        # 3) Now apply your region/country/year filter on that new list
+        filtered_lexical = filter_results(
+            lexical_substring_filtered, country_filter, region_filter, end_year_range
+        )
+    
+        # 4) Remove duplicates
+        filtered_lexical_no_dupe = remove_duplicates(filtered_lexical)
+    
+        # 5) If empty after substring + filters + dedupe, show a custom message
+        if not filtered_lexical_no_dupe:
+            st.write('No exact matches, consider unchecking "Show only exact matches"')
+        else:
+            # 6) Display the first 10 matching results
+            for res in filtered_lexical_no_dupe[:10]:
+                project_name = res.payload['metadata'].get('project_name', 'Project Link')
+                url = res.payload['metadata'].get('url', '#')
+                st.markdown(f"#### [{project_name}]({url})")
+    
+                # Snippet logic (80 words)
+                full_text = res.payload['page_content']
+                words = full_text.split()
+                preview_word_count = 80
+                preview_text = " ".join(words[:preview_word_count])
+                remainder_text = " ".join(words[preview_word_count:])
+                st.write(preview_text + ("..." if remainder_text else ""))
+    
+                # Keywords
+                top_keywords = extract_top_keywords(full_text, top_n=5)
+                if top_keywords:
+                    st.markdown(f"_{' · '.join(top_keywords)}_")
+    
+                # Metadata
+                metadata = res.payload.get('metadata', {})
+                countries = metadata.get('countries', "[]")
+                client_name = metadata.get('client', 'Unknown Client')
+                start_year = metadata.get('start_year', None)
+                end_year_ = metadata.get('end_year', None)
+    
+                try:
+                    c_list = json.loads(countries.replace("'", '"'))
+                    country_names = [
+                        get_country_name(code.upper(), region_df) 
+                        for code in c_list 
+                        if len(code) == 2
+                    ]
+                except json.JSONDecodeError:
+                    country_names = []
+    
+                start_year_str = f"{int(round(float(start_year)))}" if start_year else "Unknown"
+                end_year_str = f"{int(round(float(end_year_)))}" if end_year_ else "Unknown"
+    
+                additional_text = (
+                    f"**{', '.join(country_names)}**, commissioned by **{client_name}**, "
+                    f"**{start_year_str}-{end_year_str}**"
+                )
+                st.markdown(additional_text)
+                st.divider()
+    
     else:
         st.write(f"Showing **Top 10 Semantic Search results** for query: {var}")