Update appStore/prep_data.py

appStore/prep_data.py

@@ -48,3 +48,19 @@ def process_giz_worldwide():
     giz_df['source'] = 'GIZ_WORLDWIDE'
     return giz_df
 
+def remove_duplicates(results_list):
+    """
+    Return a new list of results with duplicates removed, 
+    based on 'url' in metadata.
+    """
+    unique_results = []
+    seen_urls = set()
+
+    for r in results_list:
+        # Safely get the URL from metadata
+        url = r.payload['metadata'].get('url', None)
+        if url not in seen_urls:
+            seen_urls.add(url)
+            unique_results.append(r)
+
+    return unique_results