Update app.py

app.py

@@ -8,10 +8,20 @@ from sentence_transformers import SentenceTransformer
 app = FastAPI()
 
 FIELDS = (
-    "full_name", "description", "watchers_count", "forks_count", "license",
-    "default_branch", "has_build_zig", "has_build_zig_zon", "fork",
-    "open_issues", "stargazers_count", "updated_at", "created_at",
-    "size"
+    "full_name",
+    "description",
+    "default_branch",
+    "open_issues",
+    "stargazers_count",
+    "forks_count",
+    "watchers_count",
+    "license",
+    "size",
+    "fork",
+    "updated_at",
+    "has_build_zig",
+    "has_build_zig_zon",
+    "created_at",
 )
 
 model = SentenceTransformer("all-MiniLM-L6-v2")
@@ -45,6 +55,23 @@ scroll_data = {
     "infiniteScrollPrograms": load_dataset_with_fields("zigistry/programs", include_readme=False)[0],
 }
 
+def filter_results_by_distance(distances, idxs, dataset, max_results=50, threshold=0.6):
+    """
+    Only return results that are likely relevant (distance-based filtering).
+    Lower distance = more similar.
+    Threshold is a fraction of the *minimum* distance found.
+    """
+    if len(distances) == 0:
+        return []
+    min_dist = np.min(distances)
+    cutoff = min_dist + ((max(distances) - min_dist) * threshold)
+    filtered = [
+        dataset[int(i)]
+        for d, i in zip(distances, idxs)
+        if d <= cutoff
+    ]
+    return filtered[:max_results]
+
 @app.get("/infiniteScrollPackages/")
 def infinite_scroll_packages(q: int = Query(0, ge=0)):
     start = q * 10
@@ -64,21 +91,18 @@ def search_packages(q: str):
     key = "packages"
     index, dataset = indices[key]
     query_embedding = model.encode([q])
-    distances, indices_ = index.search(np.array(query_embedding), len(dataset))
-    sorted_results = sorted(zip(distances[0], indices_[0]), key=lambda x: x[0])
-    results = [dataset[int(i)] for d, i in sorted_results]
-    content = results[:280] if len(results) > 280 else results
+    distances, idxs = index.search(np.array(query_embedding), len(dataset))
+    # Only keep results that are likely relevant
+    results = filter_results_by_distance(distances[0], idxs[0], dataset)
     headers = {"Access-Control-Allow-Origin": "*", "Content-Type": "application/json"}
-    return JSONResponse(content=content, headers=headers)
+    return JSONResponse(content=results, headers=headers)
 
 @app.get("/searchPrograms/")
 def search_programs(q: str):
     key = "programs"
     index, dataset = indices[key]
     query_embedding = model.encode([q])
-    distances, indices_ = index.search(np.array(query_embedding), len(dataset))
-    sorted_results = sorted(zip(distances[0], indices_[0]), key=lambda x: x[0])
-    results = [dataset[int(i)] for d, i in sorted_results]
-    content = results[:280] if len(results) > 280 else results
+    distances, idxs = index.search(np.array(query_embedding), len(dataset))
+    results = filter_results_by_distance(distances[0], idxs[0], dataset)
     headers = {"Access-Control-Allow-Origin": "*", "Content-Type": "application/json"}
-    return JSONResponse(content=content, headers=headers)
+    return JSONResponse(content=results, headers=headers)