add our app

app.py

@@ -1,147 +1,173 @@
-import io
-import random
-from typing import List, Tuple
+# app_panel.py – Panel-based CGD Survey Explorer
 
-import aiohttp
+import os, io, json, gc
 import panel as pn
-from PIL import Image
-from transformers import CLIPModel, CLIPProcessor
-
-pn.extension(design="bootstrap", sizing_mode="stretch_width")
-
-ICON_URLS = {
-    "brand-github": "https://github.com/holoviz/panel",
-    "brand-twitter": "https://twitter.com/Panel_Org",
-    "brand-linkedin": "https://www.linkedin.com/company/panel-org",
-    "message-circle": "https://discourse.holoviz.org/",
-    "brand-discord": "https://discord.gg/AXRHnJU6sP",
-}
-
-
-async def random_url(_):
-    pet = random.choice(["cat", "dog"])
-    api_url = f"https://api.the{pet}api.com/v1/images/search"
-    async with aiohttp.ClientSession() as session:
-        async with session.get(api_url) as resp:
-            return (await resp.json())[0]["url"]
-
-
-@pn.cache
-def load_processor_model(
-    processor_name: str, model_name: str
-) -> Tuple[CLIPProcessor, CLIPModel]:
-    processor = CLIPProcessor.from_pretrained(processor_name)
-    model = CLIPModel.from_pretrained(model_name)
-    return processor, model
-
-
-async def open_image_url(image_url: str) -> Image:
-    async with aiohttp.ClientSession() as session:
-        async with session.get(image_url) as resp:
-            return Image.open(io.BytesIO(await resp.read()))
-
-
-def get_similarity_scores(class_items: List[str], image: Image) -> List[float]:
-    processor, model = load_processor_model(
-        "openai/clip-vit-base-patch32", "openai/clip-vit-base-patch32"
+import pandas as pd
+import boto3, torch
+from sentence_transformers import SentenceTransformer, util
+import psycopg2
+
+pn.extension()
+
+# ───────────────────────────────────────────────
+# 1) Data / Embeddings Loaders
+# ───────────────────────────────────────────────
+DB_HOST = os.getenv("DB_HOST")
+DB_PORT = os.getenv("DB_PORT", "5432")
+DB_NAME = os.getenv("DB_NAME")
+DB_USER = os.getenv("DB_USER")
+DB_PASSWORD = os.getenv("DB_PASSWORD")
+
+@pn.cache()
+def get_data():
+    conn = psycopg2.connect(
+        host=DB_HOST, port=DB_PORT,
+        dbname=DB_NAME, user=DB_USER, password=DB_PASSWORD,
+        sslmode="require"
     )
-    inputs = processor(
-        text=class_items,
-        images=[image],
-        return_tensors="pt",  # pytorch tensors
+    df_ = pd.read_sql_query("""
+        SELECT id, country, year, section,
+               question_code, question_text,
+               answer_code,  answer_text
+          FROM survey_info;
+    """, conn)
+    conn.close()
+    return df_
+
+df = get_data()
+row_lookup = {row.id: i for i, row in df.iterrows()}
+
+@pn.cache()
+def load_embeddings():
+    BUCKET, KEY = "cgd-embeddings-bucket", "survey_info_embeddings.pt"
+    buf = io.BytesIO()
+    boto3.client("s3").download_fileobj(BUCKET, KEY, buf)
+    buf.seek(0)
+    ckpt = torch.load(buf, map_location="cpu")
+    buf.close(); gc.collect()
+    return ckpt["ids"], ckpt["embeddings"]
+
+ids_list, emb_tensor = load_embeddings()
+
+@pn.cache()
+def get_st_model():
+    return SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2", device="cpu")
+
+# ───────────────────────────────────────────────
+# 2) Widgets
+# ───────────────────────────────────────────────
+country_opts = sorted(df["country"].dropna().unique())
+year_opts = sorted(df["year"].dropna().unique())
+
+w_countries = pn.widgets.MultiSelect(name="Countries", options=country_opts)
+w_years = pn.widgets.MultiSelect(name="Years", options=year_opts)
+w_keyword = pn.widgets.TextInput(name="Keyword Search", placeholder="Search questions or answers")
+w_group = pn.widgets.Checkbox(name="Group by Question Text", value=False)
+
+# Semantic search
+w_semquery = pn.widgets.TextInput(name="Semantic Query")
+w_search_button = pn.widgets.Button(name="Search", button_type="primary", disabled=False)
+
+# ───────────────────────────────────────────────
+# 3) Filtering Logic
+# ───────────────────────────────────────────────
+@pn.depends(w_countries, w_years, w_keyword, w_group)
+def keyword_filter(countries, years, keyword, group):
+    filt = df.copy()
+    if countries:
+        filt = filt[filt["country"].isin(countries)]
+    if years:
+        filt = filt[filt["year"].isin(years)]
+    if keyword:
+        filt = filt[
+            filt["question_text"].str.contains(keyword, case=False, na=False) |
+            filt["answer_text"].str.contains(keyword, case=False, na=False) |
+            filt["question_code"].astype(str).str.contains(keyword, case=False, na=False)
+        ]
+
+    if group:
+        grouped = (
+            filt.groupby("question_text")
+            .agg({
+                "country": lambda x: sorted(set(x)),
+                "year": lambda x: sorted(set(x)),
+                "answer_text": lambda x: list(x)[:3]
+            })
+            .reset_index()
+            .rename(columns={
+                "country": "Countries",
+                "year": "Years",
+                "answer_text": "Sample Answers"
+            })
+        )
+        return pn.pane.DataFrame(grouped, sizing_mode="stretch_width", height=400)
+
+    return pn.pane.DataFrame(
+        filt[["country", "year", "question_text", "answer_text"]],
+        sizing_mode="stretch_width", height=400
     )
-    outputs = model(**inputs)
-    logits_per_image = outputs.logits_per_image
-    class_likelihoods = logits_per_image.softmax(dim=1).detach().numpy()
-    return class_likelihoods[0]
-
-
-async def process_inputs(class_names: List[str], image_url: str):
-    """
-    High level function that takes in the user inputs and returns the
-    classification results as panel objects.
-    """
-    try:
-        main.disabled = True
-        if not image_url:
-            yield "##### ⚠️ Provide an image URL"
-            return
-    
-        yield "##### ⚙ Fetching image and running model..."
-        try:
-            pil_img = await open_image_url(image_url)
-            img = pn.pane.Image(pil_img, height=400, align="center")
-        except Exception as e:
-            yield f"##### 😔 Something went wrong, please try a different URL!"
-            return
-    
-        class_items = class_names.split(",")
-        class_likelihoods = get_similarity_scores(class_items, pil_img)
-    
-        # build the results column
-        results = pn.Column("##### 🎉 Here are the results!", img)
-    
-        for class_item, class_likelihood in zip(class_items, class_likelihoods):
-            row_label = pn.widgets.StaticText(
-                name=class_item.strip(), value=f"{class_likelihood:.2%}", align="center"
-            )
-            row_bar = pn.indicators.Progress(
-                value=int(class_likelihood * 100),
-                sizing_mode="stretch_width",
-                bar_color="secondary",
-                margin=(0, 10),
-                design=pn.theme.Material,
-            )
-            results.append(pn.Column(row_label, row_bar))
-        yield results
-    finally:
-        main.disabled = False
-
-
-# create widgets
-randomize_url = pn.widgets.Button(name="Randomize URL", align="end")
-
-image_url = pn.widgets.TextInput(
-    name="Image URL to classify",
-    value=pn.bind(random_url, randomize_url),
-)
-class_names = pn.widgets.TextInput(
-    name="Comma separated class names",
-    placeholder="Enter possible class names, e.g. cat, dog",
-    value="cat, dog, parrot",
-)
 
-input_widgets = pn.Column(
-    "##### 😊 Click randomize or paste a URL to start classifying!",
-    pn.Row(image_url, randomize_url),
-    class_names,
+# ───────────────────────────────────────────────
+# 4) Semantic Search Callback
+# ───────────────────────────────────────────────
+def semantic_search(event=None):
+    query = w_semquery.value.strip()
+    if not query:
+        return
+
+    model = get_st_model()
+    q_vec = model.encode(query, convert_to_tensor=True, device="cpu").cpu()
+    sims = util.cos_sim(q_vec, emb_tensor)[0]
+    top_vals, top_idx = torch.topk(sims, k=50)
+
+    sem_ids = [ids_list[i] for i in top_idx.tolist()]
+    sem_rows = df.loc[df["id"].isin(sem_ids)].copy()
+    score_map = dict(zip(sem_ids, top_vals.tolist()))
+    sem_rows["Score"] = sem_rows["id"].map(score_map)
+    sem_rows = sem_rows.sort_values("Score", ascending=False)
+
+    # Get keyword-filtered data
+    keyword_df = keyword_filter(
+        w_countries.value,
+        w_years.value,
+        w_keyword.value,
+        False
+    ).object
+
+    remainder = keyword_df.loc[~keyword_df["id"].isin(sem_ids)].copy()
+    remainder["Score"] = ""
+
+    combined = pd.concat([sem_rows, remainder], ignore_index=True)
+
+    result_pane.object = combined[["Score", "country", "year", "question_text", "answer_text"]]
+
+w_search_button.on_click(semantic_search)
+
+result_pane = pn.pane.DataFrame(height=500, sizing_mode="stretch_width")
+
+# ───────────────────────────────────────────────
+# 5) Layout
+# ───────────────────────────────────────────────
+sidebar = pn.Column(
+    "## 🔍 Filter Questions",
+    w_countries, w_years, w_keyword, w_group,
+    pn.Spacer(height=20),
+    "## 🧠 Semantic Search",
+    w_semquery, w_search_button,
+    width=300
 )
 
-# add interactivity
-interactive_result = pn.panel(
-    pn.bind(process_inputs, image_url=image_url, class_names=class_names),
-    height=600,
-)
-
-# add footer
-footer_row = pn.Row(pn.Spacer(), align="center")
-for icon, url in ICON_URLS.items():
-    href_button = pn.widgets.Button(icon=icon, width=35, height=35)
-    href_button.js_on_click(code=f"window.open('{url}')")
-    footer_row.append(href_button)
-footer_row.append(pn.Spacer())
-
-# create dashboard
-main = pn.WidgetBox(
-    input_widgets,
-    interactive_result,
-    footer_row,
+main = pn.Column(
+    pn.pane.Markdown("## 🌍 CGD Survey Explorer"),
+    pn.Tabs(
+        ("Filtered Results", keyword_filter),
+        ("Semantic Search Results", result_pane),
+    )
 )
 
-title = "Panel Demo - Image Classification"
-pn.template.BootstrapTemplate(
-    title=title,
+pn.template.FastListTemplate(
+    title="CGD Survey Explorer",
+    sidebar=sidebar,
     main=main,
-    main_max_width="min(50%, 698px)",
-    header_background="#F08080",
-).servable(title=title)
+    theme_toggle=True,
+).servable()