Update app.py

app.py

@@ -58,11 +58,10 @@ for country_code, data in COUNTRY_MAPPING.items():
 
 # Set up the webhook listener for response creation
 @webhook_listener(events=["response.created"])
-async def update_validation_space_on_answer(response, type, timestamp):
+async def update_annotation_progress(response, type, timestamp):
     """
-    Webhook listener that triggers when a new response is added to an answering space.
-    It will automatically update the corresponding validation space with the new response
-    and update the country's annotation progress.
+    Webhook listener that triggers when a new response is added to an Argilla dataset.
+    It will update the annotation progress for the corresponding country.
     """
     try:
         # Store the event for display in the UI
@@ -71,55 +70,19 @@ async def update_validation_space_on_answer(response, type, timestamp):
         # Get the record from the response
         record = response.record
         
-        # Check if this is from an answering space
+        # Get dataset name
         dataset_name = record.dataset.name
-        if not dataset_name.endswith("_responder_preguntas"):
-            print(f"Ignoring event from non-answering dataset: {dataset_name}")
-            return  # Not an answering space, ignore
-        
-        # Extract the country from the dataset name
-        country = dataset_name.replace("_responder_preguntas", "")
-        print(f"Processing response for country: {country}")
-        
-        # Connect to the validation space
-        validation_dataset_name = f"{country}_validar_respuestas"
-        try:
-            validation_dataset = client.datasets(validation_dataset_name)
-            print(f"Found validation dataset: {validation_dataset_name}")
-        except Exception as e:
-            print(f"Error connecting to validation dataset: {e}")
-            # You would need to import the create_validation_space function
-            from build_space import create_validation_space
-            validation_dataset = create_validation_space(country)
-
-        response_dict = response.to_dict()
-        answer = response_dict["values"]['text']['value']
-
-        # Get the user ID of the original responder
-        original_user_id = str(response.user_id)
-        
-        # Create a validation record with the correct attribute
-        validation_record = {
-            "question": record.fields["question"],
-            "answer": answer,
-            "metadata": {
-                "original_responder_id": original_user_id,
-                "original_dataset": dataset_name
-            }
-        }
-        
-        # Add the record to the validation space
-        validation_dataset.records.log(records=[validation_record])
-        print(f"Added new response to validation space for {country}")
-        
-        # Update the annotation progress
-        # Get the country code from the country name or substring
+        print(f"Processing response for dataset: {dataset_name}")
+        
+        # Try to determine the country from the dataset name
         country_code = None
         for code, data in COUNTRY_MAPPING.items():
-            if data["name"].lower() in country.lower():
+            country_name = data["name"].lower()
+            if country_name in dataset_name.lower():
                 country_code = code
                 break
-        
+                
+        # If we found a matching country, update its progress
         if country_code and country_code in annotation_progress:
             # Increment the count
             annotation_progress[country_code]["count"] += 1
@@ -137,283 +100,310 @@ async def update_validation_space_on_answer(response, type, timestamp):
                 "count": count,
                 "percent": percent
             })
+            print(f"Updated progress for {annotation_progress[country_code]['name']}: {percent}%")
             
     except Exception as e:
         print(f"Error in webhook handler: {e}")
         # Store the error in the queue for display
         incoming_events.put({"event": "error", "error": str(e)})
 
-# Function to read the next event from the queue and update UI elements
+# Function to read the next event from the queue
 def read_next_event():
     if not incoming_events.empty():
         event = incoming_events.get()
         return event
     return {}
 
-# Function to get the current annotation progress data
-def get_annotation_progress():
-    return json.dumps(annotation_progress)
+# Function to calculate overall statistics
+def update_stats():
+    total = sum(data["count"] for data in annotation_progress.values())
+    percentages = [data["percent"] for data in annotation_progress.values()]
+    avg = sum(percentages) / len(percentages) if percentages else 0
+    countries_50_plus = sum(1 for p in percentages if p >= 50)
+    
+    return total, avg, countries_50_plus
 
-# D3.js map visualization HTML template
-def create_map_html(progress_data):
+# Create the map HTML container (without D3.js code)
+def create_map_html():
+    return """
+    <div id="map-container" style="width:100%; height:600px; position:relative; background-color:#111;">
+        <div style="display:flex; justify-content:center; align-items:center; height:100%; color:white; font-family:sans-serif;">
+            Loading map visualization...
+        </div>
+    </div>
+    <div id="tooltip" style="position:absolute; background-color:rgba(0,0,0,0.8); border-radius:5px; padding:8px; color:white; font-size:12px; pointer-events:none; opacity:0; transition:opacity 0.3s;"></div>
+    """
+
+# Create D3.js script that will be loaded via Gradio's JavaScript execution
+def create_d3_script(progress_data):
     return f"""
-    <!DOCTYPE html>
-    <html>
-    <head>
-        <script src="https://d3js.org/d3.v7.min.js"></script>
-        <script src="https://d3js.org/d3-geo.v2.min.js"></script>
-        <script src="https://d3js.org/topojson.v3.min.js"></script>
-        <style>
-            .country {{
-                stroke: #f32b7b;
-                stroke-width: 1px;
-            }}
-            .country:hover {{
-                stroke: #4a1942;
-                stroke-width: 2px;
-                cursor: pointer;
-            }}
-            .tooltip {{
-                position: absolute;
-                background-color: rgba(0, 0, 0, 0.8);
-                border-radius: 5px;
-                padding: 8px;
-                color: white;
-                font-size: 12px;
-                pointer-events: none;
-                opacity: 0;
-                transition: opacity 0.3s;
-            }}
-            text {{
-                fill: #f1f5f9;
-                font-family: sans-serif;
-            }}
-            .legend-text {{
-                fill: #94a3b8;
-                font-size: 10px;
-            }}
-        </style>
-    </head>
-    <body style="margin:0; background-color:#111;">
-        <div id="map-container" style="width:100%; height:600px; position:relative;"></div>
-        <div id="tooltip" class="tooltip"></div>
-        
-        <script>
-            // The progress data passed from Python
-            const progressData = {progress_data};
+    async () => {{
+        // Load D3.js modules
+        const script1 = document.createElement("script");
+        script1.src = "https://cdn.jsdelivr.net/npm/d3@7";
+        document.head.appendChild(script1);
+        
+        // Wait for D3 to load
+        await new Promise(resolve => {{
+            script1.onload = resolve;
+        }});
+        
+        console.log("D3 loaded successfully");
+        
+        // Load topojson
+        const script2 = document.createElement("script");
+        script2.src = "https://cdn.jsdelivr.net/npm/topojson@3";
+        document.head.appendChild(script2);
+        
+        await new Promise(resolve => {{
+            script2.onload = resolve;
+        }});
+        
+        console.log("TopoJSON loaded successfully");
+        
+        // The progress data passed from Python
+        const progressData = {progress_data};
+        
+        // Set up the SVG container
+        const mapContainer = document.getElementById('map-container');
+        mapContainer.innerHTML = ''; // Clear loading message
+        
+        const width = mapContainer.clientWidth;
+        const height = 600;
+        
+        const svg = d3.select("#map-container")
+            .append("svg")
+            .attr("width", width)
+            .attr("height", height)
+            .attr("viewBox", `0 0 ${{width}} ${{height}}`)
+            .style("background-color", "#111");
+        
+        // Define color scale
+        const colorScale = d3.scaleLinear()
+            .domain([0, 100])
+            .range(["#4a1942", "#f32b7b"]);
+        
+        // Set up projection focused on Latin America and Spain
+        const projection = d3.geoMercator()
+            .center([-60, 0])
+            .scale(width / 5)
+            .translate([width / 2, height / 2]);
+            
+        const path = d3.geoPath().projection(projection);
+        
+        // Tooltip setup
+        const tooltip = d3.select("#tooltip");
+        
+        // Load the world GeoJSON data
+        const response = await fetch("https://raw.githubusercontent.com/holtzy/D3-graph-gallery/master/DATA/world.geojson");
+        const data = await response.json();
+        
+        // Draw the map
+        svg.selectAll("path")
+            .data(data.features)
+            .enter()
+            .append("path")
+            .attr("d", path)
+            .attr("stroke", "#f32b7b")
+            .attr("stroke-width", 1)
+            .attr("fill", d => {{
+                // Get the ISO code from the properties
+                const iso = d.properties.iso_a2;
+                
+                if (progressData[iso]) {{
+                    return colorScale(progressData[iso].percent);
+                }}
+                return "#2d3748"; // Default gray for non-tracked countries
+            }})
+            .on("mouseover", function(event, d) {{
+                const iso = d.properties.iso_a2;
+                
+                d3.select(this)
+                    .attr("stroke", "#4a1942")
+                    .attr("stroke-width", 2);
+                
+                if (progressData[iso]) {{
+                    tooltip.style("opacity", 1)
+                        .style("left", (event.pageX + 15) + "px")
+                        .style("top", (event.pageY + 15) + "px")
+                        .html(`
+                            <strong>${{progressData[iso].name}}</strong><br/>
+                            Documents: ${{progressData[iso].count.toLocaleString()}}/${{progressData[iso].target.toLocaleString()}}<br/>
+                            Completion: ${{progressData[iso].percent}}%
+                        `);
+                }}
+            }})
+            .on("mousemove", function(event) {{
+                tooltip.style("left", (event.pageX + 15) + "px")
+                       .style("top", (event.pageY + 15) + "px");
+            }})
+            .on("mouseout", function() {{
+                d3.select(this)
+                    .attr("stroke", "#f32b7b")
+                    .attr("stroke-width", 1);
+                    
+                tooltip.style("opacity", 0);
+            }});
             
-            // Set up the SVG container
-            const width = document.getElementById('map-container').clientWidth;
-            const height = 600;
+        // Add legend
+        const legendWidth = Math.min(width - 40, 200);
+        const legendHeight = 15;
+        const legendX = width - legendWidth - 20;
+        
+        const legend = svg.append("g")
+            .attr("class", "legend")
+            .attr("transform", `translate(${{legendX}}, 30)`);
+        
+        // Create gradient for legend
+        const defs = svg.append("defs");
+        const gradient = defs.append("linearGradient")
+            .attr("id", "dataGradient")
+            .attr("x1", "0%")
+            .attr("y1", "0%")
+            .attr("x2", "100%")
+            .attr("y2", "0%");
+        
+        gradient.append("stop")
+            .attr("offset", "0%")
+            .attr("stop-color", "#4a1942");
+        
+        gradient.append("stop")
+            .attr("offset", "100%")
+            .attr("stop-color", "#f32b7b");
+        
+        // Add legend title
+        legend.append("text")
+            .attr("x", legendWidth / 2)
+            .attr("y", -10)
+            .attr("text-anchor", "middle")
+            .attr("font-size", "12px")
+            .attr("fill", "#f1f5f9")
+            .text("Annotation Progress");
+        
+        // Add legend rectangle
+        legend.append("rect")
+            .attr("width", legendWidth)
+            .attr("height", legendHeight)
+            .attr("rx", 2)
+            .attr("ry", 2)
+            .style("fill", "url(#dataGradient)");
+        
+        // Add legend labels
+        legend.append("text")
+            .attr("x", 0)
+            .attr("y", legendHeight + 15)
+            .attr("text-anchor", "start")
+            .attr("font-size", "10px")
+            .attr("fill", "#94a3b8")
+            .text("0%");
+        
+        legend.append("text")
+            .attr("x", legendWidth / 2)
+            .attr("y", legendHeight + 15)
+            .attr("text-anchor", "middle")
+            .attr("font-size", "10px")
+            .attr("fill", "#94a3b8")
+            .text("50%");
+        
+        legend.append("text")
+            .attr("x", legendWidth)
+            .attr("y", legendHeight + 15)
+            .attr("text-anchor", "end")
+            .attr("font-size", "10px")
+            .attr("fill", "#94a3b8")
+            .text("100%");
+        
+        // Handle window resize
+        globalThis.resizeMap = () => {{
+            const width = mapContainer.clientWidth;
             
-            const svg = d3.select("#map-container")
-                .append("svg")
+            // Update SVG dimensions
+            d3.select("svg")
                 .attr("width", width)
-                .attr("height", height)
-                .attr("viewBox", `0 0 ${{width}} ${{height}}`)
-                .style("background-color", "#111");
-            
-            // Define color scale
-            const colorScale = d3.scaleLinear()
-                .domain([0, 100])
-                .range(["#4a1942", "#f32b7b"]);
+                .attr("viewBox", `0 0 ${{width}} ${{height}}`);
             
-            // Set up projection focused on Latin America and Spain
-            const projection = d3.geoMercator()
-                .center([-60, 0])
-                .scale(width / 5)
+            // Update projection
+            projection.scale(width / 5)
                 .translate([width / 2, height / 2]);
-                
-            const path = d3.geoPath().projection(projection);
             
-            // Tooltip setup
-            const tooltip = d3.select("#tooltip");
+            // Update paths
+            d3.selectAll("path").attr("d", path);
             
-            // Load the world GeoJSON data
-            d3.json("https://raw.githubusercontent.com/holtzy/D3-graph-gallery/master/DATA/world.geojson")
-                .then(data => {{
-                    // Filter for Latin American countries and Spain
-                    const relevantCountryCodes = Object.keys(progressData);
-                    
-                    // Draw the map
-                    svg.selectAll("path")
-                        .data(data.features)
-                        .enter()
-                        .append("path")
-                        .attr("d", path)
-                        .attr("class", "country")
-                        .attr("fill", d => {{
-                            // Get the ISO code from the properties
-                            const iso = d.properties.iso_a2;
-                            
-                            if (progressData[iso]) {{
-                                return colorScale(progressData[iso].percent);
-                            }}
-                            return "#2d3748"; // Default gray for non-tracked countries
-                        }})
-                        .on("mouseover", function(event, d) {{
-                            const iso = d.properties.iso_a2;
-                            
-                            if (progressData[iso]) {{
-                                tooltip.style("opacity", 1)
-                                    .html(`
-                                        <strong>${{progressData[iso].name}}</strong><br/>
-                                        Documents: ${{progressData[iso].count.toLocaleString()}}/${{progressData[iso].target.toLocaleString()}}<br/>
-                                        Completion: ${{progressData[iso].percent}}%
-                                    `);
-                            }}
-                        }})
-                        .on("mousemove", function(event) {{
-                            tooltip.style("left", (event.pageX + 15) + "px")
-                                   .style("top", (event.pageY + 15) + "px");
-                        }})
-                        .on("mouseout", function() {{
-                            tooltip.style("opacity", 0);
-                        }});
-                        
-                    // Add legend
-                    const legendWidth = Math.min(width - 40, 200);
-                    const legendHeight = 15;
-                    const legendX = width - legendWidth - 20;
-                    
-                    const legend = svg.append("g")
-                        .attr("class", "legend")
-                        .attr("transform", `translate(${{legendX}}, 30)`);
-                    
-                    // Create gradient for legend
-                    const defs = svg.append("defs");
-                    const gradient = defs.append("linearGradient")
-                        .attr("id", "dataGradient")
-                        .attr("x1", "0%")
-                        .attr("y1", "0%")
-                        .attr("x2", "100%")
-                        .attr("y2", "0%");
-                    
-                    gradient.append("stop")
-                        .attr("offset", "0%")
-                        .attr("stop-color", "#4a1942");
-                    
-                    gradient.append("stop")
-                        .attr("offset", "100%")
-                        .attr("stop-color", "#f32b7b");
-                    
-                    // Add legend title
-                    legend.append("text")
-                        .attr("x", legendWidth / 2)
-                        .attr("y", -10)
-                        .attr("text-anchor", "middle")
-                        .attr("font-size", "12px")
-                        .text("Annotation Progress");
-                    
-                    // Add legend rectangle
-                    legend.append("rect")
-                        .attr("width", legendWidth)
-                        .attr("height", legendHeight)
-                        .attr("rx", 2)
-                        .attr("ry", 2)
-                        .style("fill", "url(#dataGradient)");
-                    
-                    // Add legend labels
-                    legend.append("text")
-                        .attr("class", "legend-text")
-                        .attr("x", 0)
-                        .attr("y", legendHeight + 15)
-                        .attr("text-anchor", "start")
-                        .text("0%");
-                    
-                    legend.append("text")
-                        .attr("class", "legend-text")
-                        .attr("x", legendWidth / 2)
-                        .attr("y", legendHeight + 15)
-                        .attr("text-anchor", "middle")
-                        .text("50%");
-                    
-                    legend.append("text")
-                        .attr("class", "legend-text")
-                        .attr("x", legendWidth)
-                        .attr("y", legendHeight + 15)
-                        .attr("text-anchor", "end")
-                        .text("100%");
-                }});
-                
-            // Handle window resize
-            window.addEventListener('resize', () => {{
-                const width = document.getElementById('map-container').clientWidth;
-                
-                // Update SVG dimensions
-                d3.select("svg")
-                    .attr("width", width)
-                    .attr("viewBox", `0 0 ${{width}} ${{height}}`);
-                
-                // Update projection
-                projection.scale(width / 5)
-                    .translate([width / 2, height / 2]);
-                
-                // Update paths
-                d3.selectAll("path").attr("d", path);
-                
-                // Update legend position
-                const legendWidth = Math.min(width - 40, 200);
-                const legendX = width - legendWidth - 20;
-                
-                d3.select(".legend")
-                    .attr("transform", `translate(${{legendX}}, 30)`);
-            }});
-        </script>
-    </body>
-    </html>
+            // Update legend position
+            const legendWidth = Math.min(width - 40, 200);
+            const legendX = width - legendWidth - 20;
+            
+            d3.select(".legend")
+                .attr("transform", `translate(${{legendX}}, 30)`);
+        }};
+        
+        window.addEventListener('resize', globalThis.resizeMap);
+    }}
     """
 
+# Function to update the map data and trigger a reload
+def update_map():
+    progress_json = json.dumps(annotation_progress)
+    return progress_json
+
 # Create Gradio interface
 with gr.Blocks(theme=gr.themes.Soft(primary_hue="pink", secondary_hue="purple")) as demo:
     argilla_server = client.http_client.base_url if hasattr(client, 'http_client') else "Not connected"
     
     with gr.Row():
         gr.Markdown(f"""
-        # Argilla Annotation Progress Map
+        # Latin America & Spain Annotation Progress Map
         
         ### Connected to Argilla server: {argilla_server}
         
         This dashboard visualizes annotation progress across Latin America and Spain.
-        The webhook listens for `response.created` events from datasets ending with `_responder_preguntas`.
         """)
     
     with gr.Row():
         with gr.Column(scale=2):
-            # Map visualization
-            map_html = gr.HTML(create_map_html(json.dumps(annotation_progress)), label="Annotation Progress Map")
+            # Map visualization - empty at first
+            map_html = gr.HTML(create_map_html(), label="Annotation Progress Map")
             
-            # Stats section
-            with gr.Accordion("Overall Statistics", open=False):
-                total_docs = gr.Number(value=0, label="Total Documents Collected")
-                avg_completion = gr.Number(value=0, label="Average Completion (%)")
-                countries_over_50 = gr.Number(value=0, label="Countries Over 50% Complete")
+            # Hidden element to store map data 
+            map_data = gr.JSON(value=json.dumps(annotation_progress), visible=False)
         
         with gr.Column(scale=1):
             # Recent events log
             events_json = gr.JSON(label="Recent Events", value={})
             
+            # Overall statistics
+            total_docs = gr.Number(value=0, label="Total Documents", interactive=False)
+            avg_completion = gr.Number(value=0, label="Average Completion (%)", interactive=False)
+            countries_over_50 = gr.Number(value=0, label="Countries Over 50%", interactive=False)
+            
             # Country details
-            with gr.Accordion("Country Details", open=True):
-                country_selector = gr.Dropdown(
-                    choices=[f"{data['name']} ({code})" for code, data in COUNTRY_MAPPING.items()],
-                    label="Select Country"
-                )
-                country_progress = gr.JSON(label="Country Progress", value={})
+            country_selector = gr.Dropdown(
+                choices=[f"{data['name']} ({code})" for code, data in COUNTRY_MAPPING.items()],
+                label="Select Country"
+            )
+            country_progress = gr.JSON(label="Country Progress", value={})
     
-    # Functions to update the UI
-    def update_map():
-        progress_json = json.dumps(annotation_progress)
-        return create_map_html(progress_json)
-    
-    def update_stats():
-        total = sum(data["count"] for data in annotation_progress.values())
-        percentages = [data["percent"] for data in annotation_progress.values()]
-        avg = sum(percentages) / len(percentages) if percentages else 0
-        countries_50_plus = sum(1 for p in percentages if p >= 50)
+    # Load the D3 script when data is updated
+    def load_map_script(data):
+        return None, create_d3_script(data)
         
-        return total, avg, countries_50_plus
+    # Refresh button
+    refresh_btn = gr.Button("Refresh Map")
+    refresh_btn.click(
+        fn=update_map,
+        inputs=None,
+        outputs=map_data
+    )
+    
+    # When map_data is updated, reload the D3 script
+    map_data.change(
+        fn=load_map_script,
+        inputs=map_data,
+        outputs=[None, None],
+        _js=create_d3_script(json.dumps(annotation_progress))
+    )
     
+    # Function to update country details
     def update_country_details(country_selection):
         if not country_selection:
             return {}
@@ -424,30 +414,35 @@ with gr.Blocks(theme=gr.themes.Soft(primary_hue="pink", secondary_hue="purple"))
         if code in annotation_progress:
             return annotation_progress[code]
         return {}
-
-    # Simple event processing functions for better compatibility
-    def update_events():
-        return read_next_event()
-        
-    def update_all_stats():
-        return update_stats()
-    
-    # Use separate timers for each component for better compatibility
-    gr.Timer(1, active=True).tick(update_events, outputs=events_json)
-    gr.Timer(5, active=True).tick(update_all_stats, outputs=[total_docs, avg_completion, countries_over_50])
     
+    # Update country details when a country is selected
     country_selector.change(
         fn=update_country_details,
         inputs=[country_selector],
         outputs=[country_progress]
     )
     
-    # Use refresh button instead of timer for map updates (more compatible across versions)
-    refresh_btn = gr.Button("Refresh Map")
-    refresh_btn.click(
-        fn=update_map,
-        inputs=None,
-        outputs=[map_html]
+    # Update events function for the timer
+    def update_events():
+        event = read_next_event()
+        
+        # Calculate stats
+        stats = update_stats()
+        
+        # If this is a progress update, update the map data
+        if event.get("event") == "progress_update":
+            # This will indirectly trigger a map refresh through the change event
+            return event, json.dumps(annotation_progress), stats[0], stats[1], stats[2]
+        
+        return event, None, stats[0], stats[1], stats[2]
+    
+    # Make final updates to Gradio demo
+    demo.load(None, None, None, _js=create_d3_script(json.dumps(annotation_progress)))
+
+    # Use timer to check for new events and update stats
+    gr.Timer(1, active=True).tick(
+        update_events,
+        outputs=[events_json, map_data, total_docs, avg_completion, countries_over_50]
     )
 
 # Mount the Gradio app to the FastAPI server