logging changes

app.py

@@ -9,7 +9,33 @@ from utils.retriever import retrieve_paragraphs
 from utils.generator import generate
 import json
 import ast
+from utils.logger import ChatLogger
+from pathlib import Path
+from huggingface_hub import CommitScheduler, HfApi
+import os 
+# fetch tokens from Gradio secrets
+SPACES_LOG = os.environ.get("EUDR_SPACES_LOG")
+if not SPACES_LOG:
+    raise ValueError("EUDR_SPACES_LOG not found in environment")
+# create the local logs repo
+JSON_DATASET_DIR = Path("json_dataset")
+JSON_DATASET_DIR.mkdir(parents=True, exist_ok=True)
+JSON_DATASET_PATH = JSON_DATASET_DIR / f"logs-{uuid4()}.json"
 
+# the logs are written to dataset repo periodically from local logs
+# https://huggingface.co/spaces/Wauplin/space_to_dataset_saver
+scheduler = CommitScheduler(
+     repo_id="GIZ/spaces_logs",
+     repo_type="dataset",
+     folder_path=JSON_DATASET_DIR,
+     path_in_repo="eudr_chatbot",
+     token=SPACES_LOG ) 
+
+
+
+# Initialize logger with shared scheduler
+# scheduler.start()  # Start the scheduler
+chat_logger = ChatLogger(scheduler=scheduler)
 # Sample questions for examples 
 SAMPLE_QUESTIONS = {
     "Análisis de la deforestación": [
@@ -73,7 +99,7 @@ def make_html_source(source,i):
 
     return card
 
-async def chat_response(query, history, method, country, uploaded_file):
+async def chat_response(query, history, method, country, uploaded_file, request=None):
     """Generate chat response based on method and inputs"""
     
     # Skip processing if this is an auto-generated file analysis message
@@ -128,6 +154,14 @@ async def chat_response(query, history, method, country, uploaded_file):
             docs_html = "".join(docs_html)
             response = await generate(query=query, context=retrieved_paragraphs)
             
+            # Log the interaction
+            chat_logger.log(
+                query=query,
+                answer=response,
+                retrieved_content=context_retrieved_lst,
+                request=request
+            )
+
         except Exception as e:
             response = f"Error retrieving information: {str(e)}"
 
@@ -491,24 +525,33 @@ with gr.Blocks(title="EUDR Bot", theme=theme, css="style.css") as demo:
         outputs=[textbox]
     )
     
-    # Sample questions dropdown
-    dropdown_samples.change(
-        change_sample_questions,
-        [dropdown_samples],
-        sample_groups
-    )
-    
+ 
+    # Feedback buttons
+    def log_feedback(feedback, chatbot):
+        # Get the last interaction from chatbot history
+        if chatbot and len(chatbot) > 0:
+            last_query, last_response = chatbot[-1]
+            chat_logger.log(
+                query=last_query,
+                answer=last_response,
+                retrieved_content=[],  # We don't have access to the original retrieved content here
+                feedback=feedback
+            )
+        return (gr.update(visible=False), gr.update(visible=True))
+
     # Feedback buttons
     okay_btn.click(
-        lambda: (gr.update(visible=False), gr.update(visible=True)),
+        lambda chatbot: log_feedback("positive", chatbot),
+        inputs=[chatbot],
         outputs=[feedback_row, feedback_thanks]
     )
     
     not_okay_btn.click(
-        lambda: (gr.update(visible=False), gr.update(visible=True)),
+        lambda chatbot: log_feedback("negative", chatbot),
+        inputs=[chatbot],
         outputs=[feedback_row, feedback_thanks]
     )
 
 # Launch the app
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch()

utils/logger.py

@@ -0,0 +1,137 @@
+import json
+import logging
+from datetime import datetime
+from uuid import uuid4
+import requests
+from pathlib import Path
+from datasets import load_dataset, Dataset
+import os
+from huggingface_hub import CommitScheduler, HfApi
+import random
+
+class ChatLogger:
+    def __init__(self, scheduler):
+        """Initialize the chat logger with paths and configurations"""
+        if not scheduler:
+            raise ValueError("Scheduler is required")
+            
+        self.scheduler = scheduler
+        self.json_dataset_dir = scheduler.folder_path
+        self.logs_path = self.json_dataset_dir / f"logs-{uuid4()}.jsonl"
+
+    def get_client_ip(self, request=None):
+        """Get the client IP address from the request context"""
+        try:
+            if request:
+                # Try different headers that might contain the real IP
+                ip = request.client.host
+                # Check for proxy headers
+                forwarded_for = request.headers.get('X-Forwarded-For')
+                if forwarded_for:
+                    # X-Forwarded-For can contain multiple IPs - first one is the client
+                    ip = forwarded_for.split(',')[0].strip()
+                
+                logging.debug(f"Client IP detected: {ip}")
+                return ip
+        except Exception as e:
+            logging.error(f"Error getting client IP: {e}")
+        return "127.0.0.1"
+
+    def get_client_location(self, ip_address):
+        """Get geolocation info using ipapi.co"""
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
+        }
+        try:
+            response = requests.get(
+                f'https://ipapi.co/{ip_address}/json/',
+                headers=headers,
+                timeout=5
+            )
+            if response.status_code == 200:
+                data = response.json()
+                # Add random noise between -0.01 and 0.01 degrees (roughly ±1km)
+                lat = data.get('latitude')
+                lon = data.get('longitude')
+                if lat is not None and lon is not None:
+                    lat += random.uniform(-0.01, 0.01)
+                    lon += random.uniform(-0.01, 0.01)
+                
+                return {
+                    'city': data.get('city'),
+                    'region': data.get('region'),
+                    'country': data.get('country_name'),
+                    'latitude': lat,
+                    'longitude': lon
+                }
+            elif response.status_code == 429:
+                logging.warning(f"Rate limit exceeded for IP lookup")
+                return None
+            else:
+                logging.error(f"Error in IP lookup: Status code {response.status_code}")
+                return None
+                
+        except requests.exceptions.RequestException as e:
+            logging.error(f"Request failed in IP lookup: {str(e)}")
+            return None
+
+    def create_log_entry(self, query, answer, retrieved_content, feedback=None, request=None):
+        """Create a structured log entry with all required fields"""
+        timestamp = datetime.now().timestamp()
+        
+        # Get client location if request is provided
+        ip = self.get_client_ip(request) if request else None
+        location = self.get_client_location(ip) if ip else None
+        
+        log_entry = {
+            "record_id": str(uuid4()),
+            "session_id": str(uuid4()),  # In practice, this should be passed in from the session
+            "time": str(timestamp),
+            "client_location": location,
+            "question": query,
+            "answer": answer,
+            "retrieved_content": retrieved_content if isinstance(retrieved_content, list) else [retrieved_content],
+            "feedback": feedback
+        }
+        
+        return log_entry
+
+    def save_local(self, log_entry):
+        """Save log entry to local JSONL file"""
+        try:
+            # Reorder fields for consistency
+            field_order = [
+                "record_id",
+                "session_id",
+                "time",
+                "client_location",
+                "question",
+                "answer",
+                "retrieved_content",
+                "feedback"
+            ]
+            ordered_logs = {k: log_entry.get(k) for k in field_order if k in log_entry}
+            
+            with self.scheduler.lock:
+                with open(self.logs_path, 'a') as f:
+                    json.dump(ordered_logs, f)
+                    f.write('\n')
+                    logging.info("Log entry saved")
+            return True
+        except Exception as e:
+            logging.error(f"Error saving to local file: {str(e)}")
+            return False
+
+    def log(self, query, answer, retrieved_content, feedback=None, request=None):
+        """Main logging method that handles both local and HF storage"""
+        # Create log entry
+        log_entry = self.create_log_entry(
+            query=query,
+            answer=answer,
+            retrieved_content=retrieved_content,
+            feedback=feedback,
+            request=request
+        )
+        
+        # Save locally with thread safety
+        return self.save_local(log_entry)