Upload 6 files

.env

@@ -0,0 +1 @@
+GOOGLE_API_KEY=you_api

backend.py

@@ -0,0 +1,177 @@
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+import sqlite3
+import pandas as pd
+import os
+from dotenv import load_dotenv
+import google.generativeai as genai
+
+app = FastAPI()
+
+# Load environment variables and configure Genai
+load_dotenv()
+genai.configure(api_key=os.getenv('GOOGLE_API_KEY'))
+
+# Define the schema for the incoming request
+class Query(BaseModel):
+    question: str
+    data_source: str
+
+def get_gemini_response(question, prompt):
+    model = genai.GenerativeModel('gemini-1.5-pro') # https://ai.google.dev/pricing?authuser=1#1_5pro
+    response = model.generate_content([prompt, question])
+    return response.text
+
+# Update column and table names for the new dataset
+sql_cols_human = 'REQUESTID', 'DATETIMEINIT', 'SOURCE', 'DESCRIPTION', 'REQCATEGORY', 'STATUS', 'REFERREDTO', 'DATETIMECLOSED', 'City', 'State', 'Ward', 'Postcode'
+csv_columns_human = ['REQUESTID', 'DATETIMEINIT', 'SOURCE', 'DESCRIPTION', 'REQCATEGORY', 'STATUS', 'REFERREDTO', 'DATETIMECLOSED', 'City', 'State', 'Ward', 'Postcode']
+sql_cols = 'REQUESTID', 'DATETIMEINIT', 'SOURCE', 'DESCRIPTION', 'REQCATEGORY', 'STATUS', 'REFERREDTO', 'DATETIMECLOSED', 'City', 'State', 'Ward', 'Postcode'
+# csv_columns = ["REQUESTID", "DATETIMEINIT",  "SOURCE", "DESCRIPTION", "REQCATEGORY", "STATUS", "REFERREDTO", "DATETIMECLOSED", "PROBADDRESS" "City", "State", "Ward", "Postcode"]
+
+def get_csv_columns():
+    df = pd.read_csv('wandsworth_callcenter_sampled.csv')
+    return df.columns.tolist()
+
+csv_columns = get_csv_columns()
+print(csv_columns)
+
+sql_prompt = f"""
+You are an expert in converting English questions to SQLite code!
+The SQLite database has the name CALLCENTER_REQUESTS and has the following Columns: {', '.join(sql_cols)}
+
+Here are some key details about the dataset:
+- `SOURCE`: Phone, Online Form, FixMyStreet, Email, Telephone/Email, Telephone Voicemail, Other, Local Council Office.
+- `REQCATEGORY`: Blocked Drains, Council Building Maintenance, Fly-Tipping, Street and Pavement Maintenance, Recycling, Traffic Signage Issues, Parks Maintenance, Graffiti Removal, Tree Maintenance.
+- `STATUS`: Resolved, In Progress, Cancelled by Customer, Referred to External Agency, Work Order Created, Under Review.
+- `REFERREDTO`: Council Enforcement, Transport for London (TfL), Thames Water, Royal Mail, UK Power Networks.
+
+For example:
+- Would you please list all unresolved calls? command: SELECT * FROM CALLCENTER_REQUESTS WHERE STATUS='In Progress';
+- Would you please count the total number of calls? command: SELECT COUNT(*) FROM CALLCENTER_REQUESTS;
+- List all unique wards please? command: SELECT DISTINCT Ward FROM CALLCENTER_REQUESTS;
+
+Also, the SQL code should not have ''' in the beginning or at the end, and SQL word in output.
+Ensure that you only generate valid SQLite database queries, not pandas or Python code.
+"""
+
+
+
+csv_prompt = f"""
+You are an expert in analyzing CSV data and converting English questions to pandas query syntax.
+The CSV file is named 'wandsworth_callcenter_sampled.csv' and contains residents' call information in Wandsworth Council.
+The available columns in the CSV file are: {', '.join(csv_columns)}
+
+Here are some key details about the dataset:
+- `SOURCE`: Phone, Online Form, FixMyStreet, Email, Telephone/Email, Telephone Voicemail, Other, Local Council Office.
+- `REQCATEGORY`: Blocked Drains, Council Building Maintenance, Fly-Tipping, Street and Pavement Maintenance, Recycling, Traffic Signage Issues, Parks Maintenance, Graffiti Removal, Tree Maintenance.
+- `STATUS`: Resolved, In Progress, Cancelled by Customer, Referred to External Agency, Work Order Created, Under Review.
+- `REFERREDTO`: Council Enforcement, Transport for London (TfL), Thames Water, Royal Mail, UK Power Networks.
+
+For example:
+- How many calls in total? len(df.REQUESTID)
+- What are all the calls referred to external agencies? df[df['REFERREDTO'].notna()]
+- Would you please show the top 5 most frequent call categories? df['REQCATEGORY'].value_counts().head(5)
+
+Please ensure:
+1. Always reference columns using `df['COLUMN_NAME']`.
+2. Do not use Python lists like `['COLUMN_NAME']` to refer to columns.
+3. Provide only the pandas query syntax without any additional explanation or markdown formatting.
+Make sure to use only the columns that are available in the CSV file.
+Ensure that you only generate valid pandas queries. NO SQL or other types of code/syntax.
+
+"""
+
+def execute_sql_query(query):
+    conn = sqlite3.connect('wandsworth_callcenter_sampled.db')
+    try:
+        cursor = conn.cursor()
+        cursor.execute(query)
+        result = cursor.fetchall()
+        return result
+    except sqlite3.Error as e:
+        # Capture and explain SQL errors
+        sql_error_message = str(e)
+        # Send the error message back to Gemini for explanation        
+        error_prompt = f"""
+        You are an expert SQL debugger and an assistant of the director. An error occurred while executing the following query:
+        {query}
+
+        The error was: {sql_error_message}
+        Please explain the error in simple laymen terms. Do Not explain. 
+        Do Not include any programming code, e.g. sql or python syntax, etc.
+        And finally politely remind the user there are only information about the following columns{', '.join(sql_cols_human)}.
+        Explain this in layman's terms and remind the user that the dataset contains the following columns: {', '.join(sql_cols_human)}.
+        """
+        explanation = get_gemini_response("", error_prompt)
+        raise HTTPException(status_code=400, detail={"error": sql_error_message, "explanation": explanation})
+    finally:
+        conn.close()
+
+
+
+
+def execute_pandas_query(query):
+    df = pd.read_csv('wandsworth_callcenter_sampled.csv')
+    df.columns = df.columns.str.upper()  # Normalize column names to uppercase
+    print(f"df is loaded. The first line is: {df.head(1)}")
+
+    # Remove code block indicators (e.g., ```python and ```)
+    query = query.replace("```python", "").replace("```", "").strip()
+
+    # Split query into lines
+    query_lines = query.split("\n")  # Split into individual statements
+    try:
+        result = None
+        exec_context = {'df': df, 'pd': pd}  # Execution context for exec()
+        for line in query_lines:
+            line = line.strip()  # Remove extra spaces
+            if line:  # Skip empty lines
+                print(f"Executing line: {line}")
+                exec(line, exec_context)  # Execute each line in the context
+
+        # Retrieve the final result if the last line is a statement
+        result = eval(query_lines[-1].strip(), exec_context)  # Evaluate the last line for the result
+
+        print(f"Query Result Before Serialization: {result}")
+
+        # Handle DataFrame results
+        if isinstance(result, pd.DataFrame):
+            # Replace NaN and infinite values with JSON-compliant values
+            result = result.replace([float('inf'), -float('inf')], None).fillna(value="N/A")
+            return result.to_dict(orient='records')
+
+        # Handle Series results
+        elif isinstance(result, pd.Series):
+            result = result.replace([float('inf'), -float('inf')], None).fillna(value="N/A")
+            return result.to_dict()
+
+        # Handle scalar results
+        else:
+            return result
+
+    except Exception as e:
+        print(f"Error: {e}")
+        raise HTTPException(status_code=400, detail=f"Pandas Error: {str(e)}")
+
+
+
+
+
+@app.post("/query")
+async def process_query(query: Query):
+    if query.data_source == "SQL Database":
+        ai_response = get_gemini_response(query.question, sql_prompt)
+        try:
+            result = execute_sql_query(ai_response)
+            return {"query": ai_response, "result": result}
+        except HTTPException as e:
+            error_detail = e.detail
+            return {"query": ai_response, "error": error_detail["error"], "explanation": error_detail["explanation"]}
+    else:  # CSV Data
+        ai_response = get_gemini_response(query.question, csv_prompt)
+        print(f"\n\nai_response: {ai_response}")
+        try:
+            result = execute_pandas_query(ai_response)
+            return {"query": ai_response, "result": result, "columns": csv_columns}
+        except HTTPException as e:
+            raise HTTPException(status_code=400, detail=f"Error in pandas query: {e.detail}")

frontend.py

@@ -0,0 +1,142 @@
+import streamlit as st
+import requests
+import pandas as pd
+
+# Page Configuration
+st.set_page_config(
+    page_title="CallDataAI - Wandsworth Council Call Center Analysis",
+    page_icon="📞",
+    layout="wide",
+    initial_sidebar_state="expanded",
+)
+
+# Sidebar
+st.sidebar.title("📞 CallDataAI")
+st.sidebar.markdown(
+    """
+    **Welcome to CallDataAI**, your AI-powered assistant for analyzing Wandsworth Council's Call Center data. Use the menu below to:
+    - Select the data source (SQL/CSV)
+    - Run pre-defined or custom queries
+    - Gain actionable insights
+    """
+)
+
+# Data source selection
+st.sidebar.markdown("### Select Data Source:")
+data_source = st.sidebar.radio("", ('SQL Database', 'CSV Database'))
+
+# Common queries section
+st.sidebar.markdown("### Common Queries:")
+common_queries = {
+    'SQL Database': [
+        'List all unique Source',
+        'List all unique request categories',
+        'List all unique wards and their postcodes',
+        'Count the total number of calls',
+        'List all unresolved calls',
+        'What are the total number of requests per year?',
+        'What are the average time (days) to close request per request category?',
+    ],
+    'CSV Database': [
+        'Count total number of call requests',
+        'List all calls referred to external agencies',
+        'Show top 5 most frequent call categories',
+    ]
+}
+
+for idx, query in enumerate(common_queries[data_source]):
+    if st.sidebar.button(query, key=f"query_button_{idx}"):  # Add unique key
+        st.session_state["common_query"] = query
+
+
+
+
+# Title and Description
+st.title("📞 CallDataAI - Wandsworth Council Call Center Analysis")
+st.markdown(
+    """
+    **CallDataAI** is an AI-powered chatbot designed for analyzing Wandsworth Council's Call Center data.
+    Input natural language queries to explore the data and gain actionable insights.
+    """
+)
+
+# Input Section
+with st.container():
+    st.markdown("### Enter Your Question")
+    question = st.text_input(
+        "Input:", key="input", value=st.session_state.get("common_query", ""), placeholder="Type your query here..."
+    )
+    submit = st.button("Submit", type="primary")
+
+# Main Content
+if submit:
+    # Send request to FastAPI backend
+    with st.spinner("Processing your request..."):
+        response = requests.post(
+            "http://localhost:8000/query", json={"question": question, "data_source": data_source}
+        )
+
+    # Handle response
+    if response.status_code == 200:
+        data = response.json()
+
+        # Error Handling
+        if "error" in data:
+            with st.expander("Error Explanation"):
+                st.error(data["explanation"])
+
+        # Display Results
+        else:
+            col1, col2 = st.columns(2)
+
+            with col1:
+                st.markdown(f"### Generated {'SQL' if data_source == 'SQL Database' else 'Pandas'} Query")
+                st.code(data["query"], language="sql" if data_source == "SQL Database" else "python")
+
+            with col2:
+                st.markdown("### Query Results")
+                result = data["result"]
+
+                if isinstance(result, list) and len(result) > 0:
+                    if isinstance(result[0], dict):
+                        df = pd.DataFrame(result)
+                        st.dataframe(df)
+                    elif isinstance(result[0], list):
+                        df = pd.DataFrame(result)
+                        st.dataframe(df)
+                    else:
+                        st.write(result)
+
+                elif isinstance(result, dict):
+                    st.json(result)
+
+                else:
+                    st.write(result)
+
+            if data_source == "CSV Database":
+                st.markdown("### Available CSV Columns")
+                st.write(data["columns"])
+
+            # Update chat history in session state
+            if "chat_history" not in st.session_state:
+                st.session_state["chat_history"] = []
+
+            st.session_state["chat_history"].append(f"🔧({data_source}): {question}")
+            st.session_state["chat_history"].append(f"🤖: {data['query']}")
+
+    else:
+        st.error(f"Error processing your request: {response.text}")
+
+# Chat History Section
+with st.container():
+    st.markdown("### Chat History")
+    if "chat_history" in st.session_state:
+        for message in st.session_state["chat_history"]:
+            st.text(message)
+    if st.button("Clear Chat History"):
+        st.session_state["chat_history"] = []
+        st.success("Chat history cleared!")
+
+st.markdown("---")
+st.markdown("Developed by Lorentz Yeung, 2024 Christmas")
+st.markdown("Contact: [email protected] or [email protected]")

requirements.txt

@@ -0,0 +1,9 @@
+fastapi==0.110.3
+google-generativeai==0.8.3
+pandas==2.2.3
+pydantic==2.9.2
+python-dotenv==1.0.1
+uvicorn==0.30.6
+streamlit==1.40.1
+requests==2.32.3
+