Update

.gitignore

@@ -41,4 +41,9 @@ flagged/
 # Misc
 .DS_Store 
 .gradio/
-lib/
+lib/
+
+# Scrapy
+.scrapy/
+
+_*.py

README.md

@@ -34,7 +34,7 @@ A Gradio interface that scrapes and analyzes real estate projects using LangChai
 
 3. Install the dependencies:
     ```sh
-    uv pip install -r requirements.txt
+    uv pip sync requirements.txt
     ```
 
 4. Run the application:
@@ -44,5 +44,5 @@ A Gradio interface that scrapes and analyzes real estate projects using LangChai
 
 5. Upgrade all packages to their latest versions:
     ```sh
-    uv pip install --upgrade -r requirements.txt
+   uv pip compile - -o requirements.txt 
     ```

app.py

@@ -6,8 +6,9 @@ from langchain_community.agent_toolkits.sql.toolkit import SQLDatabaseToolkit
 import gradio as gr
 from dotenv import load_dotenv
 from langchain.schema import HumanMessage, AIMessage
+from langchain.prompts import ChatPromptTemplate
 import os.path
-
+from init_db import db_description
 # Load environment variables
 load_dotenv()
 
@@ -18,15 +19,38 @@ if not os.path.exists('estate.db'):
     )
 
 # Initialize model and database
-model = ChatOpenAI(model="gpt-4", streaming=True)
+model = ChatOpenAI(model="gpt-4o-2024-08-06", streaming=True)
 db = SQLDatabase.from_uri("sqlite:///estate.db")
 
 # Set up SQL toolkit and tools
 toolkit = SQLDatabaseToolkit(db=db, llm=model)
 tools = toolkit.get_tools()
 
+sql_distance_query = """
+SELECT 
+    id, 
+    url,
+    structure->>'$.lat' AS lat,
+    structure->>'$.lng' AS lng,
+    (6371000 * acos(
+        cos(radians(50.08804)) * cos(radians(CAST(structure->>'$.lat' AS FLOAT))) * 
+        cos(radians(CAST(structure->>'$.lng' AS FLOAT)) - radians(14.42076)) + 
+        sin(radians(50.08804)) * sin(radians(CAST(structure->>'$.lat' AS FLOAT)))
+    )) AS distance
+FROM project
+WHERE structure->>'$.lat' IS NOT NULL 
+AND structure->>'$.lng' IS NOT NULL
+ORDER BY distance ASC
+LIMIT 5;
+"""
+
 # Get the OpenAI tools agent prompt
-prompt = hub.pull("hwchase17/openai-tools-agent")
+prompt = ChatPromptTemplate.from_messages([
+  ("system", "You are a helpful assistant. You speak Czech. You can answer questions about real estate projects (novostavby) in Czech Republic. You have access to a database of real estate projects (translate output to Czech too). Some info about the structure of the database: " + db_description + "Note that some json values can be null so you must sometimes check with IS NOT NULL. For calculating you can adapt this query: " + sql_distance_query),
+  ("placeholder", "{chat_history}"),
+  ("human", "{input}"),
+  ("placeholder", "{agent_scratchpad}"),
+])
 
 # Create the agent with OpenAI tools format
 agent = create_openai_tools_agent(
@@ -61,9 +85,10 @@ def chat_with_sql(message, history):
 demo = gr.ChatInterface(
     fn=chat_with_sql,
     title="Estate Chat",
-    description="Ask questions about 🇨🇿 real estate projects!",
+    description="Zeptej se me na cokoli o novostavbách v ČR 🇨🇿",
     examples=[
-        "What is project with the lowest deposit?",
+        "Projekt s nejnižším vkladem",
+        "Nejlevnejsi byt v Praze",
     ],
     type="messages"
 )

init_db.py

@@ -0,0 +1,51 @@
+from peewee import *
+from datetime import datetime
+from playhouse.sqlite_ext import *
+
+# Initialize database
+
+json_structure = "JSON column with hash with keys 'title': str, official title of the project, 'deposit': int, initial deposit (in percentage), 'min_price': int, lowest available property price with VAT. It should not be booked or sold., 'city': str, city of the project, 'lat': float, gps coordinates of the project, 'lng': float, gps coordinates of the project, 'start_year': int, year of construction start, 'end_year': int, estimated year of construction end. Some values can be null/unknown"
+
+db = SqliteExtDatabase('estate.db', pragmas=(
+    ('cache_size', -1024 * 64),  # 64MB page-cache.
+    ('journal_mode', 'wal'),  # Use WAL-mode (you should always use this!).
+    ('foreign_keys', 1)))  # Enforce foreign-key constraints.
+# Define the Project model
+
+db_description = f"""Table "project" - list of real estate projects (novostavby) in Czech Republic
+    url: url of the project.
+    structure: {json_structure}.
+    content: contents of the website.
+    created_at: date and time of creation.
+"""
+
+class Project(Model):
+    url = CharField(unique=True)
+    structure = JSONField(null=True)
+    content = TextField(null=True)
+    created_at = DateTimeField(default=datetime.now)
+
+    class Meta:
+        database = db
+
+def init_database():
+    """Initialize the database and create tables"""
+    print("Initializing database...")
+    db.connect()
+    db.drop_tables([Project])
+    db.create_tables([Project])
+    print("Created tables successfully!")
+    
+    # Add some test data if needed
+    test_urls = [
+        'https://brnojedna.cz/'
+    ]
+    
+    for url in test_urls:
+        Project.get_or_create(url=url)
+    
+    print(f"Number of projects in database: {Project.select().count()}")
+    db.close()
+
+if __name__ == "__main__":
+    init_database() 

requirements.in

@@ -6,5 +6,7 @@ python-dotenv
 beautifulsoup4
 requests
 peewee
-browser_use
 openai
+crawl4ai
+datasketch
+geocoder