Update app.py

app.py

@@ -1,49 +1,59 @@
-import logging
-
-# Set up logging
-logging.basicConfig(level=logging.DEBUG)
-from langchain_openai import OpenAIEmbeddings
 import os
 import re
-import folium
-import gradio as gr
 import time
 import requests
-from googlemaps import Client as GoogleMapsClient
-from gtts import gTTS
+import logging
+import folium
+import gradio as gr
 import tempfile
-import string
+import numpy as np
+from gtts import gTTS
+from googlemaps import Client as GoogleMapsClient
+from diffusers import StableDiffusion3Pipeline
+
+from langchain_openai import OpenAIEmbeddings, ChatOpenAI
+from langchain_pinecone import PineconeVectorStore
+from langchain.prompts import PromptTemplate
+from langchain.chains import RetrievalQA
+from langchain.chains.conversation.memory import ConversationBufferWindowMemory
+from langchain.agents import Tool, initialize_agent
+
+# Set up logging
+logging.basicConfig(level=logging.DEBUG)
 
+# Initialize OpenAI embeddings
 embeddings = OpenAIEmbeddings(api_key=os.environ['OPENAI_API_KEY'])
 
-from pinecone import Pinecone, ServerlessSpec
+# Initialize Pinecone
+from pinecone import Pinecone
 pc = Pinecone(api_key=os.environ['PINECONE_API_KEY'])
 
 index_name = "omaha-details"
-
-from langchain_pinecone import PineconeVectorStore
-
 vectorstore = PineconeVectorStore(index_name=index_name, embedding=embeddings)
 retriever = vectorstore.as_retriever(search_kwargs={'k': 5})
 
-from langchain_openai import ChatOpenAI
-from langchain.prompts import PromptTemplate
-from langchain.chains import RetrievalQA
-from langchain.chains.conversation.memory import ConversationBufferWindowMemory
-from langchain.agents import Tool, initialize_agent
+# Initialize ChatOpenAI model
+chat_model = ChatOpenAI(api_key=os.environ['OPENAI_API_KEY'],
+                        temperature=0, model='gpt-4o')
+
+conversational_memory = ConversationBufferWindowMemory(
+    memory_key='chat_history',
+    k=10,
+    return_messages=True
+)
 
-# Build prompt
-template1 = """You are an expert concierge who is helpful and a renowned guide for Omaha, Nebraska.Based on todays weather is sunny bright day and date is 17th june 2024 , Use the following pieces of context, 
-memory, and message history, along with your knowledge of perennial events in Omaha, Nebraska, to answer the question at the end.If you don't know the answer, just say "Homie, I need to get more data for this," and don't try to make up an answer. 
+# Define prompt templates
+template1 = """You are an expert concierge who is helpful and a renowned guide for Omaha, Nebraska. Based on today's weather being a sunny bright day and the date is 17th June 2024, use the following pieces of context, 
+memory, and message history, along with your knowledge of perennial events in Omaha, Nebraska, to answer the question at the end. If you don't know the answer, just say "Homie, I need to get more data for this," and don't try to make up an answer. 
 Use fifteen sentences maximum. Keep the answer as detailed as possible. Always include the address, time, date, and
 event type and description. Always say "It was my pleasure!" at the end of the answer.
 {context}
 Question: {question}
 Helpful Answer:"""
 
-template2 = """You are an expert concierge who is helpful and a renowned guide for Omaha, Nebraska.Based on todays weather is sunny bright day and date is 17th june 2024 ,take the location or address but don't show the location or address on the output prompts Use the following pieces of context, 
-memory, and message history, along with your knowledge of perennial events in Omaha, Nebraska, to answer the question at the end.If you don't know the answer, just say "Homie, I need to get more data for this," and don't try to make up an answer. 
-Keep the answer short and sweet crisp.Always say "It was my pleasure!" at the end of the answer.
+template2 = """You are an expert concierge who is helpful and a renowned guide for Omaha, Nebraska. Based on today's weather being a sunny bright day and the date is 17th June 2024, take the location or address but don't show the location or address on the output prompts. Use the following pieces of context, 
+memory, and message history, along with your knowledge of perennial events in Omaha, Nebraska, to answer the question at the end. If you don't know the answer, just say "Homie, I need to get more data for this," and don't try to make up an answer. 
+Keep the answer short and sweet and crisp. Always say "It was my pleasure!" at the end of the answer.
 {context}
 Question: {question}
 Helpful Answer:"""
@@ -51,15 +61,6 @@ Helpful Answer:"""
 QA_CHAIN_PROMPT_1 = PromptTemplate(input_variables=["context", "question"], template=template1)
 QA_CHAIN_PROMPT_2 = PromptTemplate(input_variables=["context", "question"], template=template2)
 
-chat_model = ChatOpenAI(api_key=os.environ['OPENAI_API_KEY'],
-                        temperature=0, model='gpt-4o')
-
-conversational_memory = ConversationBufferWindowMemory(
-    memory_key='chat_history',
-    k=10,
-    return_messages=True
-)
-
 # Define the retrieval QA chain
 def build_qa_chain(prompt_template):
     qa_chain = RetrievalQA.from_chain_type(
@@ -72,7 +73,7 @@ def build_qa_chain(prompt_template):
         Tool(
             name='Knowledge Base',
             func=qa_chain,
-            description='use this tool when answering general knowledge queries to get more information about the topic'
+            description='Use this tool when answering general knowledge queries to get more information about the topic'
         )
     ]
     return qa_chain, tools
@@ -108,17 +109,6 @@ def generate_answer(message, choice):
     addresses = extract_addresses(response['output'])
     return response['output'], addresses
 
-# def bot(history, choice):
-#     if not history:
-#         return history
-#     response = generate_answer(history[-1][0], choice)
-#     history[-1][1] = ""
-#     for character in response:
-#         history[-1][1] += character
-#         time.sleep(0.05)
-#         yield history
-
-# Define the bot function
 def bot(history, choice):
     if not history:
         return history
@@ -130,6 +120,7 @@ def bot(history, choice):
         yield history
     if addresses:
         return history, addresses
+
 def add_message(history, message):
     history.append((message, None))
     return history, gr.Textbox(value="", interactive=True, placeholder="Enter message or upload file...", show_label=False)
@@ -137,27 +128,25 @@ def add_message(history, message):
 def print_like_dislike(x: gr.LikeData):
     print(x.index, x.value, x.liked)
 
-# Function to extract addresses from the chatbot's response
 def extract_addresses(response):
     if not isinstance(response, str):
         response = str(response)
-    address_pattern_1 = r'([A-Z].*,\sOmaha,\sNE\s\d{5})'
-    address_pattern_2 = r'(\d{4}\s.*,\sOmaha,\sNE\s\d{5})'
-    address_pattern_3 = r'([A-Z].*,\sNE\s\d{5})'
-    address_pattern_4 = r'([A-Z].*,.*\sSt,\sOmaha,\sNE\s\d{5})'
-    address_pattern_5 = r'([A-Z].*,.*\sStreets,\sOmaha,\sNE\s\d{5})'
-    address_pattern_6 = r'(\d{2}.*\sStreets)'
-    address_pattern_7 = r'([A-Z].*\s\d{2},\sOmaha,\sNE\s\d{5})'
-    addresses = re.findall(address_pattern_1, response) + re.findall(address_pattern_2, response) + \
-                re.findall(address_pattern_3, response) + re.findall(address_pattern_4, response) + \
-                re.findall(address_pattern_5, response) + re.findall(address_pattern_6, response) + \
-                re.findall(address_pattern_7, response)
+    address_patterns = [
+        r'([A-Z].*,\sOmaha,\sNE\s\d{5})',
+        r'(\d{4}\s.*,\sOmaha,\sNE\s\d{5})',
+        r'([A-Z].*,\sNE\s\d{5})',
+        r'([A-Z].*,.*\sSt,\sOmaha,\sNE\s\d{5})',
+        r'([A-Z].*,.*\sStreets,\sOmaha,\sNE\s\d{5})',
+        r'(\d{2}.*\sStreets)',
+        r'([A-Z].*\s\d{2},\sOmaha,\sNE\s\d{5})'
+    ]
+    addresses = []
+    for pattern in address_patterns:
+        addresses.extend(re.findall(pattern, response))
     return addresses
 
-# Store all found addresses
 all_addresses = []
 
-# Map generation function using Google Maps Geocoding API
 def generate_map(location_names):
     global all_addresses
     all_addresses.extend(location_names)
@@ -179,7 +168,6 @@ def generate_map(location_names):
     map_html = m._repr_html_()
     return map_html
 
-# Function to fetch local news
 def fetch_local_news():
     api_key = os.environ['SERP_API']
     url = f'https://serpapi.com/search.json?engine=google_news&q=ohama headline&api_key={api_key}'
@@ -254,23 +242,7 @@ def fetch_local_news():
         return news_html
     else:
         return "<p>Failed to fetch local news</p>"
-# Function to get the current weather
-def get_current_weather():
-    try:
-        api_key = os.environ['WEATHER_API']
-        url = f'https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline/omaha?unitGroup=metric&include=events%2Calerts%2Chours%2Cdays%2Ccurrent&key={api_key}'
-        response = requests.get(url)
-        response.raise_for_status()
-        jsonData = response.json()
-        current_conditions = jsonData.get("currentConditions", {})
-        temp = current_conditions.get("temp", "N/A")
-        condition = current_conditions.get("conditions", "N/A")
-        return f"{temp}°C and {condition}"
-    except requests.exceptions.RequestException as e:
-        logging.error(f"Failed to fetch local weather: {e}")
-        return "N/A"
-        
-# Function to fetch local events
+
 def fetch_local_events():
     api_key = os.environ['SERP_API']
     url = f'https://serpapi.com/search.json?engine=google_events&q=Events+in+Omaha&hl=en&gl=us&api_key={api_key}'
@@ -348,9 +320,6 @@ def fetch_local_events():
     else:
         return "<p>Failed to fetch local events</p>"
 
-
-
-# Function to fetch local weather
 def fetch_local_weather():
     try:
         api_key = os.environ['WEATHER_API']
@@ -431,48 +400,6 @@ def get_weather_icon(condition):
     }
     return condition_map.get(condition, "c01d")
 
-
-# Voice Control
-import numpy as np
-import torch
-from transformers import pipeline, AutoModelForSpeechSeq2Seq, AutoProcessor
-
-model_id = 'openai/whisper-large-v3'
-device = "cuda:0" if torch.cuda.is_available() else "cpu"
-torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
-model = AutoModelForSpeechSeq2Seq.from_pretrained(model_id, torch_dtype=torch_dtype,
-                                                  #low_cpu_mem_usage=True,
-                                                  use_safetensors=True).to(device)
-processor = AutoProcessor.from_pretrained(model_id)
-
-# Optimized ASR pipeline
-pipe_asr = pipeline("automatic-speech-recognition", model=model, tokenizer=processor.tokenizer, feature_extractor=processor.feature_extractor, max_new_tokens=128, chunk_length_s=15, batch_size=16, torch_dtype=torch_dtype, device=device, return_timestamps=True)
-
-base_audio_drive = "/data/audio"
-
-import numpy as np
-
-def transcribe_function(stream, new_chunk):
-    try:
-        sr, y = new_chunk[0], new_chunk[1]
-    except TypeError:
-        print(f"Error chunk structure: {type(new_chunk)}, content: {new_chunk}")
-        return stream, "", None
-
-    y = y.astype(np.float32) / np.max(np.abs(y))
-
-    if stream is not None:
-        stream = np.concatenate([stream, y])
-    else:
-        stream = y
-
-    result = pipe_asr({"array": stream, "sampling_rate": sr}, return_timestamps=False)
-
-    full_text = result.get("text", "")
-    
-    return stream, full_text, result
-
-# Map Retrieval Function for location finder
 def update_map_with_response(history):
     if not history:
         return ""
@@ -487,8 +414,7 @@ def show_map_if_details(history,choice):
     if choice in ["Details", "Conversational"]:
         return gr.update(visible=True), update_map_with_response(history)
     else:
-        return gr.update(visible=False), ""
-        #return gr.update(visible(False), "")
+        return gr.update(visible(False), "")
 
 def generate_audio_elevenlabs(text):
     XI_API_KEY = os.environ['ELEVENLABS_API']
@@ -520,6 +446,22 @@ def generate_audio_elevenlabs(text):
         logging.error(f"Error generating audio: {response.text}")
         return None
 
+# Stable Diffusion setup
+pipe = StableDiffusion3Pipeline.from_pretrained("stabilityai/stable-diffusion-3-medium-diffusers", torch_dtype=torch.float16)
+pipe = pipe.to("cuda")
+
+def generate_image(prompt):
+    image = pipe(
+        prompt,
+        negative_prompt="",
+        num_inference_steps=28,
+        guidance_scale=7.0,
+    ).images[0]
+    return image
+
+# Hardcoded prompt for image generation
+hardcoded_prompt = "A cat holding a sign that says hello world"
+
 # Gradio Blocks interface
 with gr.Blocks(theme='rawrsor1/Everforest') as demo:
     with gr.Row():
@@ -537,8 +479,6 @@ with gr.Blocks(theme='rawrsor1/Everforest') as demo:
             ''')
             chatbot = gr.Chatbot([], elem_id="chatbot", bubble_full_width=False)
     
-
-            
         with gr.Column():
             weather_output = gr.HTML(value=fetch_local_weather())
         
@@ -580,17 +520,18 @@ with gr.Blocks(theme='rawrsor1/Everforest') as demo:
                 gr.Markdown("<h1>Listen the audio</h1>", elem_id="audio-markdown")
 
                 audio_output = gr.Audio()
-                #bot_msg.then(generate_audio_elevenlabs, chatbot, audio_output)
                 bot_msg_audio = bot_msg.then(lambda history: generate_audio_elevenlabs(history[-1][1]), chatbot, audio_output)
             with gr.Column():
                 gr.Markdown("<h1>Local Events</h1>", elem_id="events-markdown")
 
                 news_output = gr.HTML(value=fetch_local_events())
 
-    setup_ui()    
-    
-    
+            with gr.Column():
+                gr.Markdown("<h1>Generated Image</h1>", elem_id="image-markdown")
+                
+                image_output = gr.Image(value=generate_image(hardcoded_prompt))
 
+    setup_ui()    
 
 demo.queue()
 demo.launch(share=True)