Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -9,18 +9,17 @@ import os
|
|
| 9 |
import glob
|
| 10 |
from pathlib import Path
|
| 11 |
from datetime import datetime
|
| 12 |
-
import edge_tts
|
| 13 |
-
import asyncio
|
| 14 |
-
import base64
|
| 15 |
import requests
|
| 16 |
from collections import defaultdict
|
| 17 |
-
from audio_recorder_streamlit import audio_recorder
|
| 18 |
-
import streamlit.components.v1 as components
|
| 19 |
import re
|
| 20 |
from urllib.parse import quote
|
| 21 |
from xml.etree import ElementTree as ET
|
|
|
|
|
|
|
| 22 |
|
| 23 |
-
#
|
|
|
|
|
|
|
| 24 |
if 'search_history' not in st.session_state:
|
| 25 |
st.session_state['search_history'] = []
|
| 26 |
if 'last_voice_input' not in st.session_state:
|
|
@@ -39,7 +38,14 @@ if 'arxiv_last_query' not in st.session_state:
|
|
| 39 |
st.session_state['arxiv_last_query'] = ""
|
| 40 |
if 'old_val' not in st.session_state:
|
| 41 |
st.session_state['old_val'] = None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
|
|
|
|
|
|
|
|
|
|
| 43 |
def highlight_text(text, query):
|
| 44 |
"""Highlight case-insensitive occurrences of query in text with bold formatting."""
|
| 45 |
if not query:
|
|
@@ -47,100 +53,124 @@ def highlight_text(text, query):
|
|
| 47 |
pattern = re.compile(re.escape(query), re.IGNORECASE)
|
| 48 |
return pattern.sub(lambda m: f"**{m.group(0)}**", text)
|
| 49 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
class VideoSearch:
|
| 51 |
def __init__(self):
|
| 52 |
self.text_model = SentenceTransformer('all-MiniLM-L6-v2')
|
| 53 |
-
self.load_dataset()
|
| 54 |
-
|
| 55 |
-
def fetch_dataset_rows(self):
|
| 56 |
-
"""Fetch dataset from Hugging Face API"""
|
| 57 |
-
try:
|
| 58 |
-
url = "https://datasets-server.huggingface.co/first-rows?dataset=omegalabsinc%2Fomega-multimodal&config=default&split=train"
|
| 59 |
-
response = requests.get(url, timeout=30)
|
| 60 |
-
if response.status_code == 200:
|
| 61 |
-
data = response.json()
|
| 62 |
-
if 'rows' in data:
|
| 63 |
-
processed_rows = []
|
| 64 |
-
for row_data in data['rows']:
|
| 65 |
-
row = row_data.get('row', row_data)
|
| 66 |
-
for key in row:
|
| 67 |
-
if any(term in key.lower() for term in ['embed', 'vector', 'encoding']):
|
| 68 |
-
if isinstance(row[key], str):
|
| 69 |
-
try:
|
| 70 |
-
row[key] = [float(x.strip()) for x in row[key].strip('[]').split(',') if x.strip()]
|
| 71 |
-
except:
|
| 72 |
-
continue
|
| 73 |
-
processed_rows.append(row)
|
| 74 |
-
|
| 75 |
-
df = pd.DataFrame(processed_rows)
|
| 76 |
-
st.session_state['search_columns'] = [col for col in df.columns
|
| 77 |
-
if col not in ['video_embed', 'description_embed', 'audio_embed']]
|
| 78 |
-
return df
|
| 79 |
-
return self.load_example_data()
|
| 80 |
-
except:
|
| 81 |
-
return self.load_example_data()
|
| 82 |
-
|
| 83 |
-
def prepare_features(self):
|
| 84 |
-
"""Prepare embeddings with adaptive field detection"""
|
| 85 |
-
try:
|
| 86 |
-
embed_cols = [col for col in self.dataset.columns
|
| 87 |
-
if any(term in col.lower() for term in ['embed', 'vector', 'encoding'])]
|
| 88 |
-
|
| 89 |
-
embeddings = {}
|
| 90 |
-
for col in embed_cols:
|
| 91 |
-
try:
|
| 92 |
-
data = []
|
| 93 |
-
for row in self.dataset[col]:
|
| 94 |
-
if isinstance(row, str):
|
| 95 |
-
values = [float(x.strip()) for x in row.strip('[]').split(',') if x.strip()]
|
| 96 |
-
elif isinstance(row, list):
|
| 97 |
-
values = row
|
| 98 |
-
else:
|
| 99 |
-
continue
|
| 100 |
-
data.append(values)
|
| 101 |
-
|
| 102 |
-
if data:
|
| 103 |
-
embeddings[col] = np.array(data)
|
| 104 |
-
except:
|
| 105 |
-
continue
|
| 106 |
-
|
| 107 |
-
if 'video_embed' in embeddings:
|
| 108 |
-
self.video_embeds = embeddings['video_embed']
|
| 109 |
-
else:
|
| 110 |
-
self.video_embeds = next(iter(embeddings.values()))
|
| 111 |
-
|
| 112 |
-
if 'description_embed' in embeddings:
|
| 113 |
-
self.text_embeds = embeddings['description_embed']
|
| 114 |
-
else:
|
| 115 |
-
self.text_embeds = self.video_embeds
|
| 116 |
-
|
| 117 |
-
except:
|
| 118 |
-
# Fallback to random embeddings
|
| 119 |
-
num_rows = len(self.dataset)
|
| 120 |
-
self.video_embeds = np.random.randn(num_rows, 384)
|
| 121 |
-
self.text_embeds = np.random.randn(num_rows, 384)
|
| 122 |
-
|
| 123 |
-
def load_example_data(self):
|
| 124 |
-
"""Load example data as fallback"""
|
| 125 |
-
example_data = [
|
| 126 |
-
{
|
| 127 |
-
"video_id": "cd21da96-fcca-4c94-a60f-0b1e4e1e29fc",
|
| 128 |
-
"youtube_id": "IO-vwtyicn4",
|
| 129 |
-
"description": "This video shows a close-up of an ancient text carved into a surface.",
|
| 130 |
-
"views": 45489,
|
| 131 |
-
"start_time": 1452,
|
| 132 |
-
"end_time": 1458,
|
| 133 |
-
"video_embed": [0.014160037972033024, -0.003111184574663639, -0.016604168340563774],
|
| 134 |
-
"description_embed": [-0.05835828185081482, 0.02589797042310238, 0.11952091753482819]
|
| 135 |
-
}
|
| 136 |
-
]
|
| 137 |
-
return pd.DataFrame(example_data)
|
| 138 |
-
|
| 139 |
-
def load_dataset(self):
|
| 140 |
-
self.dataset = self.fetch_dataset_rows()
|
| 141 |
-
self.prepare_features()
|
| 142 |
|
| 143 |
def search(self, query, column=None, top_k=20):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
# Semantic search
|
| 145 |
query_embedding = self.text_model.encode([query])[0]
|
| 146 |
video_sims = cosine_similarity([query_embedding], self.video_embeds)[0]
|
|
@@ -150,7 +180,6 @@ class VideoSearch:
|
|
| 150 |
# If a column is selected (not All Fields), strictly filter by textual match
|
| 151 |
if column and column in self.dataset.columns and column != "All Fields":
|
| 152 |
mask = self.dataset[column].astype(str).str.contains(query, case=False, na=False)
|
| 153 |
-
# Only keep rows that contain the query text in the selected column
|
| 154 |
combined_sims = combined_sims[mask]
|
| 155 |
filtered_dataset = self.dataset[mask].copy()
|
| 156 |
else:
|
|
@@ -161,7 +190,7 @@ class VideoSearch:
|
|
| 161 |
if top_k == 0:
|
| 162 |
return []
|
| 163 |
top_indices = np.argsort(combined_sims)[-top_k:][::-1]
|
| 164 |
-
|
| 165 |
results = []
|
| 166 |
filtered_dataset = filtered_dataset.iloc[top_indices]
|
| 167 |
filtered_sims = combined_sims[top_indices]
|
|
@@ -171,62 +200,16 @@ class VideoSearch:
|
|
| 171 |
if col not in ['video_embed', 'description_embed', 'audio_embed']:
|
| 172 |
result[col] = getattr(row, col)
|
| 173 |
results.append(result)
|
| 174 |
-
|
| 175 |
-
return results
|
| 176 |
-
|
| 177 |
-
@st.cache_resource
|
| 178 |
-
def get_speech_model():
|
| 179 |
-
return edge_tts.Communicate
|
| 180 |
-
|
| 181 |
-
async def generate_speech(text, voice=None):
|
| 182 |
-
if not text.strip():
|
| 183 |
-
return None
|
| 184 |
-
if not voice:
|
| 185 |
-
voice = st.session_state['tts_voice']
|
| 186 |
-
try:
|
| 187 |
-
communicate = get_speech_model()(text, voice)
|
| 188 |
-
audio_file = f"speech_{datetime.now().strftime('%Y%m%d_%H%M%S')}.mp3"
|
| 189 |
-
await communicate.save(audio_file)
|
| 190 |
-
return audio_file
|
| 191 |
-
except Exception as e:
|
| 192 |
-
st.error(f"Error generating speech: {e}")
|
| 193 |
-
return None
|
| 194 |
|
| 195 |
-
|
| 196 |
-
"""Display file manager interface"""
|
| 197 |
-
st.subheader("π File Manager")
|
| 198 |
-
col1, col2 = st.columns(2)
|
| 199 |
-
with col1:
|
| 200 |
-
uploaded_file = st.file_uploader("Upload File", type=['txt', 'md', 'mp3'])
|
| 201 |
-
if uploaded_file:
|
| 202 |
-
with open(uploaded_file.name, "wb") as f:
|
| 203 |
-
f.write(uploaded_file.getvalue())
|
| 204 |
-
st.success(f"Uploaded: {uploaded_file.name}")
|
| 205 |
-
st.rerun()
|
| 206 |
-
|
| 207 |
-
with col2:
|
| 208 |
-
if st.button("π Clear All Files"):
|
| 209 |
-
for f in glob.glob("*.txt") + glob.glob("*.md") + glob.glob("*.mp3"):
|
| 210 |
-
os.remove(f)
|
| 211 |
-
st.success("All files cleared!")
|
| 212 |
-
st.rerun()
|
| 213 |
-
|
| 214 |
-
files = glob.glob("*.txt") + glob.glob("*.md") + glob.glob("*.mp3")
|
| 215 |
-
if files:
|
| 216 |
-
st.write("### Existing Files")
|
| 217 |
-
for f in files:
|
| 218 |
-
with st.expander(f"π {os.path.basename(f)}"):
|
| 219 |
-
if f.endswith('.mp3'):
|
| 220 |
-
st.audio(f)
|
| 221 |
-
else:
|
| 222 |
-
with open(f, 'r', encoding='utf-8') as file:
|
| 223 |
-
st.text_area("Content", file.read(), height=100)
|
| 224 |
-
if st.button(f"Delete {os.path.basename(f)}", key=f"del_{f}"):
|
| 225 |
-
os.remove(f)
|
| 226 |
-
st.rerun()
|
| 227 |
|
|
|
|
|
|
|
|
|
|
| 228 |
def arxiv_search(query, max_results=5):
|
| 229 |
"""Perform a simple Arxiv search using their API and return top results."""
|
|
|
|
|
|
|
| 230 |
base_url = "http://export.arxiv.org/api/query?"
|
| 231 |
search_url = base_url + f"search_query={quote(query)}&start=0&max_results={max_results}"
|
| 232 |
r = requests.get(search_url)
|
|
@@ -259,156 +242,183 @@ def perform_arxiv_lookup(q, vocal_summary=True, titles_summary=True, full_audio=
|
|
| 259 |
if link:
|
| 260 |
st.markdown(f"[View Paper]({link})")
|
| 261 |
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 267 |
else:
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
audio_file = asyncio.run(generate_speech(spoken_text))
|
| 271 |
-
if audio_file:
|
| 272 |
-
st.audio(audio_file)
|
| 273 |
-
|
| 274 |
-
if full_audio:
|
| 275 |
-
full_text = ""
|
| 276 |
-
for i,(title, summary, _) in enumerate(results, start=1):
|
| 277 |
-
full_text += f"Result {i}: {title}. {summary} "
|
| 278 |
-
audio_file_full = asyncio.run(generate_speech(full_text))
|
| 279 |
-
if audio_file_full:
|
| 280 |
-
st.write("### Full Audio")
|
| 281 |
-
st.audio(audio_file_full)
|
| 282 |
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
with tab1:
|
| 292 |
-
st.subheader("Search Videos")
|
| 293 |
-
col1, col2 = st.columns([3, 1])
|
| 294 |
-
with col1:
|
| 295 |
-
query = st.text_input("Enter your search query:",
|
| 296 |
-
value="ancient" if not st.session_state['initial_search_done'] else "")
|
| 297 |
-
with col2:
|
| 298 |
-
search_column = st.selectbox("Search in field:",
|
| 299 |
-
["All Fields"] + st.session_state['search_columns'])
|
| 300 |
-
|
| 301 |
-
col3, col4 = st.columns(2)
|
| 302 |
-
with col3:
|
| 303 |
-
num_results = st.slider("Number of results:", 1, 100, 20)
|
| 304 |
-
with col4:
|
| 305 |
-
search_button = st.button("π Search")
|
| 306 |
-
|
| 307 |
-
if (search_button or not st.session_state['initial_search_done']) and query:
|
| 308 |
-
st.session_state['initial_search_done'] = True
|
| 309 |
-
selected_column = None if search_column == "All Fields" else search_column
|
| 310 |
-
with st.spinner("Searching..."):
|
| 311 |
-
results = search.search(query, selected_column, num_results)
|
| 312 |
-
|
| 313 |
-
st.session_state['search_history'].append({
|
| 314 |
-
'query': query,
|
| 315 |
-
'timestamp': datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
| 316 |
-
'results': results[:5]
|
| 317 |
-
})
|
| 318 |
-
|
| 319 |
-
for i, result in enumerate(results, 1):
|
| 320 |
-
# Highlight the query in the description
|
| 321 |
-
highlighted_desc = highlight_text(result['description'], query)
|
| 322 |
-
with st.expander(f"Result {i}: {result['description'][:100]}...", expanded=(i==1)):
|
| 323 |
-
cols = st.columns([2, 1])
|
| 324 |
-
with cols[0]:
|
| 325 |
-
st.markdown("**Description:**")
|
| 326 |
-
st.write(highlighted_desc)
|
| 327 |
-
st.markdown(f"**Time Range:** {result['start_time']}s - {result['end_time']}s")
|
| 328 |
-
st.markdown(f"**Views:** {result['views']:,}")
|
| 329 |
-
|
| 330 |
-
with cols[1]:
|
| 331 |
-
st.markdown(f"**Relevance Score:** {result['relevance_score']:.2%}")
|
| 332 |
-
if result.get('youtube_id'):
|
| 333 |
-
st.video(f"https://youtube.com/watch?v={result['youtube_id']}&t={result['start_time']}")
|
| 334 |
-
|
| 335 |
-
if st.button(f"π Audio Summary {i}", key=f"audio_{i}"):
|
| 336 |
-
summary = f"Video summary: {result['description'][:200]}"
|
| 337 |
-
audio_file = asyncio.run(generate_speech(summary))
|
| 338 |
-
if audio_file:
|
| 339 |
-
st.audio(audio_file)
|
| 340 |
-
|
| 341 |
-
# ---- Tab 2: Voice Input ----
|
| 342 |
-
# Reintroduce the mycomponent from earlier code for voice input accumulation
|
| 343 |
-
with tab2:
|
| 344 |
-
st.subheader("Voice Input (HTML Component)")
|
| 345 |
-
|
| 346 |
-
# Declare the custom component
|
| 347 |
-
mycomponent = components.declare_component("mycomponent", path="mycomponent")
|
| 348 |
-
|
| 349 |
-
# Use the component to get accumulated voice input
|
| 350 |
-
val = mycomponent(my_input_value="Hello")
|
| 351 |
-
|
| 352 |
-
if val:
|
| 353 |
-
val_stripped = val.replace('\n', ' ')
|
| 354 |
-
edited_input = st.text_area("βοΈ Edit Input:", value=val_stripped, height=100)
|
| 355 |
-
|
| 356 |
-
# Just allow searching the videos from the edited input
|
| 357 |
-
if st.button("π Search from Edited Voice Input"):
|
| 358 |
-
results = search.search(edited_input, None, 20)
|
| 359 |
-
for i, result in enumerate(results, 1):
|
| 360 |
-
# Highlight query in description
|
| 361 |
-
highlighted_desc = highlight_text(result['description'], edited_input)
|
| 362 |
-
with st.expander(f"Result {i}", expanded=(i==1)):
|
| 363 |
-
st.write(highlighted_desc)
|
| 364 |
-
if result.get('youtube_id'):
|
| 365 |
-
st.video(f"https://youtube.com/watch?v={result['youtube_id']}&t={result.get('start_time', 0)}")
|
| 366 |
-
|
| 367 |
-
# Optionally also let user record audio via audio_recorder (not integrated with transcription)
|
| 368 |
-
st.write("Or record audio (not ASR integrated):")
|
| 369 |
-
audio_bytes = audio_recorder()
|
| 370 |
-
if audio_bytes:
|
| 371 |
-
audio_path = f"temp_audio_{datetime.now().strftime('%Y%m%d_%H%M%S')}.wav"
|
| 372 |
-
with open(audio_path, "wb") as f:
|
| 373 |
-
f.write(audio_bytes)
|
| 374 |
-
st.success("Audio recorded successfully!")
|
| 375 |
-
# No transcription is provided since no external ASR is included here.
|
| 376 |
-
if os.path.exists(audio_path):
|
| 377 |
-
os.remove(audio_path)
|
| 378 |
-
|
| 379 |
-
# ---- Tab 3: Arxiv Search ----
|
| 380 |
-
with tab3:
|
| 381 |
-
st.subheader("Arxiv Search")
|
| 382 |
-
q = st.text_input("Enter your Arxiv search query:", value=st.session_state['arxiv_last_query'])
|
| 383 |
-
vocal_summary = st.checkbox("π Short Audio Summary", value=True)
|
| 384 |
-
titles_summary = st.checkbox("π Titles Only", value=True)
|
| 385 |
-
full_audio = st.checkbox("π Full Audio Results", value=False)
|
| 386 |
-
|
| 387 |
-
if st.button("π Arxiv Search"):
|
| 388 |
-
st.session_state['arxiv_last_query'] = q
|
| 389 |
-
perform_arxiv_lookup(q, vocal_summary=vocal_summary, titles_summary=titles_summary, full_audio=full_audio)
|
| 390 |
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 394 |
|
| 395 |
-
#
|
| 396 |
with st.sidebar:
|
| 397 |
st.subheader("βοΈ Settings & History")
|
| 398 |
if st.button("ποΈ Clear History"):
|
| 399 |
st.session_state['search_history'] = []
|
| 400 |
-
st.
|
| 401 |
-
|
| 402 |
st.markdown("### Recent Searches")
|
| 403 |
for entry in reversed(st.session_state['search_history'][-5:]):
|
| 404 |
with st.expander(f"{entry['timestamp']}: {entry['query']}"):
|
| 405 |
for i, result in enumerate(entry['results'], 1):
|
| 406 |
st.write(f"{i}. {result['description'][:100]}...")
|
| 407 |
|
| 408 |
-
st.markdown("### Voice
|
| 409 |
st.selectbox("TTS Voice:",
|
| 410 |
["en-US-AriaNeural", "en-US-GuyNeural", "en-GB-SoniaNeural"],
|
| 411 |
key="tts_voice")
|
| 412 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 413 |
if __name__ == "__main__":
|
| 414 |
main()
|
|
|
|
| 9 |
import glob
|
| 10 |
from pathlib import Path
|
| 11 |
from datetime import datetime
|
|
|
|
|
|
|
|
|
|
| 12 |
import requests
|
| 13 |
from collections import defaultdict
|
|
|
|
|
|
|
| 14 |
import re
|
| 15 |
from urllib.parse import quote
|
| 16 |
from xml.etree import ElementTree as ET
|
| 17 |
+
import base64
|
| 18 |
+
from PIL import Image
|
| 19 |
|
| 20 |
+
# -----------------------------------------
|
| 21 |
+
# Session State Initialization
|
| 22 |
+
# -----------------------------------------
|
| 23 |
if 'search_history' not in st.session_state:
|
| 24 |
st.session_state['search_history'] = []
|
| 25 |
if 'last_voice_input' not in st.session_state:
|
|
|
|
| 38 |
st.session_state['arxiv_last_query'] = ""
|
| 39 |
if 'old_val' not in st.session_state:
|
| 40 |
st.session_state['old_val'] = None
|
| 41 |
+
if 'current_file' not in st.session_state:
|
| 42 |
+
st.session_state['current_file'] = None
|
| 43 |
+
if 'file_content' not in st.session_state:
|
| 44 |
+
st.session_state['file_content'] = ""
|
| 45 |
|
| 46 |
+
# -----------------------------------------
|
| 47 |
+
# Utility Functions
|
| 48 |
+
# -----------------------------------------
|
| 49 |
def highlight_text(text, query):
|
| 50 |
"""Highlight case-insensitive occurrences of query in text with bold formatting."""
|
| 51 |
if not query:
|
|
|
|
| 53 |
pattern = re.compile(re.escape(query), re.IGNORECASE)
|
| 54 |
return pattern.sub(lambda m: f"**{m.group(0)}**", text)
|
| 55 |
|
| 56 |
+
@st.cache_data(show_spinner=False)
|
| 57 |
+
def fetch_dataset_rows():
|
| 58 |
+
"""Fetch dataset from Hugging Face API and cache it."""
|
| 59 |
+
try:
|
| 60 |
+
url = "https://datasets-server.huggingface.co/first-rows?dataset=omegalabsinc%2Fomega-multimodal&config=default&split=train"
|
| 61 |
+
response = requests.get(url, timeout=30)
|
| 62 |
+
if response.status_code == 200:
|
| 63 |
+
data = response.json()
|
| 64 |
+
if 'rows' in data:
|
| 65 |
+
processed_rows = []
|
| 66 |
+
for row_data in data['rows']:
|
| 67 |
+
row = row_data.get('row', row_data)
|
| 68 |
+
# Convert embed fields from strings to arrays
|
| 69 |
+
for key in row:
|
| 70 |
+
if any(term in key.lower() for term in ['embed', 'vector', 'encoding']):
|
| 71 |
+
if isinstance(row[key], str):
|
| 72 |
+
try:
|
| 73 |
+
row[key] = [float(x.strip()) for x in row[key].strip('[]').split(',') if x.strip()]
|
| 74 |
+
except:
|
| 75 |
+
continue
|
| 76 |
+
processed_rows.append(row)
|
| 77 |
+
|
| 78 |
+
df = pd.DataFrame(processed_rows)
|
| 79 |
+
st.session_state['search_columns'] = [col for col in df.columns
|
| 80 |
+
if col not in ['video_embed', 'description_embed', 'audio_embed']]
|
| 81 |
+
return df
|
| 82 |
+
except:
|
| 83 |
+
pass
|
| 84 |
+
return load_example_data()
|
| 85 |
+
|
| 86 |
+
def load_example_data():
|
| 87 |
+
"""Load example data as fallback."""
|
| 88 |
+
example_data = [
|
| 89 |
+
{
|
| 90 |
+
"video_id": "cd21da96-fcca-4c94-a60f-0b1e4e1e29fc",
|
| 91 |
+
"youtube_id": "IO-vwtyicn4",
|
| 92 |
+
"description": "This video shows a close-up of an ancient text carved into a surface.",
|
| 93 |
+
"views": 45489,
|
| 94 |
+
"start_time": 1452,
|
| 95 |
+
"end_time": 1458,
|
| 96 |
+
"video_embed": [0.014160037972033024, -0.003111184574663639, -0.016604168340563774],
|
| 97 |
+
"description_embed": [-0.05835828185081482, 0.02589797042310238, 0.11952091753482819]
|
| 98 |
+
}
|
| 99 |
+
]
|
| 100 |
+
return pd.DataFrame(example_data)
|
| 101 |
+
|
| 102 |
+
@st.cache_data(show_spinner=False)
|
| 103 |
+
def load_dataset():
|
| 104 |
+
df = fetch_dataset_rows()
|
| 105 |
+
return df
|
| 106 |
+
|
| 107 |
+
def prepare_features(dataset):
|
| 108 |
+
"""Prepare embeddings with adaptive field detection."""
|
| 109 |
+
try:
|
| 110 |
+
embed_cols = [col for col in dataset.columns
|
| 111 |
+
if any(term in col.lower() for term in ['embed', 'vector', 'encoding'])]
|
| 112 |
+
|
| 113 |
+
embeddings = {}
|
| 114 |
+
for col in embed_cols:
|
| 115 |
+
try:
|
| 116 |
+
data = []
|
| 117 |
+
for row in dataset[col]:
|
| 118 |
+
if isinstance(row, str):
|
| 119 |
+
values = [float(x.strip()) for x in row.strip('[]').split(',') if x.strip()]
|
| 120 |
+
elif isinstance(row, list):
|
| 121 |
+
values = row
|
| 122 |
+
else:
|
| 123 |
+
continue
|
| 124 |
+
data.append(values)
|
| 125 |
+
|
| 126 |
+
if data:
|
| 127 |
+
embeddings[col] = np.array(data)
|
| 128 |
+
except:
|
| 129 |
+
continue
|
| 130 |
+
|
| 131 |
+
# Assign default embeddings
|
| 132 |
+
video_embeds = embeddings.get('video_embed', None)
|
| 133 |
+
text_embeds = embeddings.get('description_embed', None)
|
| 134 |
+
|
| 135 |
+
# If missing either, fall back to what is available
|
| 136 |
+
if video_embeds is None and embeddings:
|
| 137 |
+
video_embeds = next(iter(embeddings.values()))
|
| 138 |
+
if text_embeds is None:
|
| 139 |
+
text_embeds = video_embeds if video_embeds is not None else np.random.randn(len(dataset), 384)
|
| 140 |
+
|
| 141 |
+
if video_embeds is None:
|
| 142 |
+
# Fallback to random embeddings if none found
|
| 143 |
+
num_rows = len(dataset)
|
| 144 |
+
video_embeds = np.random.randn(num_rows, 384)
|
| 145 |
+
text_embeds = np.random.randn(num_rows, 384)
|
| 146 |
+
|
| 147 |
+
return video_embeds, text_embeds
|
| 148 |
+
except:
|
| 149 |
+
# Fallback to random embeddings
|
| 150 |
+
num_rows = len(dataset)
|
| 151 |
+
return np.random.randn(num_rows, 384), np.random.randn(num_rows, 384)
|
| 152 |
+
|
| 153 |
class VideoSearch:
|
| 154 |
def __init__(self):
|
| 155 |
self.text_model = SentenceTransformer('all-MiniLM-L6-v2')
|
| 156 |
+
self.dataset = load_dataset()
|
| 157 |
+
self.video_embeds, self.text_embeds = prepare_features(self.dataset)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
|
| 159 |
def search(self, query, column=None, top_k=20):
|
| 160 |
+
# If no query, return all records
|
| 161 |
+
if not query.strip():
|
| 162 |
+
# Just return all rows as results
|
| 163 |
+
results = []
|
| 164 |
+
df_copy = self.dataset.copy()
|
| 165 |
+
# Add a neutral relevance score (e.g. 1.0)
|
| 166 |
+
for row in df_copy.itertuples():
|
| 167 |
+
result = {'relevance_score': 1.0}
|
| 168 |
+
for col in df_copy.columns:
|
| 169 |
+
if col not in ['video_embed', 'description_embed', 'audio_embed']:
|
| 170 |
+
result[col] = getattr(row, col)
|
| 171 |
+
results.append(result)
|
| 172 |
+
return results[:top_k]
|
| 173 |
+
|
| 174 |
# Semantic search
|
| 175 |
query_embedding = self.text_model.encode([query])[0]
|
| 176 |
video_sims = cosine_similarity([query_embedding], self.video_embeds)[0]
|
|
|
|
| 180 |
# If a column is selected (not All Fields), strictly filter by textual match
|
| 181 |
if column and column in self.dataset.columns and column != "All Fields":
|
| 182 |
mask = self.dataset[column].astype(str).str.contains(query, case=False, na=False)
|
|
|
|
| 183 |
combined_sims = combined_sims[mask]
|
| 184 |
filtered_dataset = self.dataset[mask].copy()
|
| 185 |
else:
|
|
|
|
| 190 |
if top_k == 0:
|
| 191 |
return []
|
| 192 |
top_indices = np.argsort(combined_sims)[-top_k:][::-1]
|
| 193 |
+
|
| 194 |
results = []
|
| 195 |
filtered_dataset = filtered_dataset.iloc[top_indices]
|
| 196 |
filtered_sims = combined_sims[top_indices]
|
|
|
|
| 200 |
if col not in ['video_embed', 'description_embed', 'audio_embed']:
|
| 201 |
result[col] = getattr(row, col)
|
| 202 |
results.append(result)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 203 |
|
| 204 |
+
return results
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 205 |
|
| 206 |
+
# -----------------------------------------
|
| 207 |
+
# Arxiv Search Functions
|
| 208 |
+
# -----------------------------------------
|
| 209 |
def arxiv_search(query, max_results=5):
|
| 210 |
"""Perform a simple Arxiv search using their API and return top results."""
|
| 211 |
+
if not query.strip():
|
| 212 |
+
return []
|
| 213 |
base_url = "http://export.arxiv.org/api/query?"
|
| 214 |
search_url = base_url + f"search_query={quote(query)}&start=0&max_results={max_results}"
|
| 215 |
r = requests.get(search_url)
|
|
|
|
| 242 |
if link:
|
| 243 |
st.markdown(f"[View Paper]({link})")
|
| 244 |
|
| 245 |
+
# -----------------------------------------
|
| 246 |
+
# File Manager
|
| 247 |
+
# -----------------------------------------
|
| 248 |
+
def show_file_manager():
|
| 249 |
+
"""Display file manager interface for uploading and browsing local files."""
|
| 250 |
+
st.subheader("π File Manager")
|
| 251 |
+
col1, col2 = st.columns(2)
|
| 252 |
+
with col1:
|
| 253 |
+
uploaded_file = st.file_uploader("Upload File", type=['txt', 'md', 'mp3'])
|
| 254 |
+
if uploaded_file:
|
| 255 |
+
with open(uploaded_file.name, "wb") as f:
|
| 256 |
+
f.write(uploaded_file.getvalue())
|
| 257 |
+
st.success(f"Uploaded: {uploaded_file.name}")
|
| 258 |
+
st.session_state.should_rerun = True
|
| 259 |
+
|
| 260 |
+
with col2:
|
| 261 |
+
if st.button("π Clear All Files"):
|
| 262 |
+
for f in glob.glob("*.txt") + glob.glob("*.md") + glob.glob("*.mp3"):
|
| 263 |
+
os.remove(f)
|
| 264 |
+
st.success("All files cleared!")
|
| 265 |
+
st.session_state.should_rerun = True
|
| 266 |
+
|
| 267 |
+
files = glob.glob("*.txt") + glob.glob("*.md") + glob.glob("*.mp3")
|
| 268 |
+
if files:
|
| 269 |
+
st.write("### Existing Files")
|
| 270 |
+
for f in files:
|
| 271 |
+
with st.expander(f"π {os.path.basename(f)}"):
|
| 272 |
+
if f.endswith('.mp3'):
|
| 273 |
+
st.audio(f)
|
| 274 |
+
else:
|
| 275 |
+
with open(f, 'r', encoding='utf-8') as file:
|
| 276 |
+
st.text_area("Content", file.read(), height=100)
|
| 277 |
+
if st.button(f"Delete {os.path.basename(f)}", key=f"del_{f}"):
|
| 278 |
+
os.remove(f)
|
| 279 |
+
st.session_state.should_rerun = True
|
| 280 |
+
|
| 281 |
+
# -----------------------------------------
|
| 282 |
+
# Editor: Allow user to select a text file and edit it
|
| 283 |
+
# -----------------------------------------
|
| 284 |
+
def display_editor():
|
| 285 |
+
# Let user pick a file from local directory to edit
|
| 286 |
+
text_files = glob.glob("*.txt") + glob.glob("*.md")
|
| 287 |
+
selected_file = st.selectbox("Select a file to edit:", ["None"] + text_files)
|
| 288 |
+
if selected_file != "None":
|
| 289 |
+
with open(selected_file, 'r', encoding='utf-8') as f:
|
| 290 |
+
content = f.read()
|
| 291 |
+
new_content = st.text_area("βοΈ Edit Content:", value=content, height=300)
|
| 292 |
+
if st.button("πΎ Save"):
|
| 293 |
+
with open(selected_file, 'w', encoding='utf-8') as f:
|
| 294 |
+
f.write(new_content)
|
| 295 |
+
st.success("File saved!")
|
| 296 |
+
st.session_state.should_rerun = True
|
| 297 |
+
|
| 298 |
+
# -----------------------------------------
|
| 299 |
+
# Media (Images & Videos)
|
| 300 |
+
# -----------------------------------------
|
| 301 |
+
def show_media():
|
| 302 |
+
st.header("πΈ Images & π₯ Videos")
|
| 303 |
+
tabs = st.tabs(["πΌ Images", "π₯ Video"])
|
| 304 |
+
with tabs[0]:
|
| 305 |
+
imgs = glob.glob("*.png") + glob.glob("*.jpg") + glob.glob("*.jpeg")
|
| 306 |
+
if imgs:
|
| 307 |
+
c = st.slider("Columns", 1, 5, 3)
|
| 308 |
+
cols = st.columns(c)
|
| 309 |
+
for i, f in enumerate(imgs):
|
| 310 |
+
with cols[i % c]:
|
| 311 |
+
st.image(Image.open(f), use_column_width=True)
|
| 312 |
else:
|
| 313 |
+
st.write("No images found.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 314 |
|
| 315 |
+
with tabs[1]:
|
| 316 |
+
vids = glob.glob("*.mp4") + glob.glob("*.webm") + glob.glob("*.mov")
|
| 317 |
+
if vids:
|
| 318 |
+
for v in vids:
|
| 319 |
+
with st.expander(f"π₯ {os.path.basename(v)}"):
|
| 320 |
+
st.video(v)
|
| 321 |
+
else:
|
| 322 |
+
st.write("No videos found.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 323 |
|
| 324 |
+
# -----------------------------------------
|
| 325 |
+
# Video Search
|
| 326 |
+
# -----------------------------------------
|
| 327 |
+
def display_video_search():
|
| 328 |
+
st.subheader("Search Videos")
|
| 329 |
+
search_instance = VideoSearch()
|
| 330 |
+
col1, col2 = st.columns([3, 1])
|
| 331 |
+
with col1:
|
| 332 |
+
query = st.text_input("Enter your search query:", value="ancient" if not st.session_state['initial_search_done'] else "")
|
| 333 |
+
with col2:
|
| 334 |
+
search_column = st.selectbox("Search in field:", ["All Fields"] + st.session_state['search_columns'])
|
| 335 |
+
|
| 336 |
+
col3, col4 = st.columns(2)
|
| 337 |
+
with col3:
|
| 338 |
+
num_results = st.slider("Number of results:", 1, 100, 20)
|
| 339 |
+
with col4:
|
| 340 |
+
search_button = st.button("π Search")
|
| 341 |
+
|
| 342 |
+
if (search_button or not st.session_state['initial_search_done']) and query is not None:
|
| 343 |
+
st.session_state['initial_search_done'] = True
|
| 344 |
+
selected_column = None if search_column == "All Fields" else search_column
|
| 345 |
+
with st.spinner("Searching..."):
|
| 346 |
+
results = search_instance.search(query, selected_column, num_results)
|
| 347 |
+
|
| 348 |
+
st.session_state['search_history'].append({
|
| 349 |
+
'query': query,
|
| 350 |
+
'timestamp': datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
| 351 |
+
'results': results[:5]
|
| 352 |
+
})
|
| 353 |
+
|
| 354 |
+
for i, result in enumerate(results, 1):
|
| 355 |
+
highlighted_desc = highlight_text(result['description'], query)
|
| 356 |
+
with st.expander(f"Result {i}: {result['description'][:100]}...", expanded=(i == 1)):
|
| 357 |
+
cols = st.columns([2, 1])
|
| 358 |
+
with cols[0]:
|
| 359 |
+
st.markdown("**Description:**")
|
| 360 |
+
st.write(highlighted_desc)
|
| 361 |
+
st.markdown(f"**Time Range:** {result['start_time']}s - {result['end_time']}s")
|
| 362 |
+
st.markdown(f"**Views:** {result['views']:,}")
|
| 363 |
+
|
| 364 |
+
with cols[1]:
|
| 365 |
+
st.markdown(f"**Relevance Score:** {result['relevance_score']:.2%}")
|
| 366 |
+
if result.get('youtube_id'):
|
| 367 |
+
st.video(f"https://youtube.com/watch?v={result['youtube_id']}&t={result['start_time']}")
|
| 368 |
+
|
| 369 |
+
# -----------------------------------------
|
| 370 |
+
# Main Application (Integrated)
|
| 371 |
+
# -----------------------------------------
|
| 372 |
+
def main():
|
| 373 |
+
st.sidebar.markdown("### π²BikeAIπ Multi-Agent Research")
|
| 374 |
+
# We remove the "π€ Voice" option since voice input is removed
|
| 375 |
+
tab_main = st.sidebar.radio("Action:", ["πΈ Media", "π ArXiv", "π Editor"])
|
| 376 |
|
| 377 |
+
# File manager in the sidebar
|
| 378 |
with st.sidebar:
|
| 379 |
st.subheader("βοΈ Settings & History")
|
| 380 |
if st.button("ποΈ Clear History"):
|
| 381 |
st.session_state['search_history'] = []
|
| 382 |
+
st.experimental_rerun()
|
| 383 |
+
|
| 384 |
st.markdown("### Recent Searches")
|
| 385 |
for entry in reversed(st.session_state['search_history'][-5:]):
|
| 386 |
with st.expander(f"{entry['timestamp']}: {entry['query']}"):
|
| 387 |
for i, result in enumerate(entry['results'], 1):
|
| 388 |
st.write(f"{i}. {result['description'][:100]}...")
|
| 389 |
|
| 390 |
+
st.markdown("### TTS Voice (unused)")
|
| 391 |
st.selectbox("TTS Voice:",
|
| 392 |
["en-US-AriaNeural", "en-US-GuyNeural", "en-GB-SoniaNeural"],
|
| 393 |
key="tts_voice")
|
| 394 |
|
| 395 |
+
# Main content based on selection
|
| 396 |
+
if tab_main == "πΈ Media":
|
| 397 |
+
# Show media and video search combined
|
| 398 |
+
show_media()
|
| 399 |
+
st.write("---")
|
| 400 |
+
display_video_search()
|
| 401 |
+
|
| 402 |
+
elif tab_main == "π ArXiv":
|
| 403 |
+
st.subheader("Arxiv Search")
|
| 404 |
+
q = st.text_input("Enter your Arxiv search query:", value=st.session_state['arxiv_last_query'])
|
| 405 |
+
vocal_summary = st.checkbox("π Short Audio Summary (Placeholder - no TTS actually)", value=True)
|
| 406 |
+
titles_summary = st.checkbox("π Titles Only", value=True)
|
| 407 |
+
full_audio = st.checkbox("π Full Audio Results (Placeholder)", value=False)
|
| 408 |
+
|
| 409 |
+
if st.button("π Arxiv Search"):
|
| 410 |
+
st.session_state['arxiv_last_query'] = q
|
| 411 |
+
perform_arxiv_lookup(q, vocal_summary=vocal_summary, titles_summary=titles_summary, full_audio=full_audio)
|
| 412 |
+
|
| 413 |
+
elif tab_main == "π Editor":
|
| 414 |
+
show_file_manager()
|
| 415 |
+
st.write("---")
|
| 416 |
+
display_editor()
|
| 417 |
+
|
| 418 |
+
# Rerun if needed
|
| 419 |
+
if st.session_state.should_rerun:
|
| 420 |
+
st.session_state.should_rerun = False
|
| 421 |
+
st.experimental_rerun()
|
| 422 |
+
|
| 423 |
if __name__ == "__main__":
|
| 424 |
main()
|