Update app.py

app.py

@@ -8,9 +8,10 @@ from sentence_transformers import SentenceTransformer
 import faiss
 import numpy as np
 import pandas as pd
+import asyncio
+import aiohttp
 
 # Initialize models and variables
-summarizer = pipeline("summarization", model="sshleifer/distilbart-cnn-6-6")  # Using a smaller model for resource efficiency
 embedding_model = SentenceTransformer('all-MiniLM-L6-v2')
 faiss_index = None
 bookmarks = []
@@ -21,56 +22,73 @@ def parse_bookmarks(file_content):
     extracted_bookmarks = []
     for link in soup.find_all('a'):
         url = link.get('href')
-        title = link.text
+        title = link.text.strip()
         if url and title:
             extracted_bookmarks.append({'url': url, 'title': title})
     return extracted_bookmarks
 
-def fetch_url_info(bookmark):
+async def fetch_url_info(session, bookmark):
     url = bookmark['url']
     if url in fetch_cache:
         bookmark.update(fetch_cache[url])
         return bookmark
 
     try:
-        response = requests.get(url, timeout=5)
-        bookmark['etag'] = response.headers.get('ETag', 'N/A')
-        bookmark['status_code'] = response.status_code
-
-        if response.status_code >= 400:
-            bookmark['dead_link'] = True
-            bookmark['content'] = ''
-        else:
-            bookmark['dead_link'] = False
-            soup = BeautifulSoup(response.content, 'html.parser')
-            meta_tags = {meta.get('name', ''): meta.get('content', '') for meta in soup.find_all('meta')}
-            bookmark['meta_tags'] = meta_tags
-            bookmark['content'] = soup.get_text(separator=' ', strip=True)
+        async with session.get(url, timeout=5) as response:
+            bookmark['etag'] = response.headers.get('ETag', 'N/A')
+            bookmark['status_code'] = response.status
+
+            if response.status >= 400:
+                bookmark['dead_link'] = True
+                bookmark['description'] = ''
+            else:
+                bookmark['dead_link'] = False
+                content = await response.text()
+                soup = BeautifulSoup(content, 'html.parser')
+
+                # Extract meta description or Open Graph description
+                meta_description = soup.find('meta', attrs={'name': 'description'})
+                og_description = soup.find('meta', attrs={'property': 'og:description'})
+                if og_description and og_description.get('content'):
+                    description = og_description.get('content')
+                elif meta_description and meta_description.get('content'):
+                    description = meta_description.get('content')
+                else:
+                    description = ''
+
+                bookmark['description'] = description
     except Exception as e:
         bookmark['dead_link'] = True
         bookmark['etag'] = 'N/A'
         bookmark['status_code'] = 'N/A'
-        bookmark['meta_tags'] = {}
-        bookmark['content'] = ''
+        bookmark['description'] = ''
     finally:
         fetch_cache[url] = {
             'etag': bookmark.get('etag'),
             'status_code': bookmark.get('status_code'),
             'dead_link': bookmark.get('dead_link'),
-            'meta_tags': bookmark.get('meta_tags'),
-            'content': bookmark.get('content'),
+            'description': bookmark.get('description'),
         }
     return bookmark
 
+async def process_bookmarks_async(bookmarks):
+    async with aiohttp.ClientSession() as session:
+        tasks = []
+        for bookmark in bookmarks:
+            task = asyncio.ensure_future(fetch_url_info(session, bookmark))
+            tasks.append(task)
+        await asyncio.gather(*tasks)
+
 def generate_summary(bookmark):
-    content = bookmark.get('content', '')
-    if content:
-        # Limit content to first 500 characters to save resources
-        content = content[:500]
-        summary = summarizer(content, max_length=50, min_length=25, do_sample=False)
-        bookmark['summary'] = summary[0]['summary_text']
+    description = bookmark.get('description', '')
+    if description:
+        bookmark['summary'] = description
     else:
-        bookmark['summary'] = 'No content available to summarize.'
+        title = bookmark.get('title', '')
+        if title:
+            bookmark['summary'] = title
+        else:
+            bookmark['summary'] = 'No summary available.'
     return bookmark
 
 def vectorize_and_index(bookmarks):
@@ -83,9 +101,10 @@ def vectorize_and_index(bookmarks):
 
 def display_bookmarks():
     data = []
+    classes = []
     for i, bookmark in enumerate(bookmarks):
         status = "Dead Link" if bookmark.get('dead_link') else "Active"
-        css_class = "dead-link" if bookmark.get('dead_link') else ""
+        row_class = "dead-link" if bookmark.get('dead_link') else ""
         data.append({
             'Index': i,
             'Title': bookmark['title'],
@@ -93,17 +112,17 @@ def display_bookmarks():
             'Status': status,
             'ETag': bookmark.get('etag', 'N/A'),
             'Summary': bookmark.get('summary', ''),
-            'css_class': css_class
         })
+        classes.append(row_class)
     df = pd.DataFrame(data)
-    return df
+    # Return HTML with styles
+    return df.to_html(escape=False, index=False)
 
 def process_uploaded_file(file):
     global bookmarks, faiss_index
     if file is None:
         return "Please upload a bookmarks HTML file.", pd.DataFrame()
     try:
-        # Decode the binary data to a string
         file_content = file.decode('utf-8')
     except UnicodeDecodeError:
         return "Error decoding the file. Please ensure it's a valid HTML file.", pd.DataFrame()
@@ -113,14 +132,17 @@ def process_uploaded_file(file):
     if not bookmarks:
         return "No bookmarks found in the uploaded file.", pd.DataFrame()
 
+    # Asynchronously fetch bookmark info
+    asyncio.run(process_bookmarks_async(bookmarks))
+
+    # Generate summaries using descriptions
     for bookmark in bookmarks:
-        fetch_url_info(bookmark)
         generate_summary(bookmark)
 
     faiss_index, embeddings = vectorize_and_index(bookmarks)
     message = f"Successfully processed {len(bookmarks)} bookmarks."
-    bookmark_df = display_bookmarks()
-    return message, bookmark_df
+    bookmark_html = display_bookmarks()
+    return message, bookmark_html
 
 def chatbot_response(user_query):
     if faiss_index is None or not bookmarks:
@@ -146,13 +168,14 @@ def edit_bookmark(bookmark_idx, new_title, new_url):
             return "Invalid bookmark index.", display_bookmarks()
         bookmarks[bookmark_idx]['title'] = new_title
         bookmarks[bookmark_idx]['url'] = new_url
-        fetch_url_info(bookmarks[bookmark_idx])
+        # Re-fetch bookmark info
+        asyncio.run(process_bookmarks_async([bookmarks[bookmark_idx]]))
         generate_summary(bookmarks[bookmark_idx])
         # Rebuild the FAISS index
         faiss_index, embeddings = vectorize_and_index(bookmarks)
         message = "Bookmark updated successfully."
-        updated_df = display_bookmarks()
-        return message, updated_df
+        updated_html = display_bookmarks()
+        return message, updated_html
     except Exception as e:
         return f"Error: {str(e)}", display_bookmarks()
 
@@ -169,8 +192,8 @@ def delete_bookmark(bookmark_idx):
         else:
             faiss_index = None
         message = "Bookmark deleted successfully."
-        updated_df = display_bookmarks()
-        return message, updated_df
+        updated_html = display_bookmarks()
+        return message, updated_html
     except Exception as e:
         return f"Error: {str(e)}", display_bookmarks()
 
@@ -185,8 +208,7 @@ def build_app():
             bookmark_table = gr.HTML(label="Bookmarks")
 
             def update_bookmark_table(file):
-                message, df = process_uploaded_file(file)
-                html_table = df.to_html(escape=False, index=False)
+                message, html_table = process_uploaded_file(file)
                 return message, html_table
 
             process_button.click(
@@ -220,8 +242,7 @@ def build_app():
             delete_button = gr.Button("Delete Bookmark")
 
             def update_manage_table():
-                df = display_bookmarks()
-                html_table = df.to_html(escape=False, index=False)
+                html_table = display_bookmarks()
                 return html_table
 
             refresh_button.click(