Update app.py

app.py

@@ -45,8 +45,19 @@ faiss_index = None
 bookmarks = []
 fetch_cache = {}
 
-# Lock for thread-safe operations
-lock = threading.Lock()
+# Groq API Rate Limits
+GROQ_RPM = 30  # requests per minute
+GROQ_TPM = 40000  # tokens per minute
+SECONDS_PER_MINUTE = 60
+MIN_TIME_BETWEEN_CALLS = SECONDS_PER_MINUTE / GROQ_RPM  # 2 seconds between calls
+MAX_CONCURRENT_CALLS = 3  # Keep concurrent calls limited to prevent rate limits
+TOKEN_BUFFER = 0.9  # Use 90% of token limit to be safe
+
+# Rate limiting tools
+api_lock = threading.Lock()
+request_times = []  # Track request timestamps
+token_usage = []    # Track token usage
+LLM_SEMAPHORE = threading.Semaphore(MAX_CONCURRENT_CALLS)
 
 # Define the categories
 CATEGORIES = [
@@ -83,10 +94,34 @@ if not GROQ_API_KEY:
 openai.api_key = GROQ_API_KEY
 openai.api_base = "https://api.groq.com/openai/v1"
 
-# Initialize global variables for rate limiting
-api_lock = threading.Lock()
-last_api_call_time = 0
+def manage_rate_limits():
+    """
+    Manage both request and token rate limits.
+    Returns the time to wait (if any) before making next request.
+    """
+    current_time = time.time()
+    minute_ago = current_time - SECONDS_PER_MINUTE
+
+    # Clean up old entries
+    global request_times, token_usage
+    request_times = [t for t in request_times if t > minute_ago]
+    token_usage = [t for t, _ in token_usage if t > minute_ago]
 
+    # Check request rate
+    if len(request_times) >= GROQ_RPM:
+        oldest_request = request_times[0]
+        return max(0, SECONDS_PER_MINUTE - (current_time - oldest_request))
+
+    # Check token rate
+    total_tokens = sum(tokens for _, tokens in token_usage)
+    if total_tokens >= GROQ_TPM * TOKEN_BUFFER:
+        return 1.0  # Wait a second if near token limit
+
+    return 0
+
+def estimate_tokens(text):
+    """Estimate tokens in text using GPT-3 tokenizer approximation"""
+    return len(text.split()) * 1.3  # Rough estimate: 1.3 tokens per word
 def extract_main_content(soup):
     """
     Extract the main content from a webpage while filtering out boilerplate content.
@@ -155,186 +190,14 @@ def get_page_metadata(soup):
             metadata['title'] = og_title.get('content', '').strip()
 
     return metadata
-def generate_summary_and_assign_category(bookmark):
-    """
-    Generate a concise summary and assign a category using a single LLM call.
-    """
-    logger.info(f"Generating summary and assigning category for bookmark: {bookmark.get('url')}")
-
-    max_retries = 3
-    retry_count = 0
-    base_wait = 5  # Increased base wait time to 5 seconds
-
-    while retry_count < max_retries:
-        try:
-            # Rate Limiting Logic - Modified
-            with api_lock:
-                global last_api_call_time
-                current_time = time.time()
-                elapsed = current_time - last_api_call_time
-                if elapsed < base_wait:
-                    sleep_duration = base_wait - elapsed
-                    logger.info(f"Rate limiting: Waiting for {sleep_duration:.2f} seconds...")
-                    time.sleep(sleep_duration)
-                last_api_call_time = time.time()
-
-            html_content = bookmark.get('html_content', '')
-            soup = BeautifulSoup(html_content, 'html.parser')
-            metadata = get_page_metadata(soup)
-            main_content = extract_main_content(soup)
-
-            # Prepare content for the prompt
-            content_parts = []
-            if metadata['title']:
-                content_parts.append(f"Title: {metadata['title']}")
-            if metadata['description']:
-                content_parts.append(f"Description: {metadata['description']}")
-            if metadata['keywords']:
-                content_parts.append(f"Keywords: {metadata['keywords']}")
-            if main_content:
-                content_parts.append(f"Main Content: {main_content}")
-
-            content_text = '\n'.join(content_parts)
-
-            # Detect insufficient or erroneous content
-            error_keywords = ['Access Denied', 'Security Check', 'Cloudflare', 'captcha', 'unusual traffic']
-            if not content_text or len(content_text.split()) < 50:
-                use_prior_knowledge = True
-                logger.info(f"Content for {bookmark.get('url')} is insufficient. Instructing LLM to use prior knowledge.")
-            elif any(keyword.lower() in content_text.lower() for keyword in error_keywords):
-                use_prior_knowledge = True
-                logger.info(f"Content for {bookmark.get('url')} contains error messages. Instructing LLM to use prior knowledge.")
-            else:
-                use_prior_knowledge = False
-
-            if use_prior_knowledge:
-                prompt = f"""
-You are a knowledgeable assistant with up-to-date information as of 2023.
-URL: {bookmark.get('url')}
-Provide:
-1. A concise summary (max two sentences) about this website.
-2. Assign the most appropriate category from the list below.
-Categories:
-{', '.join([f'"{cat}"' for cat in CATEGORIES])}
-Format:
-Summary: [Your summary]
-Category: [One category]
-"""
-            else:
-                prompt = f"""
-You are an assistant that creates concise webpage summaries and assigns categories.
-Content:
-{content_text}
-Provide:
-1. A concise summary (max two sentences) focusing on the main topic.
-2. Assign the most appropriate category from the list below.
-Categories:
-{', '.join([f'"{cat}"' for cat in CATEGORIES])}
-Format:
-Summary: [Your summary]
-Category: [One category]
-"""
-
-            def estimate_tokens(text):
-                return len(text) / 4
-
-            prompt_tokens = estimate_tokens(prompt)
-            max_tokens = 150
-            total_tokens = prompt_tokens + max_tokens
-
-            tokens_per_minute = 40000
-            tokens_per_second = tokens_per_minute / 60
-            required_delay = total_tokens / tokens_per_second
-            sleep_time = max(required_delay, base_wait)  # Use at least base_wait seconds
-
-            response = openai.ChatCompletion.create(
-                model='llama-3.1-70b-versatile',
-                messages=[
-                    {"role": "user", "content": prompt}
-                ],
-                max_tokens=int(max_tokens),
-                temperature=0.5,
-            )
-
-            content = response['choices'][0]['message']['content'].strip()
-            if not content:
-                raise ValueError("Empty response received from the model.")
-
-            summary_match = re.search(r"Summary:\s*(.*)", content)
-            category_match = re.search(r"Category:\s*(.*)", content)
-
-            if summary_match:
-                bookmark['summary'] = summary_match.group(1).strip()
-            else:
-                bookmark['summary'] = 'No summary available.'
-
-            if category_match:
-                category = category_match.group(1).strip().strip('"')
-                if category in CATEGORIES:
-                    bookmark['category'] = category
-                else:
-                    bookmark['category'] = 'Uncategorized'
-            else:
-                bookmark['category'] = 'Uncategorized'
-
-            # Simple keyword-based validation
-            summary_lower = bookmark['summary'].lower()
-            url_lower = bookmark['url'].lower()
-            if 'social media' in summary_lower or 'twitter' in summary_lower or 'x.com' in url_lower:
-                bookmark['category'] = 'Social Media'
-            elif 'wikipedia' in url_lower:
-                bookmark['category'] = 'Reference and Knowledge Bases'
-
-            logger.info("Successfully generated summary and assigned category")
-            
-            # Add consistent delay after successful processing
-            time.sleep(sleep_time)
-            break
-
-        except openai.error.RateLimitError as e:
-            retry_count += 1
-            # Use exponential backoff with a maximum wait time
-            wait_time = min(base_wait * (2 ** retry_count), 30)  # Cap at 30 seconds
-            logger.warning(f"Rate limit reached. Waiting for {wait_time} seconds before retrying... (Attempt {retry_count}/{max_retries})")
-            time.sleep(wait_time)
-            if retry_count == max_retries:
-                bookmark['summary'] = 'Summary generation failed due to rate limits.'
-                bookmark['category'] = 'Uncategorized'
-                break
-        except Exception as e:
-            logger.error(f"Error generating summary and assigning category: {e}", exc_info=True)
-            bookmark['summary'] = 'No summary available.'
-            bookmark['category'] = 'Uncategorized'
-            break
 
-def parse_bookmarks(file_content):
-    """
-    Parse bookmarks from HTML file.
-    """
-    logger.info("Parsing bookmarks")
-    try:
-        soup = BeautifulSoup(file_content, 'html.parser')
-        extracted_bookmarks = []
-        for link in soup.find_all('a'):
-            url = link.get('href')
-            title = link.text.strip()
-            if url and title:
-                if url.startswith('http://') or url.startswith('https://'):
-                    extracted_bookmarks.append({'url': url, 'title': title})
-                else:
-                    logger.info(f"Skipping non-http/https URL: {url}")
-        logger.info(f"Extracted {len(extracted_bookmarks)} bookmarks")
-        return extracted_bookmarks
-    except Exception as e:
-        logger.error("Error parsing bookmarks: %s", e, exc_info=True)
-        raise
 def fetch_url_info(bookmark):
     """
     Fetch information about a URL.
     """
     url = bookmark['url']
     if url in fetch_cache:
-        with lock:
+        with api_lock:
             bookmark.update(fetch_cache[url])
         return
 
@@ -378,7 +241,7 @@ def fetch_url_info(bookmark):
         bookmark['html_content'] = ''
         logger.error(f"Error fetching URL info for {url}: {e}", exc_info=True)
     finally:
-        with lock:
+        with api_lock:
             fetch_cache[url] = {
                 'etag': bookmark.get('etag'),
                 'status_code': bookmark.get('status_code'),
@@ -388,6 +251,87 @@ def fetch_url_info(bookmark):
                 'slow_link': bookmark.get('slow_link', False),
             }
 
+def process_bookmarks_batch(bookmarks_batch):
+    """Process a batch of bookmarks with controlled rate limiting"""
+    for bookmark in bookmarks_batch:
+        with LLM_SEMAPHORE:
+            while True:
+                with api_lock:
+                    wait_time = manage_rate_limits()
+                    if wait_time <= 0:
+                        break
+                    logger.info(f"Rate limiting: Waiting for {wait_time:.2f} seconds...")
+                    time.sleep(wait_time)
+
+                try:
+                    html_content = bookmark.get('html_content', '')
+                    soup = BeautifulSoup(html_content, 'html.parser')
+                    metadata = get_page_metadata(soup)
+                    main_content = extract_main_content(soup)
+
+                    # Prepare shortened prompt to reduce tokens
+                    content = f"Title: {metadata['title']}\nURL: {bookmark['url']}"
+                    if len(main_content) > 1000:  # Limit content length
+                        main_content = main_content[:1000] + "..."
+                    
+                    prompt = f"""Analyze this webpage:
+{content}
+Content: {main_content}
+Provide in format:
+Summary: [2 sentences max]
+Category: [{', '.join(CATEGORIES)}]"""
+
+                    # Estimate tokens
+                    input_tokens = estimate_tokens(prompt)
+                    max_tokens = 150
+                    total_tokens = input_tokens + max_tokens
+
+                    # Make API call
+                    response = openai.ChatCompletion.create(
+                        model='llama-3.1-70b-versatile',
+                        messages=[{"role": "user", "content": prompt}],
+                        max_tokens=max_tokens,
+                        temperature=0.5,
+                    )
+
+                    # Track rate limits
+                    with api_lock:
+                        current_time = time.time()
+                        request_times.append(current_time)
+                        token_usage.append((current_time, total_tokens))
+
+                    content = response['choices'][0]['message']['content'].strip()
+                    
+                    # Process response
+                    summary_match = re.search(r"Summary:\s*(.*?)(?:\n|$)", content)
+                    category_match = re.search(r"Category:\s*(.*?)(?:\n|$)", content)
+
+                    bookmark['summary'] = summary_match.group(1).strip() if summary_match else 'No summary available.'
+                    
+                    if category_match:
+                        category = category_match.group(1).strip().strip('"')
+                        bookmark['category'] = category if category in CATEGORIES else 'Uncategorized'
+                    else:
+                        bookmark['category'] = 'Uncategorized'
+
+                    # Quick category validation
+                    if 'social media' in bookmark['url'].lower() or 'twitter' in bookmark['url'].lower() or 'x.com' in bookmark['url'].lower():
+                        bookmark['category'] = 'Social Media'
+                    elif 'wikipedia' in bookmark['url'].lower():
+                        bookmark['category'] = 'Reference and Knowledge Bases'
+
+                    logger.info(f"Successfully processed bookmark: {bookmark['url']}")
+                    break
+
+                except openai.error.RateLimitError as e:
+                    wait_time = int(e.headers.get('Retry-After', 5))
+                    logger.warning(f"Rate limit hit, waiting {wait_time} seconds...")
+                    time.sleep(wait_time)
+                except Exception as e:
+                    logger.error(f"Error processing bookmark: {e}")
+                    bookmark['summary'] = 'Processing failed.'
+                    bookmark['category'] = 'Uncategorized'
+                    break
 def vectorize_and_index(bookmarks_list):
     """
     Create vector embeddings for bookmarks and build FAISS index with ID mapping.
@@ -459,7 +403,7 @@ def display_bookmarks():
 
 def process_uploaded_file(file, state_bookmarks):
     """
-    Process the uploaded bookmarks file.
+    Process uploaded file with optimized batch processing
     """
     global bookmarks, faiss_index
     logger.info("Processing uploaded file")
@@ -470,52 +414,63 @@ def process_uploaded_file(file, state_bookmarks):
 
     try:
         file_content = file.decode('utf-8')
-    except UnicodeDecodeError as e:
-        logger.error(f"Error decoding the file: {e}", exc_info=True)
-        return "Error decoding the file. Please ensure it's a valid HTML file.", '', state_bookmarks, display_bookmarks(), gr.update(choices=[])
-
-    try:
         bookmarks = parse_bookmarks(file_content)
-    except Exception as e:
-        logger.error(f"Error parsing bookmarks: {e}", exc_info=True)
-        return "Error parsing the bookmarks HTML file.", '', state_bookmarks, display_bookmarks(), gr.update(choices=[])
 
-    if not bookmarks:
-        logger.warning("No bookmarks found in the uploaded file")
-        return "No bookmarks found in the uploaded file.", '', state_bookmarks, display_bookmarks(), gr.update(choices=[])
+        if not bookmarks:
+            return "No bookmarks found in the file.", '', state_bookmarks, display_bookmarks(), gr.update(choices=[])
 
-    # Assign unique IDs to bookmarks
-    for idx, bookmark in enumerate(bookmarks):
-        bookmark['id'] = idx
+        # Assign IDs
+        for idx, bookmark in enumerate(bookmarks):
+            bookmark['id'] = idx
 
-    # Fetch bookmark info concurrently
-    logger.info("Fetching URL info concurrently")
-    with ThreadPoolExecutor(max_workers=10) as executor:
-        executor.map(fetch_url_info, bookmarks)
+        # First fetch all URLs concurrently
+        with ThreadPoolExecutor(max_workers=10) as executor:
+            executor.map(fetch_url_info, bookmarks)
 
-    # Process bookmarks concurrently with LLM calls
-    logger.info("Processing bookmarks with LLM concurrently")
-    with ThreadPoolExecutor(max_workers=1) as executor:
-        executor.map(generate_summary_and_assign_category, bookmarks)
+        # Process bookmarks in parallel with controlled concurrency
+        batch_size = min(MAX_CONCURRENT_CALLS, len(bookmarks))
+        batches = [bookmarks[i:i + batch_size] for i in range(0, len(bookmarks), batch_size)]
+        
+        with ThreadPoolExecutor(max_workers=MAX_CONCURRENT_CALLS) as executor:
+            executor.map(process_bookmarks_batch, batches)
 
-    try:
+        # Build FAISS index
         faiss_index = vectorize_and_index(bookmarks)
-    except Exception as e:
-        logger.error(f"Error building FAISS index: {e}", exc_info=True)
-        return "Error building search index.", '', state_bookmarks, display_bookmarks(), gr.update(choices=[])
 
-    message = f"✅ Successfully processed {len(bookmarks)} bookmarks."
-    logger.info(message)
+        # Update display and state
+        bookmark_html = display_bookmarks()
+        choices = [f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})"
+                  for i, bookmark in enumerate(bookmarks)]
+        state_bookmarks = bookmarks.copy()
 
-    # Generate displays and updates
-    bookmark_html = display_bookmarks()
-    choices = [f"{i+1}. {bookmark['title']} (Category: {bookmark['category']})"
-               for i, bookmark in enumerate(bookmarks)]
+        return "✅ Processing complete!", bookmark_html, state_bookmarks, bookmark_html, gr.update(choices=choices)
 
-    # Update state
-    state_bookmarks = bookmarks.copy()
+    except Exception as e:
+        logger.error(f"Error processing file: {e}")
+        return f"Error processing file: {str(e)}", '', state_bookmarks, display_bookmarks(), gr.update(choices=[])
+
+def parse_bookmarks(file_content):
+    """
+    Parse bookmarks from HTML file.
+    """
+    logger.info("Parsing bookmarks")
+    try:
+        soup = BeautifulSoup(file_content, 'html.parser')
+        extracted_bookmarks = []
+        for link in soup.find_all('a'):
+            url = link.get('href')
+            title = link.text.strip()
+            if url and title:
+                if url.startswith('http://') or url.startswith('https://'):
+                    extracted_bookmarks.append({'url': url, 'title': title})
+                else:
+                    logger.info(f"Skipping non-http/https URL: {url}")
+        logger.info(f"Extracted {len(extracted_bookmarks)} bookmarks")
+        return extracted_bookmarks
+    except Exception as e:
+        logger.error("Error parsing bookmarks: %s", e, exc_info=True)
+        raise
 
-    return message, bookmark_html, state_bookmarks, bookmark_html, gr.update(choices=choices)
 def delete_selected_bookmarks(selected_indices, state_bookmarks):
     """
     Delete selected bookmarks and remove their vectors from the FAISS index.
@@ -578,7 +533,6 @@ def edit_selected_bookmarks_category(selected_indices, new_category, state_bookm
     state_bookmarks = bookmarks.copy()
 
     return message, gr.update(choices=choices), display_bookmarks(), state_bookmarks
-
 def export_bookmarks():
     """
     Export bookmarks to an HTML file.
@@ -622,81 +576,82 @@ def chatbot_response(user_query, chat_history):
     try:
         chat_history.append({"role": "user", "content": user_query})
 
-        # Implement better rate limiting
-        max_retries = 5
-        base_wait = 5  # Increased base wait time to 5 seconds
-        for attempt in range(max_retries):
-            try:
+        with LLM_SEMAPHORE:
+            while True:
                 with api_lock:
-                    global last_api_call_time
-                    current_time = time.time()
-                    elapsed = current_time - last_api_call_time
-                    if elapsed < base_wait:
-                        sleep_duration = base_wait - elapsed
-                        logger.info(f"Rate limiting: Waiting for {sleep_duration:.2f} seconds...")
-                        time.sleep(sleep_duration)
-                    last_api_call_time = time.time()
-
-                # Search for relevant bookmarks
-                query_vector = embedding_model.encode([user_query]).astype('float32')
-                k = 5
-                distances, ids = faiss_index.search(query_vector, k)
-                ids = ids.flatten()
-
-                id_to_bookmark = {bookmark['id']: bookmark for bookmark in bookmarks}
-                matching_bookmarks = [id_to_bookmark.get(id) for id in ids if id in id_to_bookmark]
-
-                if not matching_bookmarks:
-                    answer = "No relevant bookmarks found for your query."
+                    wait_time = manage_rate_limits()
+                    if wait_time <= 0:
+                        break
+                    logger.info(f"Rate limiting: Waiting for {wait_time:.2f} seconds...")
+                    time.sleep(wait_time)
+
+                try:
+                    # Search for relevant bookmarks
+                    query_vector = embedding_model.encode([user_query]).astype('float32')
+                    k = 5
+                    distances, ids = faiss_index.search(query_vector, k)
+                    ids = ids.flatten()
+
+                    id_to_bookmark = {bookmark['id']: bookmark for bookmark in bookmarks}
+                    matching_bookmarks = [id_to_bookmark.get(id) for id in ids if id in id_to_bookmark]
+
+                    if not matching_bookmarks:
+                        answer = "No relevant bookmarks found for your query."
+                        chat_history.append({"role": "assistant", "content": answer})
+                        return chat_history
+
+                    # Prepare concise prompt
+                    bookmarks_info = "\n".join([
+                        f"Title: {bookmark['title']}\nURL: {bookmark['url']}\nSummary: {bookmark['summary']}"
+                        for bookmark in matching_bookmarks
+                    ])
+
+                    prompt = f"""User Query: "{user_query}"
+Found Bookmarks:
+{bookmarks_info}
+Provide a helpful, concise response."""
+
+                    # Estimate tokens and make API call
+                    input_tokens = estimate_tokens(prompt)
+                    max_tokens = 300
+                    total_tokens = input_tokens + max_tokens
+
+                    response = openai.ChatCompletion.create(
+                        model='llama-3.1-70b-versatile',
+                        messages=[{"role": "user", "content": prompt}],
+                        max_tokens=max_tokens,
+                        temperature=0.7,
+                    )
+
+                    # Track rate limits
+                    with api_lock:
+                        current_time = time.time()
+                        request_times.append(current_time)
+                        token_usage.append((current_time, total_tokens))
+
+                    answer = response['choices'][0]['message']['content'].strip()
+                    logger.info("Chatbot response generated")
+                    
                     chat_history.append({"role": "assistant", "content": answer})
                     return chat_history
 
-                bookmarks_info = "\n".join([
-                    f"Title: {bookmark['title']}\nURL: {bookmark['url']}\nSummary: {bookmark['summary']}"
-                    for bookmark in matching_bookmarks
-                ])
-
-                prompt = f"""
-A user asked: "{user_query}"
-Based on the bookmarks below, provide a helpful answer to the user's query, referencing the relevant bookmarks.
-Bookmarks:
-{bookmarks_info}
-Provide a concise and helpful response.
-"""
-
-                response = openai.ChatCompletion.create(
-                    model='llama-3.1-70b-versatile',
-                    messages=[
-                        {"role": "user", "content": prompt}
-                    ],
-                    max_tokens=300,
-                    temperature=0.7,
-                )
-
-                answer = response['choices'][0]['message']['content'].strip()
-                logger.info("Chatbot response generated")
-                
-                # Add a small delay between successful requests
-                time.sleep(base_wait)
-                
-                chat_history.append({"role": "assistant", "content": answer})
-                return chat_history
-
-            except openai.error.RateLimitError as e:
-                wait_time = min(base_wait * (2 ** attempt), 30)  # Cap maximum wait time at 30 seconds
-                logger.warning(f"Rate limit reached. Attempt {attempt + 1}/{max_retries}. Waiting for {wait_time} seconds...")
-                time.sleep(wait_time)
-                if attempt == max_retries - 1:
-                    error_message = "⚠️ The service is currently experiencing high demand. Please try again in a few moments."
+                except openai.error.RateLimitError as e:
+                    wait_time = int(e.headers.get('Retry-After', 5))
+                    logger.warning(f"Rate limit hit, waiting {wait_time} seconds...")
+                    time.sleep(wait_time)
+                    continue
+                except Exception as e:
+                    error_message = f"⚠️ Error processing your query: {str(e)}"
+                    logger.error(error_message, exc_info=True)
                     chat_history.append({"role": "assistant", "content": error_message})
                     return chat_history
-                continue
 
     except Exception as e:
         error_message = f"⚠️ Error processing your query: {str(e)}"
         logger.error(error_message, exc_info=True)
         chat_history.append({"role": "assistant", "content": error_message})
         return chat_history
+
 def build_app():
     """
     Build and launch the Gradio app.