Update app.py

app.py

@@ -86,6 +86,7 @@ openai.api_base = "https://api.groq.com/openai/v1"
 # Initialize global variables for rate limiting
 api_lock = threading.Lock()
 last_api_call_time = 0
+
 def extract_main_content(soup):
     """
     Extract the main content from a webpage while filtering out boilerplate content.
@@ -154,7 +155,6 @@ def get_page_metadata(soup):
             metadata['title'] = og_title.get('content', '').strip()
 
     return metadata
-
 def generate_summary_and_assign_category(bookmark):
     """
     Generate a concise summary and assign a category using a single LLM call.
@@ -298,6 +298,7 @@ Category: [One category]
             bookmark['summary'] = 'No summary available.'
             bookmark['category'] = 'Uncategorized'
             break
+
 def parse_bookmarks(file_content):
     """
     Parse bookmarks from HTML file.
@@ -448,6 +449,7 @@ def display_bookmarks():
         cards += card_html
     logger.info("HTML display generated")
     return cards
+
 def process_uploaded_file(file, state_bookmarks):
     """
     Process the uploaded bookmarks file.
@@ -614,41 +616,35 @@ def chatbot_response(user_query, chat_history):
     try:
         chat_history.append({"role": "user", "content": user_query})
 
-        # Implement better rate limiting
-        max_retries = 5
-        base_wait = 2  # Base wait time in seconds
-        for attempt in range(max_retries):
-            try:
-                with api_lock:
-                    global last_api_call_time
-                    current_time = time.time()
-                    elapsed = current_time - last_api_call_time
-                    if elapsed < base_wait:
-                        sleep_duration = base_wait - elapsed
-                        logger.info(f"Rate limiting: Waiting for {sleep_duration:.2f} seconds...")
-                        time.sleep(sleep_duration)
-                    last_api_call_time = time.time()
-
-                # Search for relevant bookmarks
-                query_vector = embedding_model.encode([user_query]).astype('float32')
-                k = 5
-                distances, ids = faiss_index.search(query_vector, k)
-                ids = ids.flatten()
-
-                id_to_bookmark = {bookmark['id']: bookmark for bookmark in bookmarks}
-                matching_bookmarks = [id_to_bookmark.get(id) for id in ids if id in id_to_bookmark]
-
-                if not matching_bookmarks:
-                    answer = "No relevant bookmarks found for your query."
-                    chat_history.append({"role": "assistant", "content": answer})
-                    return chat_history
-
-                bookmarks_info = "\n".join([
-                    f"Title: {bookmark['title']}\nURL: {bookmark['url']}\nSummary: {bookmark['summary']}"
-                    for bookmark in matching_bookmarks
-                ])
-
-                prompt = f"""
+        with api_lock:
+            global last_api_call_time
+            current_time = time.time()
+            elapsed = current_time - last_api_call_time
+            if elapsed < 2:
+                sleep_duration = 2 - elapsed
+                logger.info(f"Sleeping for {sleep_duration:.2f} seconds to respect rate limits.")
+                time.sleep(sleep_duration)
+            last_api_call_time = time.time()
+
+        query_vector = embedding_model.encode([user_query]).astype('float32')
+        k = 5
+        distances, ids = faiss_index.search(query_vector, k)
+        ids = ids.flatten()
+
+        id_to_bookmark = {bookmark['id']: bookmark for bookmark in bookmarks}
+        matching_bookmarks = [id_to_bookmark.get(id) for id in ids if id in id_to_bookmark]
+
+        if not matching_bookmarks:
+            answer = "No relevant bookmarks found for your query."
+            chat_history.append({"role": "assistant", "content": answer})
+            return chat_history
+
+        bookmarks_info = "\n".join([
+            f"Title: {bookmark['title']}\nURL: {bookmark['url']}\nSummary: {bookmark['summary']}"
+            for bookmark in matching_bookmarks
+        ])
+
+        prompt = f"""
 A user asked: "{user_query}"
 Based on the bookmarks below, provide a helpful answer to the user's query, referencing the relevant bookmarks.
 Bookmarks:
@@ -656,34 +652,39 @@ Bookmarks:
 Provide a concise and helpful response.
 """
 
-                response = openai.ChatCompletion.create(
-                    model='llama-3.1-70b-versatile',
-                    messages=[
-                        {"role": "user", "content": prompt}
-                    ],
-                    max_tokens=300,
-                    temperature=0.7,
-                )
+        def estimate_tokens(text):
+            return len(text) / 4
 
-                answer = response['choices'][0]['message']['content'].strip()
-                logger.info("Chatbot response generated")
-                
-                # Add a small delay between successful requests
-                time.sleep(base_wait)
-                
-                chat_history.append({"role": "assistant", "content": answer})
-                return chat_history
-
-            except openai.error.RateLimitError as e:
-                wait_time = min(base_wait * (2 ** attempt), 30)  # Cap maximum wait time at 30 seconds
-                logger.warning(f"Rate limit reached. Attempt {attempt + 1}/{max_retries}. Waiting for {wait_time} seconds...")
-                time.sleep(wait_time)
-                if attempt == max_retries - 1:
-                    error_message = "⚠️ The service is currently experiencing high demand. Please try again in a few moments."
-                    chat_history.append({"role": "assistant", "content": error_message})
-                    return chat_history
-                continue
+        prompt_tokens = estimate_tokens(prompt)
+        max_tokens = 300
+        total_tokens = prompt_tokens + max_tokens
+
+        tokens_per_minute = 40000
+        tokens_per_second = tokens_per_minute / 60
+        required_delay = total_tokens / tokens_per_second
+        sleep_time = max(required_delay, 2)
+
+        response = openai.ChatCompletion.create(
+            model='llama-3.1-70b-versatile',
+            messages=[
+                {"role": "user", "content": prompt}
+            ],
+            max_tokens=int(max_tokens),
+            temperature=0.7,
+        )
+
+        answer = response['choices'][0]['message']['content'].strip()
+        logger.info("Chatbot response generated")
+        time.sleep(sleep_time)
+
+        chat_history.append({"role": "assistant", "content": answer})
+        return chat_history
 
+    except openai.error.RateLimitError as e:
+        wait_time = int(e.headers.get("Retry-After", 5))
+        logger.warning(f"Rate limit reached. Waiting for {wait_time} seconds before retrying...")
+        time.sleep(wait_time)
+        return chatbot_response(user_query, chat_history)
     except Exception as e:
         error_message = f"⚠️ Error processing your query: {str(e)}"
         logger.error(error_message, exc_info=True)