qdrant changes

backend/rag.py

@@ -16,6 +16,12 @@ class RAG:
         # Setup Vector Store (Qdrant configuration is handled via environment variables)
         self.vector_store = VectorStore()
         
+        # Verify vector store is properly initialized
+        if not self.vector_store.verify_collection_health():
+            print("Warning: Vector store collection health check failed")
+        else:
+            print("Vector store initialized successfully")
+        
         # Setup Text Splitter
         self.text_splitter = RecursiveCharacterTextSplitter(
             chunk_size=1000,

backend/vector_store.py

@@ -6,6 +6,7 @@ import os
 import logging
 from typing import List, Dict, Any
 from dotenv import load_dotenv
+import time
 
 # Load environment variables
 load_dotenv()
@@ -35,7 +36,7 @@ class VectorStore:
         self.embedding_model = self._initialize_embedding_model()
         
         # Create collection with proper indices
-        self._create_collection_if_not_exists()
+        self._ensure_collection_exists()
     
     def _initialize_embedding_model(self):
         """Initialize the embedding model from a local directory"""
@@ -51,82 +52,30 @@ class VectorStore:
             print(f"Failed to load local model: {e}")
             raise RuntimeError("Failed to initialize embedding model from local path")
 
+    def _collection_exists_and_accessible(self) -> bool:
+        """
+        Check if collection exists and is accessible by trying to get its info.
+        
+        Returns:
+            bool: True if collection exists and is accessible
+        """
+        try:
+            # Try to get collection info - this is more reliable than just listing collections
+            collection_info = self.client.get_collection(self.collection_name)
+            print(f"Collection '{self.collection_name}' exists and is accessible")
+            return True
+        except Exception as e:
+            print(f"Collection '{self.collection_name}' is not accessible: {e}")
+            return False
 
-    
-    # def _initialize_embedding_model(self):
-    #     """Initialize the embedding model with offline support"""
-    #     try:
-    #         # Try to load the model normally first
-    #         print("Attempting to load sentence transformer model...")
-    #         model = SentenceTransformer("all-MiniLM-L6-v2")
-    #         print("Successfully loaded sentence transformer model")
-    #         return model
-            
-    #     except Exception as e:
-    #         print(f"Failed to load model online: {e}")
-    #         print("Attempting to load model in offline mode...")
-            
-    #         try:
-    #             # Try to load from cache with offline mode
-    #             import os
-    #             os.environ['TRANSFORMERS_OFFLINE'] = '1'
-    #             os.environ['HF_HUB_OFFLINE'] = '1'
-                
-    #             model = SentenceTransformer("all-MiniLM-L6-v2", cache_folder=None)
-    #             print("Successfully loaded model in offline mode")
-    #             return model
-                
-    #         except Exception as offline_error:
-    #             print(f"Failed to load model in offline mode: {offline_error}")
-                
-    #             # Try to find a local cache directory
-    #             try:
-    #                 import transformers
-    #                 cache_dir = os.path.join(os.path.expanduser("~"), ".cache", "huggingface", "transformers")
-    #                 if os.path.exists(cache_dir):
-    #                     print(f"Looking for cached model in: {cache_dir}")
-                        
-    #                     # Try to load from specific cache directory
-    #                     model = SentenceTransformer("all-MiniLM-L6-v2", cache_folder=cache_dir)
-    #                     print("Successfully loaded model from cache")
-    #                     return model
-                        
-    #             except Exception as cache_error:
-    #                 print(f"Failed to load from cache: {cache_error}")
-                
-    #             # If all else fails, provide instructions
-    #             error_msg = """
-    #             Failed to initialize sentence transformer model. This is likely due to network connectivity issues.
-                
-    #             Solutions:
-    #             1. Check your internet connection
-    #             2. If behind a corporate firewall, ensure huggingface.co is accessible
-    #             3. Pre-download the model when you have internet access by running:
-    #                python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('all-MiniLM-L6-v2')"
-    #             4. Or manually download the model and place it in your cache directory
-                
-    #             For now, the application will not work without the embedding model.
-    #             """
-                
-    #             print(error_msg)
-    #             raise RuntimeError(f"Cannot initialize embedding model: {str(e)}")
-    
-    def _create_collection_if_not_exists(self) -> bool:
+    def _create_collection(self) -> bool:
         """
-        Create collection with proper payload indices if it doesn't exist.
+        Create the collection with proper configuration.
         
         Returns:
-            bool: True if collection exists or was created successfully
+            bool: True if collection was created successfully
         """
         try:
-            # Check if collection exists
-            collections = self.client.get_collections()
-            collection_names = [col.name for col in collections.collections]
-            print("list of collections : ", collection_names)
-            if self.collection_name in collection_names:
-                print(f"Collection '{self.collection_name}' already exists")
-                return True
-            
             print(f"Creating new collection: {self.collection_name}")
             
             # Vector size for all-MiniLM-L6-v2 is 384
@@ -145,6 +94,9 @@ class VectorStore:
                 ),
             )
             
+            # Wait a moment for collection to be fully created
+            time.sleep(1)
+            
             # Create payload indices
             payload_indices = {
                 "document_id": PayloadSchemaType.KEYWORD,
@@ -152,11 +104,14 @@ class VectorStore:
             }
             
             for field_name, schema_type in payload_indices.items():
-                self.client.create_payload_index(
-                    collection_name=self.collection_name,
-                    field_name=field_name,
-                    field_schema=schema_type
-                )
+                try:
+                    self.client.create_payload_index(
+                        collection_name=self.collection_name,
+                        field_name=field_name,
+                        field_schema=schema_type
+                    )
+                except Exception as idx_error:
+                    print(f"Warning: Failed to create index for {field_name}: {idx_error}")
             
             print(f"Successfully created collection: {self.collection_name}")
             return True
@@ -166,47 +121,105 @@ class VectorStore:
             logger.error(error_msg, exc_info=True)
             print(error_msg)
             return False
-    
-    def add_document(self, text: str, metadata: Dict = None) -> bool:
-        """Add a document to the collection"""
+
+    def _ensure_collection_exists(self) -> bool:
+        """
+        Ensure collection exists and is accessible, create if necessary.
+        
+        Returns:
+            bool: True if collection exists or was created successfully
+        """
         try:
-            # Generate embedding
-            embedding = self.embedding_model.encode([text])[0]
-            
-            # Generate document ID
-            document_id = str(uuid.uuid4())
-            
-            # Create payload with indexed fields
-            payload = {
-                "document_id": document_id,  # KEYWORD index
-                "content": text,             # TEXT index - stores the actual text content
-            }
-            
-            # Add metadata fields if provided
-            if metadata:
-                payload.update(metadata)
+            # First, check if collection exists and is accessible
+            if self._collection_exists_and_accessible():
+                return True
             
-            # Create point
-            point = PointStruct(
-                id=document_id,
-                vector=embedding.tolist(),
-                payload=payload
-            )
+            # If not accessible, try to create it
+            print(f"Collection '{self.collection_name}' not found or not accessible, creating...")
+            return self._create_collection()
             
-            # Store in Qdrant
-            self.client.upsert(
-                collection_name=self.collection_name,
-                points=[point]
-            )
-            
-            return True
         except Exception as e:
-            print(f"Error adding document: {e}")
+            error_msg = f"Failed to ensure collection exists: {str(e)}"
+            logger.error(error_msg, exc_info=True)
+            print(error_msg)
             return False
     
+    def add_document(self, text: str, metadata: Dict = None) -> bool:
+        """Add a document to the collection with retry logic"""
+        max_retries = 3
+        retry_delay = 1
+        
+        for attempt in range(max_retries):
+            try:
+                # Ensure collection exists before adding document
+                if not self._collection_exists_and_accessible():
+                    print("Collection not accessible, trying to recreate...")
+                    if not self._create_collection():
+                        raise Exception("Failed to create collection")
+                
+                # Generate embedding
+                embedding = self.embedding_model.encode([text])[0]
+                
+                # Generate document ID
+                document_id = str(uuid.uuid4())
+                
+                # Create payload with indexed fields
+                payload = {
+                    "document_id": document_id,  # KEYWORD index
+                    "content": text,             # TEXT index - stores the actual text content
+                }
+                
+                # Add metadata fields if provided
+                if metadata:
+                    payload.update(metadata)
+                
+                # Create point
+                point = PointStruct(
+                    id=document_id,
+                    vector=embedding.tolist(),
+                    payload=payload
+                )
+                
+                # Store in Qdrant
+                result = self.client.upsert(
+                    collection_name=self.collection_name,
+                    points=[point]
+                )
+                
+                # Check if upsert was successful
+                if hasattr(result, 'status') and result.status == 'completed':
+                    return True
+                elif hasattr(result, 'operation_id'):
+                    return True
+                else:
+                    print(f"Unexpected upsert result: {result}")
+                    return True  # Assume success if no error was raised
+                
+            except Exception as e:
+                print(f"Error adding document (attempt {attempt + 1}/{max_retries}): {e}")
+                if "Not found" in str(e) and "doesn't exist" in str(e):
+                    # Collection doesn't exist, try to recreate
+                    print("Collection not found, attempting to recreate...")
+                    self._create_collection()
+                
+                if attempt < max_retries - 1:
+                    print(f"Retrying in {retry_delay} seconds...")
+                    time.sleep(retry_delay)
+                    retry_delay *= 2  # Exponential backoff
+                else:
+                    print(f"Failed to add document after {max_retries} attempts")
+                    return False
+        
+        return False
+    
     def search_similar(self, query: str, limit: int = 5) -> List[Dict]:
-        """Search for similar documents"""
+        """Search for similar documents with error handling"""
         try:
+            # Ensure collection exists before searching
+            if not self._collection_exists_and_accessible():
+                print("Collection not accessible for search")
+                return []
+            
             # Generate query embedding
             query_embedding = self.embedding_model.encode([query])[0]
             
@@ -246,4 +259,23 @@ class VectorStore:
             }
         except Exception as e:
             print(f"Error getting collection info: {e}")
-            return {} 
+            return {}
+    
+    def verify_collection_health(self) -> bool:
+        """Verify that the collection is healthy and accessible"""
+        try:
+            # Try to get collection info
+            info = self.get_collection_info()
+            if not info:
+                return False
+            
+            # Try a simple search to verify functionality
+            test_results = self.search_similar("test query", limit=1)
+            # This should not fail even if no results are found
+            
+            print(f"Collection health check passed. Points count: {info.get('points_count', 0)}")
+            return True
+            
+        except Exception as e:
+            print(f"Collection health check failed: {e}")
+            return False