fix: search

app.py

@@ -33,8 +33,12 @@ def load_dataset(query):
     # Always fetch fresh results for the specific query
     with st.spinner("Searching autism research papers..."):
         import faiss_index.index as idx
-        # Make the query more specific to autism and b12
-        search_query = f"{query} AND (cat:q-bio.NC OR cat:q-bio.QM OR cat:q-bio.GN OR cat:q-bio.CB OR cat:q-bio.MN)"
+        # Ensure both autism and the query terms are included
+        if 'autism' not in query.lower():
+            search_query = f"autism {query}"
+        else:
+            search_query = query
+            
         papers = idx.fetch_arxiv_papers(search_query, max_results=25)
         if not papers:
             st.warning("No relevant papers found. Please try rephrasing your question.")
@@ -88,6 +92,7 @@ def generate_answer(question, context, max_length=150):
 
 # Streamlit App
 st.title("🧩 AMA Autism")
+st.write("This app searches through scientific papers to answer your questions about autism. For best results, be specific in your questions.")
 query = st.text_input("Please ask me anything about autism ✨")
 
 if query:

faiss_index/index.py

@@ -18,34 +18,43 @@ def fetch_arxiv_papers(query, max_results=10):
     """Fetch papers from arXiv and format them for RAG"""
     client = arxiv.Client()
     
-    # Construct a more focused search query
-    search_terms = query.lower().split()
-    if 'autism' not in search_terms:
-        search_terms.insert(0, 'autism')
+    # Clean and prepare the search query
+    query = query.replace('and', '').strip()  # Remove 'and' as it's treated as AND operator
+    terms = [term.strip() for term in query.split() if term.strip()]
     
-    # Add specific category filters for medical and biological papers
-    search_query = f"({' AND '.join(search_terms)}) AND (cat:q-bio.NC OR cat:q-bio.QM OR cat:q-bio.GN OR cat:q-bio.CB OR cat:q-bio.MN)"
+    # Create a more flexible search query
+    search_query = ' OR '.join([f'abs:"{term}" OR ti:"{term}"' for term in terms])
+    search_query = f'({search_query}) AND (cat:q-bio* OR cat:med*)'
+    
+    logging.info(f"Searching arXiv with query: {search_query}")
     
     search = arxiv.Search(
         query=search_query,
-        max_results=max_results,
+        max_results=max_results * 2,  # Get more results to filter
         sort_by=arxiv.SortCriterion.Relevance
     )
     
-    results = list(client.results(search))
-    papers = []
-    
-    # Filter results to ensure they're relevant to autism
-    for i, result in enumerate(results):
-        if 'autism' in result.title.lower() or 'autism' in result.summary.lower():
-            papers.append({
-                "id": str(i),
-                "text": result.summary,
-                "title": result.title
-            })
-    
-    logging.info(f"Fetched {len(papers)} relevant papers from arXiv")
-    return papers
+    try:
+        results = list(client.results(search))
+        papers = []
+        
+        for i, result in enumerate(results):
+            # Include paper if it contains any of the search terms
+            text = (result.title + " " + result.summary).lower()
+            if any(term.lower() in text for term in terms):
+                papers.append({
+                    "id": str(i),
+                    "text": result.summary,
+                    "title": result.title
+                })
+                if len(papers) >= max_results:
+                    break
+        
+        logging.info(f"Found {len(papers)} relevant papers from arXiv")
+        return papers
+    except Exception as e:
+        logging.error(f"Error fetching papers from arXiv: {str(e)}")
+        return []
 
 def build_faiss_index(papers, dataset_dir=DATASET_DIR):
     """Build and save dataset with FAISS index for RAG"""