update

app.py

@@ -6,6 +6,8 @@ from uuid import uuid4
 from gradio_client import Client, handle_file
 from utils.retriever import retrieve_paragraphs
 from utils.generator import generate
+import json
+import ast
 
 # Sample questions for examples 
 SAMPLE_QUESTIONS = {
@@ -33,6 +35,31 @@ def finish_chat():
     """Finish chat and reset input"""
     return gr.update(interactive=True, value="")
 
+def make_html_source(source,i):
+    """
+    takes the text and converts it into html format for display in "source" side tab
+    """
+    meta = source['answer_metadata']
+    content = source['answer'].strip()
+
+    name = meta['filename']
+    card = f"""
+        <div class="card" id="doc{i}">
+            <div class="card-content">
+                <h2>Doc {i} - {meta['filename']} - Page {int(meta['page'])}</h2>
+                <p>{content}</p>
+            </div>
+            <div class="card-footer">
+                <span>{name}</span>
+                <a href="{meta['filename']}#page={int(meta['page'])}" target="_blank" class="pdf-link">
+                    <span role="img" aria-label="Open PDF">🔗</span>
+                </a>
+            </div>
+        </div>
+        """
+
+    return card
+
 async def chat_response(query, history, category):
     """Generate chat response based on method and inputs"""
     
@@ -72,19 +99,32 @@ async def chat_response(query, history, category):
     
     # # Handle "Talk to Reports"
     # else: 
-    try: 
-        retrieved_paragraphs = retrieve_paragraphs(query, category)
-        response = await generate(query=query, context=retrieved_paragraphs)
-            
-    except Exception as e:
-        response = f"Error retrieving information: {str(e)}"
     
+    retrieved_paragraphs = retrieve_paragraphs(query, category)
+    context_retrieved = ast.literal_eval(retrieved_paragraphs)
+    print(retrieved_paragraphs)
+    # print(type(retrieved_paragraphs))
+            # api returns output as string, therefore we first convert string using json
+    # context_retrieved = json.loads(retrieved_paragraphs)
+    # print("converting conesxt to json")
+    # building list of only content, no metadata
+    context_retrieved_formatted = "||".join(doc['answer'] for doc in context_retrieved)
+    context_retrieved_lst = [doc['answer'] for doc in context_retrieved]
+    print(context_retrieved_lst)
+        ## -----------------Prepare HTML for displaying source documents --------------
+    docs_html = []
+    for i, d in enumerate(context_retrieved, 1):
+        docs_html.append(make_html_source(d, i))
+    docs_html = "".join(docs_html)
+    response = await generate(query=query, context=context_retrieved_lst)
+            
+
     displayed_response = ""     
     
     for i, char in enumerate(response):         
         displayed_response += char        
         history[-1] = (query, displayed_response)         
-        yield history, "**Sources:** Sample source documents would appear here..."         
+        yield history, docs_html         
         # Only add delay every few characters to avoid being too slow         
         if i % 3 == 0:  # Adjust this number to control speed             
             await asyncio.sleep(0.02)

utils/generator.py

@@ -113,67 +113,67 @@ chat_model = get_chat_model()
 # ---------------------------------------------------------------------
 # Context processing - may need further refinement (i.e. to manage other data sources)
 # ---------------------------------------------------------------------
-def extract_relevant_fields(retrieval_results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-    """
-    Extract only relevant fields from retrieval results.
+# def extract_relevant_fields(retrieval_results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+#     """
+#     Extract only relevant fields from retrieval results.
     
-    Args:
-        retrieval_results: List of JSON objects from retriever
+#     Args:
+#         retrieval_results: List of JSON objects from retriever
         
-    Returns:
-        List of processed objects with only relevant fields
-    """
+#     Returns:
+#         List of processed objects with only relevant fields
+#     """
 
-    retrieval_results = ast.literal_eval(retrieval_results)
+#     retrieval_results = ast.literal_eval(retrieval_results)
 
-    processed_results = []
+#     processed_results = []
     
-    for result in retrieval_results:
-        # Extract the answer content
-        answer = result.get('answer', '')
+#     for result in retrieval_results:
+#         # Extract the answer content
+#         answer = result.get('answer', '')
         
-        # Extract document identification from metadata
-        metadata = result.get('answer_metadata', {})
-        doc_info = {
-            'answer': answer,
-            'filename': metadata.get('filename', 'Unknown'),
-            'page': metadata.get('page', 'Unknown'),
-            'year': metadata.get('year', 'Unknown'),
-            'source': metadata.get('source', 'Unknown'),
-            'document_id': metadata.get('_id', 'Unknown')
-        }
+#         # Extract document identification from metadata
+#         metadata = result.get('answer_metadata', {})
+#         doc_info = {
+#             'answer': answer,
+#             'filename': metadata.get('filename', 'Unknown'),
+#             'page': metadata.get('page', 'Unknown'),
+#             'year': metadata.get('year', 'Unknown'),
+#             'source': metadata.get('source', 'Unknown'),
+#             'document_id': metadata.get('_id', 'Unknown')
+#         }
         
-        processed_results.append(doc_info)
+#         processed_results.append(doc_info)
     
-    return processed_results
+#     return processed_results
 
-def format_context_from_results(processed_results: List[Dict[str, Any]]) -> str:
-    """
-    Format processed retrieval results into a context string for the LLM.
+# def format_context_from_results(processed_results: List[Dict[str, Any]]) -> str:
+#     """
+#     Format processed retrieval results into a context string for the LLM.
     
-    Args:
-        processed_results: List of processed objects with relevant fields
+#     Args:
+#         processed_results: List of processed objects with relevant fields
         
-    Returns:
-        Formatted context string
-    """
-    if not processed_results:
-        return ""
+#     Returns:
+#         Formatted context string
+#     """
+#     if not processed_results:
+#         return ""
     
-    context_parts = []
+#     context_parts = []
     
-    for i, result in enumerate(processed_results, 1):
-        doc_reference = f"[Document {i}: {result['filename']}"
-        if result['page'] != 'Unknown':
-            doc_reference += f", Page {result['page']}"
-        if result['year'] != 'Unknown':
-            doc_reference += f", Year {result['year']}"
-        doc_reference += "]"
+#     for i, result in enumerate(processed_results, 1):
+#         doc_reference = f"[Document {i}: {result['filename']}"
+#         if result['page'] != 'Unknown':
+#             doc_reference += f", Page {result['page']}"
+#         if result['year'] != 'Unknown':
+#             doc_reference += f", Year {result['year']}"
+#         doc_reference += "]"
         
-        context_part = f"{doc_reference}\n{result['answer']}\n"
-        context_parts.append(context_part)
+#         context_part = f"{doc_reference}\n{result['answer']}\n"
+#         context_parts.append(context_part)
     
-    return "\n".join(context_parts)
+#     return "\n".join(context_parts)
 
 # ---------------------------------------------------------------------
 # Core generation function for both Gradio UI and MCP
@@ -264,12 +264,12 @@ async def generate(query: str, context: Union[str, List[Dict[str, Any]]]) -> str
         if not context:
             return "Error: No retrieval results provided"
         
-        # Process the retrieval results
-        processed_results = extract_relevant_fields(context)
-        formatted_context = format_context_from_results(processed_results)
+        # # Process the retrieval results
+        # processed_results = extract_relevant_fields(context)
+           formatted_context = context
    
-        if not formatted_context.strip():
-            return "Error: No valid content found in retrieval results"
+        # if not formatted_context.strip():
+            # return "Error: No valid content found in retrieval results"
     
     elif isinstance(context, str):
         if not context.strip():

utils/retriever.py

@@ -19,7 +19,7 @@ def retrieve_paragraphs(query, category = None):
                 api_name="/retrieve"
         )
         return result
-        
+    
     except Exception as e:
         error_msg = f"Error retrieving paragraphs: {str(e)}"
         return (