Upload 2 files

generator/document_utils.py

@@ -1,35 +1,56 @@
-from typing import List
-
-class Document:
-    def __init__(self, metadata, page_content):
-        self.metadata = metadata
-        self.page_content = page_content
-
-def apply_sentence_keys_documents(relevant_docs: List[Document]):
-    result = []
-    '''for i, doc in enumerate(relevant_docs):
-        doc_id = str(i)
-        title_passage = doc.page_content.split('\nPassage: ')
-        title = title_passage[0]
-        passages = title_passage[1].split('. ')
-        
-        doc_result = []
-        doc_result.append([f"{doc_id}a", title])
-        
-        for j, passage in enumerate(passages):
-            doc_result.append([f"{doc_id}{chr(98 + j)}", passage])
-        
-        result.append(doc_result)'''
-    
-    for relevant_doc_index, relevant_doc in enumerate(relevant_docs):
-        sentences = []
-        for sentence_index, sentence in enumerate(relevant_doc.page_content.split(".")):
-            sentences.append([str(relevant_doc_index)+chr(97 + sentence_index), sentence])
-        result.append(sentences)
-    
-    return result
-
-def apply_sentence_keys_response(input_string):
-    sentences = input_string.split('. ')
-    result = [[chr(97 + i), sentence] for i, sentence in enumerate(sentences)]
-    return result
+import logging
+from typing import List
+
+logs = []
+class Document:
+    def __init__(self, metadata, page_content):
+        self.metadata = metadata
+        self.page_content = page_content
+
+def apply_sentence_keys_documents(relevant_docs: List[Document]):
+    result = []
+    '''for i, doc in enumerate(relevant_docs):
+        doc_id = str(i)
+        title_passage = doc.page_content.split('\nPassage: ')
+        title = title_passage[0]
+        passages = title_passage[1].split('. ')
+        
+        doc_result = []
+        doc_result.append([f"{doc_id}a", title])
+        
+        for j, passage in enumerate(passages):
+            doc_result.append([f"{doc_id}{chr(98 + j)}", passage])
+        
+        result.append(doc_result)'''
+    
+    for relevant_doc_index, relevant_doc in enumerate(relevant_docs):
+        sentences = []
+        for sentence_index, sentence in enumerate(relevant_doc.page_content.split(".")):
+            sentences.append([str(relevant_doc_index)+chr(97 + sentence_index), sentence])
+        result.append(sentences)
+    
+    return result
+
+def apply_sentence_keys_response(input_string):
+    sentences = input_string.split('. ')
+    result = [[chr(97 + i), sentence] for i, sentence in enumerate(sentences)]
+    return result
+
+def initialize_logging():
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+
+    # Custom log handler to capture logs and add them to the logs list
+    class LogHandler(logging.Handler):
+        def emit(self, record):
+            log_entry = self.format(record)
+            logs.append(log_entry)
+
+    # Add custom log handler to the logger
+    log_handler = LogHandler()
+    log_handler.setFormatter(logging.Formatter('%(asctime)s - %(message)s'))
+    logger.addHandler(log_handler)
+
+def get_logs():
+        """Retrieve logs for display."""
+        return "\n".join(logs[-100:])  # Only show the last 50 logs for example

generator/generate_metrics.py

@@ -1,39 +1,39 @@
-import logging
-import time
-from generator.generate_response import generate_response
-from retriever.retrieve_documents import retrieve_top_k_documents
-from generator.compute_metrics import get_metrics
-from generator.extract_attributes import extract_attributes
-
-def retrieve_and_generate_response(gen_llm, vector_store, query):
-    logging.info(f'Query: {query}')
-    
-    # Step 1: Retrieve relevant documents for given query
-    relevant_docs = retrieve_top_k_documents(vector_store, query, top_k=5)
-    #logging.info(f"Relevant documents retrieved :{len(relevant_docs)}")
-
-    # Log each retrieved document individually
-    #for i, doc in enumerate(relevant_docs):
-        #logging.info(f"Relevant document {i+1}: {doc} \n")
-
-    # Step 2: Generate a response using LLM
-    response, source_docs = generate_response(gen_llm, vector_store, query, relevant_docs)
-
-    logging.info(f"Response from LLM: {response}")
-
-    return response, source_docs
-
-def generate_metrics(val_llm, response, source_docs, query, time_to_wait):
-
-    # Add a sleep interval to avoid hitting the rate limit
-    time.sleep(time_to_wait)  # Adjust the sleep time as needed
-    
-    # Step 3: Extract attributes and total sentences for each query
-    logging.info(f"Extracting attributes through validation LLM")
-    attributes, total_sentences = extract_attributes(val_llm, query, source_docs, response)
-    logging.info(f"Extracted attributes successfully")
-    
-    # Step 4 : Call the get metrics calculate metrics
-    metrics = get_metrics(attributes, total_sentences)
-
+import logging
+import time
+from generator.generate_response import generate_response
+from retriever.retrieve_documents import retrieve_top_k_documents
+from generator.compute_metrics import get_metrics
+from generator.extract_attributes import extract_attributes
+
+def retrieve_and_generate_response(gen_llm, vector_store, query):
+    logging.info(f'Query: {query}')
+    
+    # Step 1: Retrieve relevant documents for given query
+    relevant_docs = retrieve_top_k_documents(vector_store, query, top_k=5)
+    #logging.info(f"Relevant documents retrieved :{len(relevant_docs)}")
+
+    # Log each retrieved document individually
+    #for i, doc in enumerate(relevant_docs):
+        #logging.info(f"Relevant document {i+1}: {doc} \n")
+
+    # Step 2: Generate a response using LLM
+    response, source_docs = generate_response(gen_llm, vector_store, query, relevant_docs)
+
+    logging.info(f"Response from LLM ({gen_llm.name}): {response}")
+
+    return response, source_docs
+
+def generate_metrics(val_llm, response, source_docs, query, time_to_wait):
+
+    # Add a sleep interval to avoid hitting the rate limit
+    time.sleep(time_to_wait)  # Adjust the sleep time as needed
+    
+    # Step 3: Extract attributes and total sentences for each query
+    logging.info(f"Extracting attributes through validation LLM")
+    attributes, total_sentences = extract_attributes(val_llm, query, source_docs, response)
+    logging.info(f"Extracted attributes successfully")
+    
+    # Step 4 : Call the get metrics calculate metrics
+    metrics = get_metrics(attributes, total_sentences)
+
     return attributes, metrics