code optimization, added sleep between call to groq

generator/compute_metrics.py

@@ -43,7 +43,7 @@ def get_metrics(attributes, total_sentences):
             result_json = json.loads(json_str)
             # Compute metrics using the extracted attributes
             metrics = compute_metrics(result_json, total_sentences)
-            print(metrics)
+            logging.info(metrics)
             return metrics        
         except json.JSONDecodeError as e:
             logging.error(f"JSONDecodeError: {e}")

generator/compute_rmse_auc_roc_metrics.py

@@ -3,7 +3,7 @@ from sklearn.metrics import roc_auc_score, root_mean_squared_error
 from generator.generate_metrics import generate_metrics
 import logging
 
-def compute_rmse_auc_roc_metrics(llm, dataset, vector_store, num_question):
+def compute_rmse_auc_roc_metrics(gen_llm, val_llm, dataset, vector_store, num_question):
 
      # Lists to accumulate ground truths and predictions for AUC-ROC computation
     all_ground_truth_relevance = []
@@ -25,7 +25,7 @@ def compute_rmse_auc_roc_metrics(llm, dataset, vector_store, num_question):
         query = document['question']
         logging.info(f'Query number: {i + 1}')
         # Call the generate_metrics for each query
-        metrics = generate_metrics(llm, vector_store, query)
+        metrics = generate_metrics(gen_llm, val_llm, vector_store, query)
         
         # Extract predicted metrics (ensure these are continuous if possible)
         predicted_relevance = metrics.get('Context Relevance', 0) if metrics else 0
@@ -46,22 +46,26 @@ def compute_rmse_auc_roc_metrics(llm, dataset, vector_store, num_question):
     
     # === Compute RMSE & AUC-ROC for the Entire Dataset ===
     try:
+        logging.info(f"All Ground Truth Relevance: {all_ground_truth_relevance}")
+        logging.info(f"All Predicted Relevance: {all_predicted_relevance}")
         relevance_rmse = root_mean_squared_error(all_ground_truth_relevance, all_predicted_relevance)
     except ValueError:
         relevance_rmse = None
 
     try:
+        logging.info(f"All Ground Truth Utilization: {all_ground_truth_utilization}")
+        logging.info(f"All Predicted Utilization: {all_predicted_utilization}")
         utilization_rmse = root_mean_squared_error(all_ground_truth_utilization, all_predicted_utilization)
     except ValueError:
         utilization_rmse = None
 
     try:
-        print(f"All Ground Truth Adherence: {all_ground_truth_utilization}")
-        print(f"All Predicted Utilization: {all_predicted_utilization}")
+        logging.info(f"All Ground Truth Adherence: {all_ground_truth_adherence}")
+        logging.info(f"All Predicted Adherence: {all_predicted_adherence}")
         adherence_auc = roc_auc_score(all_ground_truth_adherence, all_predicted_adherence)
     except ValueError:
         adherence_auc = None   
 
-    print(f"Relevance RMSE score: {relevance_rmse}")
-    print(f"Utilization RMSE score: {utilization_rmse}")
-    print(f"Overall Adherence AUC-ROC: {adherence_auc}")
+    logging.info(f"Relevance RMSE score: {relevance_rmse}")
+    logging.info(f"Utilization RMSE score: {utilization_rmse}")
+    logging.info(f"Overall Adherence AUC-ROC: {adherence_auc}")

generator/extract_attributes.py

@@ -1,9 +1,8 @@
 from generator.create_prompt import create_prompt
-from generator.initialize_llm import initialize_validation_llm
-from generator.document_utils import Document, apply_sentence_keys_documents, apply_sentence_keys_response
+from generator.document_utils import apply_sentence_keys_documents, apply_sentence_keys_response
 
 # Function to extract attributes
-def extract_attributes(question, relevant_docs, response):
+def extract_attributes(val_llm, question, relevant_docs, response):
     # Format documents into a string by accessing the `page_content` attribute of each Document
     #formatted_documents = "\n".join([f"Doc {i+1}: {doc.page_content}" for i, doc in enumerate(relevant_docs)])
     formatted_documents = apply_sentence_keys_documents(relevant_docs)
@@ -21,10 +20,7 @@ def extract_attributes(question, relevant_docs, response):
     
     attribute_prompt = create_prompt(formatted_documents, question, formatted_responses)
 
-    # Initialize the LLM
-    llm_val = initialize_validation_llm()
-    
     # Instead of using BaseMessage, pass the formatted prompt directly to invoke
-    result = llm_val.invoke(attribute_prompt)
+    result = val_llm.invoke(attribute_prompt)
 
     return result, total_sentences

generator/generate_metrics.py

@@ -1,29 +1,35 @@
 import logging
+import time
 from generator.generate_response import generate_response
 from retriever.retrieve_documents import retrieve_top_k_documents
 from generator.compute_metrics import get_metrics
 from generator.extract_attributes import extract_attributes
 
-def generate_metrics(llm, vector_store, query):
+def generate_metrics(gen_llm, val_llm, vector_store, query):
     logging.info(f'Query: {query}')
     
     # Step 1: Retrieve relevant documents for given query
     relevant_docs = retrieve_top_k_documents(vector_store, query, top_k=5)
-    logging.info(f"Relevant documents retrieved :{len(relevant_docs)}")
+    #logging.info(f"Relevant documents retrieved :{len(relevant_docs)}")
 
     # Log each retrieved document individually
     #for i, doc in enumerate(relevant_docs):
         #logging.info(f"Relevant document {i+1}: {doc} \n")
 
     # Step 2: Generate a response using LLM
-    response, source_docs = generate_response(llm, vector_store, query, relevant_docs)
+    response, source_docs = generate_response(gen_llm, vector_store, query, relevant_docs)
 
     logging.info(f"Response from LLM: {response}")
 
+    # Add a sleep interval to avoid hitting the rate limit
+    time.sleep(20)  # Adjust the sleep time as needed
+    
     # Step 3: Extract attributes and total sentences for each query
-    attributes, total_sentences = extract_attributes(query, source_docs, response)
+    logging.info(f"Extracting attributes through validation LLM")
+    attributes, total_sentences = extract_attributes(val_llm, query, source_docs, response)
+    logging.info(f"Extracted attributes successfully")
 
-    # Call the get_metrics
+    # Step 4 : Call the get metrics calculate metrics
     metrics = get_metrics(attributes, total_sentences)
 
     return metrics

generator/generate_response.py

@@ -7,7 +7,12 @@ def generate_response(llm, vector_store, question, relevant_docs):
         retriever=vector_store.as_retriever(),
         return_source_documents=True
     )
-    result = qa_chain.invoke(question, documents=relevant_docs)
-    response = result['result']
-    source_docs = result['source_documents']
-    return response, source_docs
+    try:
+        result = qa_chain.invoke(question, documents=relevant_docs)
+        response = result['result']
+        source_docs = result['source_documents']
+        return response, source_docs
+    except Exception as e:
+        print(f"Error during QA chain invocation: {e}")
+        raise e
+    

main.py

@@ -4,7 +4,8 @@ from generator.compute_rmse_auc_roc_metrics import compute_rmse_auc_roc_metrics
 from retriever.chunk_documents import chunk_documents
 from retriever.embed_documents import embed_documents
 from generator.generate_metrics import generate_metrics
-from generator.initialize_llm import initialize_llm
+from generator.initialize_llm import initialize_generation_llm
+from generator.initialize_llm import initialize_validation_llm
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -29,17 +30,20 @@ def main():
     logging.info("Documents embedded")
     
      # Initialize the Generation LLM
-    llm = initialize_llm()
+    gen_llm = initialize_generation_llm()
+
+    # Initialize the Validation LLM
+    val_llm = initialize_validation_llm()
 
     # Sample question
     row_num = 10
-    sample_question = dataset[row_num]['question']
+    query = dataset[row_num]['question']
 
     # Call generate_metrics for above sample question
-    generate_metrics(llm, vector_store, sample_question)
+    generate_metrics(gen_llm, val_llm, vector_store, query)
     
     #Compute RMSE and AUC-ROC for entire dataset
-    compute_rmse_auc_roc_metrics(llm, dataset, vector_store, 10)
+    compute_rmse_auc_roc_metrics(gen_llm, val_llm, dataset, vector_store, 10)
     
     logging.info("Finished!!!")