Calculation of context_relevance

generator/compute_metrics.py

@@ -1,11 +1,11 @@
-def compute_metrics(attributes):
+def compute_metrics(attributes, total_sentences):
     # Extract relevant information from attributes
     all_relevant_sentence_keys = attributes.get("all_relevant_sentence_keys", [])
     all_utilized_sentence_keys = attributes.get("all_utilized_sentence_keys", [])
     sentence_support_information = attributes.get("sentence_support_information", [])
-    
+
     # Compute Context Relevance
-    context_relevance = len(all_relevant_sentence_keys) / len(sentence_support_information) if sentence_support_information else 0
+    context_relevance = len(all_relevant_sentence_keys) / total_sentences if total_sentences else 0
     
     # Compute Context Utilization
     context_utilization = len(all_utilized_sentence_keys) / len(sentence_support_information) if sentence_support_information else 0

generator/extract_attributes.py

@@ -12,9 +12,12 @@ def extract_attributes(question, relevant_docs, response):
     formatted_documents = apply_sentence_keys_documents(relevant_docs)
     formatted_responses = apply_sentence_keys_response(response)
 
+    # Calculate the total number of sentences from formatted_documents
+    total_sentences = sum(len(doc) for doc in formatted_documents)
+
     attribute_prompt = create_prompt(formatted_documents, question, formatted_responses)
 
     # Instead of using BaseMessage, pass the formatted prompt directly to invoke
     result = llm.invoke(attribute_prompt)
 
-    return result
+    return result, total_sentences

main.py

@@ -46,11 +46,11 @@ def main():
     logging.info("Response generated")
 
     # Print the response
-    print(f"Response from LLM: {response}")
+    logging.info(f"Response from LLM: {response}")
     #print(f"Source Documents: {source_docs}")
 
     # Valuations : Extract attributes from the response and source documents
-    attributes = extract_attributes(sample_question, source_docs, response)
+    attributes, total_sentences = extract_attributes(sample_question, source_docs, response)
     
     # Only proceed if the content is not empty
     if attributes.content:
@@ -65,7 +65,7 @@ def main():
             print(json.dumps(result_json, indent=2))
 
             # Compute metrics using the extracted attributes
-            metrics = compute_metrics(result_json)
+            metrics = compute_metrics(result_json, total_sentences)
             print(metrics)
         except json.JSONDecodeError as e:
             logging.error(f"JSONDecodeError: {e}")