File size: 4,347 Bytes
f7c2fa3 b1b2c27 79dcf63 b1b2c27 79dcf63 b1b2c27 79dcf63 f7c2fa3 79dcf63 f7c2fa3 79dcf63 cfb3435 79dcf63 f7c2fa3 79dcf63 f7c2fa3 5184c29 f7c2fa3 9bde774 5184c29 f7c2fa3 5184c29 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
import json
import logging
def compute_metrics(attributes, total_sentences):
# Extract relevant information from attributes
all_relevant_sentence_keys = attributes.get("all_relevant_sentence_keys", [])
all_utilized_sentence_keys = attributes.get("all_utilized_sentence_keys", [])
sentence_support_information = attributes.get("sentence_support_information", [])
# Compute Context Relevance
context_relevance = len(all_relevant_sentence_keys) / total_sentences if total_sentences else 0
# Compute Context Utilization
context_utilization = len(all_utilized_sentence_keys) / total_sentences if total_sentences else 0
# Compute Completeness score
Ri = set(all_relevant_sentence_keys)
Ui = set(all_utilized_sentence_keys)
completeness_score = len(Ri & Ui) / len(Ri) if len(Ri) else 0
# Compute Adherence
adherence = all(info.get("fully_supported", False) for info in sentence_support_information)
#adherence = 1 if all(info.get("fully_supported", False) for info in sentence_support_information) else 0
return {
"Context Relevance": context_relevance,
"Context Utilization": context_utilization,
"Completeness Score": completeness_score,
"Adherence": adherence
}
def get_metrics(attributes, total_sentences):
if attributes.content:
try:
result_content = attributes.content # Access the content attribute
# Extract the JSON part from the result_content
json_start = result_content.find("{")
json_end = result_content.rfind("}") + 1
json_str = result_content[json_start:json_end]
result_json = json.loads(json_str)
# Compute metrics using the extracted attributes
metrics = compute_metrics(result_json, total_sentences)
logging.info(metrics)
return metrics
except json.JSONDecodeError as e:
logging.error(f"JSONDecodeError: {e}")
def get_attributes_text(attributes):
try:
result_content = attributes.content # Access the content attribute
# Extract the JSON part from the result_content
json_start = result_content.find("{")
json_end = result_content.rfind("}") + 1
json_str = result_content[json_start:json_end]
result_json = json.loads(json_str)
# Extract the required fields from json
relevance_explanation = result_json.get("relevance_explanation", "N/A")
all_relevant_sentence_keys = result_json.get("all_relevant_sentence_keys", [])
overall_supported_explanation = result_json.get("overall_supported_explanation", "N/A")
overall_supported = result_json.get("overall_supported", "N/A")
sentence_support_information = result_json.get("sentence_support_information", [])
all_utilized_sentence_keys = result_json.get("all_utilized_sentence_keys", [])
# Format the metrics for display
attributes_text = "Attributes:\n"
attributes_text = f"### Relevance Explanation:\n{relevance_explanation}\n\n"
attributes_text += f"### All Relevant Sentence Keys:\n{', '.join(all_relevant_sentence_keys)}\n\n"
attributes_text += f"### Overall Supported Explanation:\n{overall_supported_explanation}\n\n"
attributes_text += f"### Overall Supported:\n{overall_supported}\n\n"
attributes_text += "### Sentence Support Information:\n"
for info in sentence_support_information:
attributes_text += f"- Response Sentence Key: {info.get('response_sentence_key', 'N/A')}\n"
attributes_text += f" Explanation: {info.get('explanation', 'N/A')}\n"
attributes_text += f" Supporting Sentence Keys: {', '.join(info.get('supporting_sentence_keys', []))}\n"
attributes_text += f" Fully Supported: {info.get('fully_supported', 'N/A')}\n"
attributes_text += f"\n### All Utilized Sentence Keys:\n{', '.join(all_utilized_sentence_keys)}"
return attributes_text
except Exception as e:
logging.error(f"Error extracting attributes: {e}")
return f"An error occurred while extracting attributes: {e}" |