Spaces:
Running
Running
File size: 7,560 Bytes
5dc3509 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 |
import logging
import re
import ast
from utils.models import get_llm_model
logger = logging.getLogger("misinformation_detector")
def extract_most_relevant_evidence(evidence_results):
"""
Intelligently extract the most relevant piece of evidence
Args:
evidence_results (list): List of evidence items
Returns:
str: Most relevant evidence piece
"""
if not evidence_results:
return None
# If evidence is a dictionary with 'evidence' key
if isinstance(evidence_results[0], dict):
# Sort by confidence if available
sorted_evidence = sorted(
evidence_results,
key=lambda x: x.get('confidence', 0),
reverse=True
)
# Return the evidence from the highest confidence item
for item in sorted_evidence:
evidence = item.get('evidence')
if evidence:
return evidence
# If plain list of evidence
return next((ev for ev in evidence_results if ev and isinstance(ev, str)), None)
def generate_explanation(claim, evidence_results, truth_label, confidence=None):
"""
Generate an explanation for the claim's classification
Args:
claim (str): The original claim
evidence_results (list/str): Evidence supporting the classification
truth_label (str): Classification of the claim
confidence (float): Confidence level (0-1)
Returns:
str: Explanation of the claim's classification
"""
logger.info(f"Generating explanation for claim with verdict: {truth_label}")
try:
# Normalize evidence_results to a list
if not isinstance(evidence_results, list):
try:
evidence_results = ast.literal_eval(str(evidence_results)) if evidence_results else []
except:
evidence_results = [evidence_results] if evidence_results else []
# Get the LLM model
explanation_model = get_llm_model()
# Extract most relevant evidence
most_relevant_evidence = extract_most_relevant_evidence(evidence_results)
# Prepare evidence text for prompt
evidence_text = "\n".join([
f"Evidence {i+1}: {str(ev)[:200] + '...' if len(str(ev)) > 200 else str(ev)}"
for i, ev in enumerate(evidence_results[:3])
])
# Convert confidence to percentage and description
confidence_desc = ""
if confidence is not None:
confidence_pct = int(confidence * 100)
if confidence < 0.3:
confidence_desc = f"very low confidence ({confidence_pct}%)"
elif confidence < 0.5:
confidence_desc = f"low confidence ({confidence_pct}%)"
elif confidence < 0.7:
confidence_desc = f"moderate confidence ({confidence_pct}%)"
elif confidence < 0.9:
confidence_desc = f"high confidence ({confidence_pct}%)"
else:
confidence_desc = f"very high confidence ({confidence_pct}%)"
else:
# Determine confidence context from label if not explicitly provided
confidence_desc = (
"high confidence" if "High Confidence" in truth_label else
"moderate confidence" if "Likely" in truth_label else
"low confidence"
)
# Create prompt with specific instructions based on the type of claim
has_negation = any(neg in claim.lower() for neg in ["not", "no longer", "isn't", "doesn't", "won't", "cannot"])
# For claims with "True" verdict
if "True" in truth_label:
prompt = f"""
Claim: "{claim}"
Verdict: {truth_label} (with {confidence_desc})
Available Evidence:
{evidence_text}
Task: Generate a clear explanation that:
1. Clearly states that the claim IS TRUE based on the evidence
2. {"Pay special attention to the logical relationship since the claim contains negation" if has_negation else "Explains why the evidence supports the claim"}
3. Uses confidence level of {confidence_desc}
4. Highlights the most relevant supporting evidence
5. Is factual and precise
"""
# For claims with "False" verdict
elif "False" in truth_label:
prompt = f"""
Claim: "{claim}"
Verdict: {truth_label} (with {confidence_desc})
Available Evidence:
{evidence_text}
Task: Generate a clear explanation that:
1. Clearly states that the claim IS FALSE based on the evidence
2. {"Pay special attention to the logical relationship since the claim contains negation" if has_negation else "Explains why the evidence contradicts the claim"}
3. Uses confidence level of {confidence_desc}
4. Highlights the contradicting evidence
5. Is factual and precise
IMPORTANT: If the claim contains negation (words like 'not', 'no longer', etc.), be extra careful with the logical relationship between the evidence and the claim.
"""
# For uncertain claims
else:
prompt = f"""
Claim: "{claim}"
Verdict: {truth_label} (with {confidence_desc})
Available Evidence:
{evidence_text}
Task: Generate a clear explanation that:
1. Clearly states that there is insufficient evidence to determine if the claim is true or false
2. Explains what information is missing or why the available evidence is insufficient
3. Uses confidence level of {confidence_desc}
4. Makes NO speculation about whether the claim might be true or false
5. Mentions that the user should seek information from other reliable sources
"""
# Generate explanation with multiple attempts
max_attempts = 3
for attempt in range(max_attempts):
try:
# Invoke the model
response = explanation_model.invoke(prompt)
explanation = response.content.strip()
# Validate explanation length
if explanation and len(explanation.split()) >= 5:
return explanation
except Exception as attempt_error:
logger.error(f"Explanation generation attempt {attempt+1} failed: {str(attempt_error)}")
# Ultimate fallback explanation
if "Uncertain" in truth_label:
return f"The claim '{claim}' cannot be verified due to insufficient evidence. The available information does not provide clear support for or against this claim. Consider consulting reliable sources for verification."
elif "True" in truth_label:
return f"The claim '{claim}' is supported by the evidence with {confidence_desc}. {most_relevant_evidence or 'The evidence indicates this claim is accurate.'}"
else:
return f"The claim '{claim}' is contradicted by the evidence with {confidence_desc}. {most_relevant_evidence or 'The evidence indicates this claim is not accurate.'}"
except Exception as e:
logger.error(f"Comprehensive error in explanation generation: {str(e)}")
# Final fallback
return f"The claim is classified as {truth_label} based on the available evidence." |