Spaces:
Running
Running
import logging | |
import re | |
import ast | |
from utils.models import get_llm_model | |
logger = logging.getLogger("misinformation_detector") | |
def extract_most_relevant_evidence(evidence_results): | |
""" | |
Intelligently extract the most relevant piece of evidence | |
Args: | |
evidence_results (list): List of evidence items | |
Returns: | |
str: Most relevant evidence piece | |
""" | |
if not evidence_results: | |
return None | |
# If evidence is a dictionary with 'evidence' key | |
if isinstance(evidence_results[0], dict): | |
# Sort by confidence if available | |
sorted_evidence = sorted( | |
evidence_results, | |
key=lambda x: x.get('confidence', 0), | |
reverse=True | |
) | |
# Return the evidence from the highest confidence item | |
for item in sorted_evidence: | |
evidence = item.get('evidence') | |
if evidence: | |
return evidence | |
# If plain list of evidence | |
return next((ev for ev in evidence_results if ev and isinstance(ev, str)), None) | |
def generate_explanation(claim, evidence_results, truth_label, confidence=None): | |
""" | |
Generate an explanation for the claim's classification | |
Args: | |
claim (str): The original claim | |
evidence_results (list/str): Evidence supporting the classification | |
truth_label (str): Classification of the claim | |
confidence (float): Confidence level (0-1) | |
Returns: | |
str: Explanation of the claim's classification | |
""" | |
logger.info(f"Generating explanation for claim with verdict: {truth_label}") | |
try: | |
# Normalize evidence_results to a list | |
if not isinstance(evidence_results, list): | |
try: | |
evidence_results = ast.literal_eval(str(evidence_results)) if evidence_results else [] | |
except: | |
evidence_results = [evidence_results] if evidence_results else [] | |
# Get the LLM model | |
explanation_model = get_llm_model() | |
# Extract most relevant evidence | |
most_relevant_evidence = extract_most_relevant_evidence(evidence_results) | |
# Prepare evidence text for prompt | |
evidence_text = "\n".join([ | |
f"Evidence {i+1}: {str(ev)[:200] + '...' if len(str(ev)) > 200 else str(ev)}" | |
for i, ev in enumerate(evidence_results[:3]) | |
]) | |
# Convert confidence to percentage and description | |
confidence_desc = "" | |
if confidence is not None: | |
confidence_pct = int(confidence * 100) | |
if confidence < 0.3: | |
confidence_desc = f"very low confidence ({confidence_pct}%)" | |
elif confidence < 0.5: | |
confidence_desc = f"low confidence ({confidence_pct}%)" | |
elif confidence < 0.7: | |
confidence_desc = f"moderate confidence ({confidence_pct}%)" | |
elif confidence < 0.9: | |
confidence_desc = f"high confidence ({confidence_pct}%)" | |
else: | |
confidence_desc = f"very high confidence ({confidence_pct}%)" | |
else: | |
# Determine confidence context from label if not explicitly provided | |
confidence_desc = ( | |
"high confidence" if "High Confidence" in truth_label else | |
"moderate confidence" if "Likely" in truth_label else | |
"low confidence" | |
) | |
# Create prompt with specific instructions based on the type of claim | |
has_negation = any(neg in claim.lower() for neg in ["not", "no longer", "isn't", "doesn't", "won't", "cannot"]) | |
# For claims with "True" verdict | |
if "True" in truth_label: | |
prompt = f""" | |
Claim: "{claim}" | |
Verdict: {truth_label} (with {confidence_desc}) | |
Available Evidence: | |
{evidence_text} | |
Task: Generate a clear explanation that: | |
1. Clearly states that the claim IS TRUE based on the evidence | |
2. {"Pay special attention to the logical relationship since the claim contains negation" if has_negation else "Explains why the evidence supports the claim"} | |
3. Uses confidence level of {confidence_desc} | |
4. Highlights the most relevant supporting evidence | |
5. Is factual and precise | |
""" | |
# For claims with "False" verdict | |
elif "False" in truth_label: | |
prompt = f""" | |
Claim: "{claim}" | |
Verdict: {truth_label} (with {confidence_desc}) | |
Available Evidence: | |
{evidence_text} | |
Task: Generate a clear explanation that: | |
1. Clearly states that the claim IS FALSE based on the evidence | |
2. {"Pay special attention to the logical relationship since the claim contains negation" if has_negation else "Explains why the evidence contradicts the claim"} | |
3. Uses confidence level of {confidence_desc} | |
4. Highlights the contradicting evidence | |
5. Is factual and precise | |
IMPORTANT: If the claim contains negation (words like 'not', 'no longer', etc.), be extra careful with the logical relationship between the evidence and the claim. | |
""" | |
# For uncertain claims | |
else: | |
prompt = f""" | |
Claim: "{claim}" | |
Verdict: {truth_label} (with {confidence_desc}) | |
Available Evidence: | |
{evidence_text} | |
Task: Generate a clear explanation that: | |
1. Clearly states that there is insufficient evidence to determine if the claim is true or false | |
2. Explains what information is missing or why the available evidence is insufficient | |
3. Uses confidence level of {confidence_desc} | |
4. Makes NO speculation about whether the claim might be true or false | |
5. Mentions that the user should seek information from other reliable sources | |
""" | |
# Generate explanation with multiple attempts | |
max_attempts = 3 | |
for attempt in range(max_attempts): | |
try: | |
# Invoke the model | |
response = explanation_model.invoke(prompt) | |
explanation = response.content.strip() | |
# Validate explanation length | |
if explanation and len(explanation.split()) >= 5: | |
return explanation | |
except Exception as attempt_error: | |
logger.error(f"Explanation generation attempt {attempt+1} failed: {str(attempt_error)}") | |
# Ultimate fallback explanation | |
if "Uncertain" in truth_label: | |
return f"The claim '{claim}' cannot be verified due to insufficient evidence. The available information does not provide clear support for or against this claim. Consider consulting reliable sources for verification." | |
elif "True" in truth_label: | |
return f"The claim '{claim}' is supported by the evidence with {confidence_desc}. {most_relevant_evidence or 'The evidence indicates this claim is accurate.'}" | |
else: | |
return f"The claim '{claim}' is contradicted by the evidence with {confidence_desc}. {most_relevant_evidence or 'The evidence indicates this claim is not accurate.'}" | |
except Exception as e: | |
logger.error(f"Comprehensive error in explanation generation: {str(e)}") | |
# Final fallback | |
return f"The claim is classified as {truth_label} based on the available evidence." |