auto_tester.py

import json
import os
import datetime
from qa_loader import load_qa_and_create_vectorstore
from rag_chain import generate_response
from rapidfuzz import fuzz  # Benzerlik oranı hesaplamak için

# Log klasörünü hazırla
os.makedirs("logs", exist_ok=True)

# Zaman damgalı log dosyası
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
log_file = f"logs/auto_test_results_{timestamp}.txt"

# Vektör veritabanını yükle
retriever = load_qa_and_create_vectorstore()

# Q&A dosyasını oku
with open("Q&A_cleaned.json", "r", encoding="utf-8") as f:
    qa_data = json.load(f)

# Performans istatistikleri
total_questions = len(qa_data)
correct_answers = 0
incorrect_answers = 0

# Minimum kabul edilebilir benzerlik oranı
SIMILARITY_THRESHOLD = 60  # %60 eşleşme

# Log dosyasını aç ve başlık ekle
with open(log_file, "w", encoding="utf-8") as log:
    log.write(f"Auto Test Run - {timestamp}\n")
    log.write("=" * 80 + "\n")

    for idx, item in enumerate(qa_data, start=1):
        question = item['QUESTION']
        expected_answer = item['ANSWER']

        print(f"{idx}/{total_questions} Asking: {question}")
        ai_response = generate_response(retriever, question)

        # Benzerlik oranını hesapla
        similarity_score = fuzz.ratio(expected_answer.lower(), ai_response.lower())

        if similarity_score >= SIMILARITY_THRESHOLD:
            result = f"✅ Correct (Similarity: {similarity_score:.2f}%)"
            correct_answers += 1
        else:
            result = f"❌ Incorrect (Similarity: {similarity_score:.2f}%)"
            incorrect_answers += 1

        # Log'a yaz
        log.write(f"Question {idx}/{total_questions}:\n")
        log.write(f"Q: {question}\n")
        log.write(f"Expected Answer: {expected_answer}\n")
        log.write(f"AI Response: {ai_response}\n")
        log.write(f"Similarity: {similarity_score:.2f}%\n")
        log.write(f"Result: {result}\n")
        log.write("-" * 80 + "\n")

        print(f"🔎 {result} - Logged")

    # Test sonrası performans özeti
    accuracy = (correct_answers / total_questions) * 100

    log.write("\nTEST SUMMARY\n")
    log.write("=" * 80 + "\n")
    log.write(f"Total Questions: {total_questions}\n")
    log.write(f"Correct Answers: {correct_answers}\n")
    log.write(f"Incorrect Answers: {incorrect_answers}\n")
    log.write(f"Accuracy: {accuracy:.2f}%\n")
    log.write("=" * 80 + "\n")

# Sonuç özeti terminale yazdır
print("\n🔔 TEST COMPLETED")
print(f"✅ Correct: {correct_answers}")
print(f"❌ Incorrect: {incorrect_answers}")
print(f"📊 Accuracy: {accuracy:.2f}%")
print(f"📂 Detailed log saved to: {log_file}")