remove supabase parameter from get timestamp and fix evaluate stil asking for billing though

app.py

@@ -26,9 +26,9 @@ VECTOR_STORE_PREFIX = "vector_store"
 # ---------------------------------------------------------
 # ⚡️ UTILITY
 # ---------------------------------------------------------
-def get_latest_data_timestamp_from_files(bucket_name: str, supabase) -> float:
+def get_latest_data_timestamp_from_files(bucket_name: str) -> float:
     """Get the latest timestamp from files in a Supabase storage bucket."""
-    files = list_all_files(bucket_name, supabase=supabase)
+    files = list_all_files(bucket_name)
     latest_time = 0.0
     for file in files:
         iso_time = file.get("updated_at") or file.get("created_at")
@@ -65,7 +65,7 @@ def vector_store_is_outdated() -> bool:
     if supabase_timestamp is None:
         return True
     supabase_time = datetime.fromisoformat(supabase_timestamp.replace("Z", "+00:00")).timestamp()
-    data_time = get_latest_data_timestamp_from_files("pnp-bot-storage", supabase)
+    data_time = get_latest_data_timestamp_from_files("pnp-bot-storage")
 
     return data_time > supabase_time
 

evaluate.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import time
 import random
 import logging
@@ -21,28 +22,32 @@ from app.document_processor import load_vector_store_from_supabase
 from app.prompts import sahabat_prompt
 from app.db import supabase
 
-# Setup logging
+# === Logging UTF-8 Safe ===
+class UTF8StreamHandler(logging.StreamHandler):
+    def __init__(self, stream=None):
+        if stream is None:
+            stream = open(sys.stdout.fileno(), mode='w', encoding='utf-8', buffering=1)
+        super().__init__(stream)
+
+log_filename = f'rag_evaluation_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log'
 logging.basicConfig(
     level=logging.INFO,
     format='%(asctime)s - %(levelname)s - %(message)s',
     handlers=[
-        logging.FileHandler(f'rag_evaluation_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log'),
-        logging.StreamHandler()
+        logging.FileHandler(log_filename, encoding='utf-8'),
+        UTF8StreamHandler()
     ]
 )
 logger = logging.getLogger(__name__)
 
+# === Konfigurasi ===
 load_dotenv()
-
-# Konfigurasi
 BUCKET_NAME = "pnp-bot-storage-archive"
 VECTOR_STORE_PREFIX = "vector_store"
-
-# Rate limiting settings
 MAX_CALLS_PER_MINUTE = 50
 MAX_CALLS_PER_HOUR = 1000
 
-# Dataset evaluasi untuk Politeknik Negeri Padang
+# === Dataset evaluasi ===
 evaluation_dataset = [
     {
         'question': '''Bagaimana sistem pendidikan yang diterapkan di Politeknik Negeri Padang?''',
@@ -90,6 +95,7 @@ evaluation_dataset = [
     }
 ]
 
+
 # Schema untuk evaluasi
 class CorrectnessGrade(TypedDict):
     explanation: Annotated[str, ..., "Penjelasan alasan penilaian"]
@@ -170,7 +176,7 @@ Nilai relevansi False berarti FAKTA sama sekali tidak terkait dengan PERTANYAAN.
 Jelaskan penalaran Anda secara bertahap untuk memastikan penalaran dan kesimpulan benar.
 Hindari menyebutkan jawaban benar di awal."""
 
-# Inisialisasi evaluator LLM dengan retry dan rate limiting
+# === Evaluator ===
 class SafeLLMEvaluator:
     def __init__(self, model_name="gpt-4o", temperature=0):
         self.model_name = model_name
@@ -178,350 +184,131 @@ class SafeLLMEvaluator:
         self._init_llms()
     
     def _init_llms(self):
-        """Initialize LLM evaluators with structured output"""
-        try:
-            self.grader_llm = ChatOpenAI(
-                model=self.model_name, 
-                temperature=self.temperature
-            ).with_structured_output(CorrectnessGrade, method="json_schema", strict=True)
-            
-            self.relevance_llm = ChatOpenAI(
-                model=self.model_name, 
-                temperature=self.temperature
-            ).with_structured_output(RelevanceGrade, method="json_schema", strict=True)
-            
-            self.grounded_llm = ChatOpenAI(
-                model=self.model_name, 
-                temperature=self.temperature
-            ).with_structured_output(GroundedGrade, method="json_schema", strict=True)
-            
-            self.retrieval_relevance_llm = ChatOpenAI(
-                model=self.model_name, 
-                temperature=self.temperature
-            ).with_structured_output(RetrievalRelevanceGrade, method="json_schema", strict=True)
-            
-            logger.info(f"✅ LLM evaluators initialized with model: {self.model_name}")
-            
-        except Exception as e:
-            logger.error(f"❌ Failed to initialize LLM evaluators: {e}")
-            raise
+        self.grader_llm = ChatOpenAI(model=self.model_name, temperature=self.temperature).with_structured_output(CorrectnessGrade, method="json_schema", strict=True)
+        self.relevance_llm = ChatOpenAI(model=self.model_name, temperature=self.temperature).with_structured_output(RelevanceGrade, method="json_schema", strict=True)
+        self.grounded_llm = ChatOpenAI(model=self.model_name, temperature=self.temperature).with_structured_output(GroundedGrade, method="json_schema", strict=True)
+        self.retrieval_relevance_llm = ChatOpenAI(model=self.model_name, temperature=self.temperature).with_structured_output(RetrievalRelevanceGrade, method="json_schema", strict=True)
+        logger.info(f"✅ LLM evaluators initialized with model: {self.model_name}")
 
-# Global evaluator instance
 evaluator = SafeLLMEvaluator()
 
-# Rate limiting and retry decorators
+# === Rate Limiting & Retry ===
 @sleep_and_retry
 @limits(calls=MAX_CALLS_PER_MINUTE, period=60)
-@backoff.on_exception(
-    backoff.expo,
-    (Exception,),
-    max_tries=3,
-    max_time=30,
-    jitter=backoff.random_jitter
-)
+@backoff.on_exception(backoff.expo, (Exception,), max_tries=3)
 def safe_api_call(llm, messages):
-    """Safely make API calls with rate limiting and retry"""
-    try:
-        response = llm.invoke(messages)
-        logger.debug(f"✅ API call successful")
-        return response
-    except Exception as e:
-        logger.warning(f"⚠️ API call failed: {e}")
-        raise
+    return llm.invoke(messages)
 
+# === RAG Chain ===
 @traceable(name="Create RAG Chain for Evaluation")
 def create_rag_chain(vector_store):
-    """Membuat RAG chain untuk evaluasi dengan optimasi"""
-    try:
-        llm = Replicate(
-            model="fauziisyrinapridal/sahabat-ai-v1:afb9fa89fe786362f619fd4fef34bd1f7a4a4da23073d8a6fbf54dcbe458f216",
-            model_kwargs={"temperature": 0.1, "top_p": 0.9, "max_new_tokens": 4000}  # Reduced tokens
-        )
-        
-        memory = ConversationBufferMemory(
-            memory_key="chat_history",
-            return_messages=True,
-            output_key='answer'
-        )
-        
-        # Reduced retrieval count to minimize API calls
-        chain = ConversationalRetrievalChain.from_llm(
-            llm,
-            retriever=vector_store.as_retriever(search_kwargs={"k": 4}),  # Reduced from 6 to 4
-            combine_docs_chain_kwargs={"prompt": sahabat_prompt},
-            return_source_documents=True,
-            memory=memory
-        )
-        
-        logger.info("✅ RAG chain created successfully")
-        return chain
-        
-    except Exception as e:
-        logger.error(f"❌ Failed to create RAG chain: {e}")
-        raise
+    llm = Replicate(
+        model="fauziisyrinapridal/sahabat-ai-v1:afb9fa89fe786362f619fd4fef34bd1f7a4a4da23073d8a6fbf54dcbe458f216",
+        model_kwargs={"temperature": 0.1, "top_p": 0.9, "max_new_tokens": 4000},
+        replicate_api_token=os.getenv("REPLICATE_API_TOKEN"),
+
+    )
+    memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True, output_key='answer')
+    chain = ConversationalRetrievalChain.from_llm(
+        llm, retriever=vector_store.as_retriever(search_kwargs={"k": 4}),
+        combine_docs_chain_kwargs={"prompt": sahabat_prompt},
+        return_source_documents=True, memory=memory
+    )
+    return chain
 
 @traceable(name="RAG Bot Answer")
-@backoff.on_exception(backoff.expo, Exception, max_tries=3)
 def rag_bot_answer(question: str, vector_store) -> dict:
-    """Fungsi untuk mendapatkan jawaban dari RAG bot dengan error handling"""
-    try:
-        chain = create_rag_chain(vector_store)
-        result = chain({"question": question})
-        
-        logger.info(f"✅ RAG answer generated for question: {question[:50]}...")
-        return {
-            "answer": result['answer'],
-            "documents": result.get('source_documents', [])
-        }
-    except Exception as e:
-        logger.error(f"❌ Error in rag_bot_answer: {e}")
-        return {
-            "answer": "Terjadi kesalahan dalam memproses pertanyaan.", 
-            "documents": []
-        }
-
-# Enhanced evaluator functions with rate limiting
-def correctness_evaluator(question: str, answer: str, ground_truth: str) -> tuple[bool, str]:
-    """Evaluator untuk kebenaran jawaban dengan error handling"""
-    try:
-        answers = f"""PERTANYAAN: {question}
-JAWABAN BENAR: {ground_truth}
-JAWABAN SISWA: {answer}"""
-
-        messages = [
-            {"role": "system", "content": correctness_instructions},
-            {"role": "user", "content": answers}
-        ]
-        
-        grade = safe_api_call(evaluator.grader_llm, messages)
-        logger.debug(f"✅ Correctness evaluation completed")
-        return grade["correct"], grade["explanation"]
-        
-    except Exception as e:
-        logger.error(f"❌ Correctness evaluation failed: {e}")
-        return False, f"Error in evaluation: {str(e)}"
-
-def relevance_evaluator(question: str, answer: str) -> tuple[bool, str]:
-    """Evaluator untuk relevansi jawaban dengan error handling"""
-    try:
-        content = f"PERTANYAAN: {question}\nJAWABAN SISWA: {answer}"
-        messages = [
-            {"role": "system", "content": relevance_instructions},
-            {"role": "user", "content": content}
-        ]
-        
-        grade = safe_api_call(evaluator.relevance_llm, messages)
-        logger.debug(f"✅ Relevance evaluation completed")
-        return grade["relevant"], grade["explanation"]
-        
-    except Exception as e:
-        logger.error(f"❌ Relevance evaluation failed: {e}")
-        return False, f"Error in evaluation: {str(e)}"
-
-def groundedness_evaluator(answer: str, documents) -> tuple[bool, str]:
-    """Evaluator untuk groundedness jawaban dengan error handling"""
-    try:
-        if not documents:
-            return False, "No documents provided for grounding evaluation"
-        
-        doc_string = "\n\n".join([doc.page_content for doc in documents])
-        content = f"FAKTA: {doc_string}\nJAWABAN SISWA: {answer}"
-        messages = [
-            {"role": "system", "content": grounded_instructions},
-            {"role": "user", "content": content}
-        ]
-        
-        grade = safe_api_call(evaluator.grounded_llm, messages)
-        logger.debug(f"✅ Groundedness evaluation completed")
-        return grade["grounded"], grade["explanation"]
-        
-    except Exception as e:
-        logger.error(f"❌ Groundedness evaluation failed: {e}")
-        return False, f"Error in evaluation: {str(e)}"
-
-def retrieval_relevance_evaluator(question: str, documents) -> tuple[bool, str]:
-    """Evaluator untuk relevansi retrieval dengan error handling"""
-    try:
-        if not documents:
-            return False, "No documents provided for retrieval evaluation"
-        
-        doc_string = "\n\n".join([doc.page_content for doc in documents])
-        content = f"FAKTA: {doc_string}\nPERTANYAAN: {question}"
-        messages = [
-            {"role": "system", "content": retrieval_relevance_instructions},
-            {"role": "user", "content": content}
-        ]
-        
-        grade = safe_api_call(evaluator.retrieval_relevance_llm, messages)
-        logger.debug(f"✅ Retrieval relevance evaluation completed")
-        return grade["relevant"], grade["explanation"]
-        
-    except Exception as e:
-        logger.error(f"❌ Retrieval relevance evaluation failed: {e}")
-        return False, f"Error in evaluation: {str(e)}"
-
-def controlled_delay(min_delay=2, max_delay=5):
-    """Add controlled delay to avoid rate limits"""
-    delay = random.uniform(min_delay, max_delay)
-    logger.debug(f"⏳ Waiting {delay:.2f} seconds...")
-    time.sleep(delay)
-
+    chain = create_rag_chain(vector_store)
+    result = chain({"question": question})
+    return {"answer": result['answer'], "documents": result.get('source_documents', [])}
+
+# === Evaluator Functions ===
+def correctness_evaluator(question, answer, ground_truth):
+    messages = [{"role": "system", "content": correctness_instructions},
+                {"role": "user", "content": f"PERTANYAAN: {question}\nJAWABAN BENAR: {ground_truth}\nJAWABAN SISWA: {answer}"}]
+    grade = safe_api_call(evaluator.grader_llm, messages)
+    return grade["correct"], grade["explanation"]
+
+def relevance_evaluator(question, answer):
+    messages = [{"role": "system", "content": relevance_instructions},
+                {"role": "user", "content": f"PERTANYAAN: {question}\nJAWABAN SISWA: {answer}"}]
+    grade = safe_api_call(evaluator.relevance_llm, messages)
+    return grade["relevant"], grade["explanation"]
+
+def groundedness_evaluator(answer, documents):
+    doc_string = "\n\n".join([doc.page_content for doc in documents])
+    messages = [{"role": "system", "content": grounded_instructions},
+                {"role": "user", "content": f"FAKTA: {doc_string}\nJAWABAN SISWA: {answer}"}]
+    grade = safe_api_call(evaluator.grounded_llm, messages)
+    return grade["grounded"], grade["explanation"]
+
+def retrieval_relevance_evaluator(question, documents):
+    doc_string = "\n\n".join([doc.page_content for doc in documents])
+    messages = [{"role": "system", "content": retrieval_relevance_instructions},
+                {"role": "user", "content": f"FAKTA: {doc_string}\nPERTANYAAN: {question}"}]
+    grade = safe_api_call(evaluator.retrieval_relevance_llm, messages)
+    return grade["relevant"], grade["explanation"]
+
+# === Delay helper ===
+def controlled_delay(min_delay=1, max_delay=3):
+    time.sleep(random.uniform(min_delay, max_delay))
+
+# === Evaluation Runner ===
 @traceable(name="Run RAG Evaluation Enhanced")
-def run_enhanced_evaluation(batch_size: int = None, start_index: int = 0):
-    """Menjalankan evaluasi RAG dengan optimasi dan monitoring"""
-    logger.info("🚀 Memulai evaluasi RAG Enhanced untuk Politeknik Negeri Padang...")
-    
-    # Load vector store
-    logger.info("📚 Memuat vector store dari Supabase...")
-    try:
-        vector_store = load_vector_store_from_supabase(supabase, BUCKET_NAME, VECTOR_STORE_PREFIX)
-        if not vector_store:
-            logger.error("❌ Gagal memuat vector store!")
-            return None
-        logger.info("✅ Vector store berhasil dimuat!")
-    except Exception as e:
-        logger.error(f"❌ Error loading vector store: {e}")
-        return None
-    
-    # Determine evaluation scope
-    if batch_size:
-        end_index = min(start_index + batch_size, len(evaluation_dataset))
-        dataset_subset = evaluation_dataset[start_index:end_index]
-        logger.info(f"📊 Evaluating batch {start_index}-{end_index-1} ({len(dataset_subset)} questions)")
-    else:
-        dataset_subset = evaluation_dataset
-        logger.info(f"📊 Evaluating all {len(dataset_subset)} questions")
-    
-    # Hasil evaluasi
+def run_enhanced_evaluation():
+    logger.info("🚀 Starting evaluation...")
+    vector_store = load_vector_store_from_supabase(supabase, BUCKET_NAME, VECTOR_STORE_PREFIX)
     results = []
-    total_questions = len(dataset_subset)
-    start_time = time.time()
-    
-    # Progress tracking
-    success_count = 0
-    error_count = 0
-    
-    for i, item in enumerate(dataset_subset, 1):
-        question_start_time = time.time()
-        logger.info(f"\n🔍 Evaluasi pertanyaan {i}/{total_questions}")
-        
-        question = item['question']
-        ground_truth = item['ground_truth']
-        
+
+    for idx, item in enumerate(evaluation_dataset, 1):
+        question = item["question"]
+        ground_truth = item["ground_truth"]
+
         try:
-            # Dapatkan jawaban dari RAG
-            logger.info(f"🤖 Getting RAG answer...")
             rag_result = rag_bot_answer(question, vector_store)
-            answer = rag_result['answer']
-            documents = rag_result['documents']
-            
-            logger.info(f"❓ Pertanyaan: {question[:100]}...")
-            logger.info(f"💬 Jawaban: {answer[:100]}...")
-            logger.info(f"📄 Dokumen ditemukan: {len(documents)}")
-            
-            # Add delay before evaluations
-            controlled_delay(1, 3)
-            
-            # Evaluasi dengan error handling
-            logger.info("📈 Running evaluations...")
-            
-            correctness_score, correctness_explanation = correctness_evaluator(question, answer, ground_truth)
-            controlled_delay(1, 2)
-            
-            relevance_score, relevance_explanation = relevance_evaluator(question, answer)
-            controlled_delay(1, 2)
-            
-            groundedness_score, groundedness_explanation = groundedness_evaluator(answer, documents)
-            controlled_delay(1, 2)
-            
-            retrieval_relevance_score, retrieval_explanation = retrieval_relevance_evaluator(question, documents)
-            
-            result = {
-                'question_index': start_index + i,
-                'question': question,
-                'answer': answer,
-                'ground_truth': ground_truth,
-                'documents_count': len(documents),
-                'correctness': correctness_score,
-                'correctness_explanation': correctness_explanation,
-                'relevance': relevance_score,
-                'relevance_explanation': relevance_explanation,
-                'groundedness': groundedness_score,
-                'groundedness_explanation': groundedness_explanation,
-                'retrieval_relevance': retrieval_relevance_score,
-                'retrieval_explanation': retrieval_explanation,
-                'processing_time': time.time() - question_start_time
-            }
-            
-            results.append(result)
-            success_count += 1
-            
-            logger.info(f"📈 Skor - Benar: {correctness_score}, Relevan: {relevance_score}, "
-                       f"Berdasarkan Dokumen: {groundedness_score}, Retrieval Relevan: {retrieval_relevance_score}")
-            logger.info(f"⏱️  Waktu pemrosesan: {result['processing_time']:.2f} detik")
-            
+            answer = rag_result["answer"]
+            documents = rag_result["documents"]
+
+            correctness, correctness_exp = correctness_evaluator(question, answer, ground_truth)
+            relevance, relevance_exp = relevance_evaluator(question, answer)
+            grounded, grounded_exp = groundedness_evaluator(answer, documents)
+            retrieval, retrieval_exp = retrieval_relevance_evaluator(question, documents)
+
+            results.append({
+                "question": question,
+                "answer": answer,
+                "correctness": correctness,
+                "correctness_explanation": correctness_exp,
+                "relevance": relevance,
+                "relevance_explanation": relevance_exp,
+                "groundedness": grounded,
+                "groundedness_explanation": grounded_exp,
+                "retrieval_relevance": retrieval,
+                "retrieval_explanation": retrieval_exp,
+            })
+
+            logger.info(f"[{idx}] ✅ Done: {question[:50]}...")
+
         except Exception as e:
-            error_count += 1
-            logger.error(f"❌ Error processing question {i}: {e}")
-            
-            # Create error result
-            error_result = {
-                'question_index': start_index + i,
-                'question': question,
-                'answer': "ERROR",
-                'ground_truth': ground_truth,
-                'documents_count': 0,
-                'correctness': False,
-                'correctness_explanation': f"Error: {str(e)}",
-                'relevance': False,
-                'relevance_explanation': f"Error: {str(e)}",
-                'groundedness': False,
-                'groundedness_explanation': f"Error: {str(e)}",
-                'retrieval_relevance': False,
-                'retrieval_explanation': f"Error: {str(e)}",
-                'processing_time': time.time() - question_start_time
-            }
-            results.append(error_result)
-        
-        # Progress update
-        elapsed_time = time.time() - start_time
-        avg_time_per_question = elapsed_time / i
-        estimated_total_time = avg_time_per_question * total_questions
-        remaining_time = estimated_total_time - elapsed_time
-        
-        logger.info(f"📊 Progress: {i}/{total_questions} ({i/total_questions*100:.1f}%)")
-        logger.info(f"⏱️  Waktu berlalu: {elapsed_time:.1f}s, Estimasi sisa: {remaining_time:.1f}s")
-        
-        # Add delay between questions
-        if i < total_questions:
-            controlled_delay(2, 4)
-    
-    # Hitung statistik keseluruhan
-    total_time = time.time() - start_time
-    successful_results = [r for r in results if r['answer'] != "ERROR"]
-    
-    if successful_results:
-        total_correctness = sum(r['correctness'] for r in successful_results)
-        total_relevance = sum(r['relevance'] for r in successful_results)
-        total_groundedness = sum(r['groundedness'] for r in successful_results)
-        total_retrieval_relevance = sum(r['retrieval_relevance'] for r in successful_results)
-        successful_count = len(successful_results)
-    else:
-        total_correctness = total_relevance = total_groundedness = total_retrieval_relevance = 0
-        successful_count = 0
-    
-    # Print results
-    logger.info(f"\n📊 HASIL EVALUASI ENHANCED:")
-    logger.info(f"{'='*60}")
-    logger.info(f"Total Pertanyaan: {total_questions}")
-    logger.info(f"Berhasil Diproses: {success_count}")
-    logger.info(f"Error: {error_count}")
-    logger.info(f"Total Waktu: {total_time:.1f} detik ({total_time/60:.1f} menit)")
-    logger.info(f"Rata-rata per Pertanyaan: {total_time/total_questions:.1f} detik")
-    
-    if successful_count > 0:
-        logger.info(f"\n🎯 SKOR EVALUASI (dari {successful_count} pertanyaan berhasil):")
-        logger.info(f"Kebenaran (Correctness): {total_correctness}/{successful_count} ({total_correctness/successful_count*100:.1f}%)")
-        logger.info(f"Relevansi (Relevance): {total_relevance}/{successful_count} ({total_relevance/successful_count*100:.1f}%)")
-        logger.info(f"Berdasarkan Dokumen (Groundedness): {total_groundedness}/{successful_count} ({total_groundedness/successful_count*100:.1f}%)")
-        logger.info(f"Retrieval Relevan: {total_retrieval_relevance}/{successful_count} ({total_retrieval_relevance/successful_count*100:.1f}%)")
+            logger.error(f"❌ Error on Q{idx}: {e}")
+            results.append({
+                "question": question,
+                "answer": "ERROR",
+                "correctness": False,
+                "correctness_explanation": str(e),
+                "relevance": False,
+                "relevance_explanation": str(e),
+                "groundedness": False,
+                "groundedness_explanation": str(e),
+                "retrieval_relevance": False,
+                "retrieval_explanation": str(e),
+            })
+
+        controlled_delay()
+
+    logger.info("🎯 Evaluation finished")
+    return results
+
+# === Jalankan saat script dieksekusi langsung ===
+if __name__ == "__main__":
+    run_enhanced_evaluation()