Update calendar_rag.py

calendar_rag.py

@@ -2,30 +2,123 @@ from haystack import *
 from haystack.components.generators.openai import OpenAIGenerator
 from haystack.components.builders import PromptBuilder
 from haystack.components.embedders import SentenceTransformersDocumentEmbedder
-from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever
+from haystack.components.retrievers.in_memory import *
 from haystack.document_stores.in_memory import InMemoryDocumentStore
 from haystack.utils import Secret
-from tenacity import retry, stop_after_attempt, wait_exponential
 from pathlib import Path
 import hashlib
 from datetime import *
 from typing import *
-import numpy as np
-from sklearn.metrics.pairwise import cosine_similarity
-from rouge_score import rouge_scorer
-import pandas as pd
 from dataclasses import *
 import json
 import logging
-import os
 import re
 import pickle
 
 # Setup logging
 logging.basicConfig(level=logging.INFO)
-
 logger = logging.getLogger(__name__)
 
+@dataclass
+class ValidationResult:
+    """Stores the result of a validation check"""
+    is_valid: bool
+    errors: List[str]
+    warnings: List[str]
+    normalized_data: Dict[str, str]
+
+@dataclass
+class ApplicationInfo:
+    application_portal: str
+    program_email: str
+
+@dataclass
+class RequiredDocument:
+    name: str
+    description: str
+    conditions: Optional[str] = None
+
+@dataclass
+class SelectionStep:
+    step_number: str
+    description: str
+
+@dataclass
+class ProgramDetailInfo:
+    application_info: ApplicationInfo
+    required_documents: Dict[str, Dict[str, RequiredDocument]]
+    submission_process: str
+    selection_process: List[SelectionStep]
+    
+@dataclass
+class Transportation:
+    boat: str
+    bts: str
+    mrt: str
+    airport_link: str
+    bus: Dict[str, str]
+
+@dataclass
+class Contact:
+    email: str
+    facebook: Dict[str, str]
+
+@dataclass
+class ContactDetail:
+    event_type: str
+    department: str
+    faculty: str
+    university: str
+    location: str
+    contact: Contact
+    transportation: Transportation
+
+@dataclass
+class Course:
+    code: str
+    title_th: str
+    title_en: str
+    credits: int
+
+@dataclass
+class CourseCategory:
+    description: Optional[str]
+    credits: Union[str, int]
+    minimum_credits: Optional[int]
+    courses: List[Course]
+
+@dataclass
+class CourseStructure:
+    event_type: str
+    program_name: str
+    department: str
+    total_credits: int
+    degree_level: str
+    structure: Dict[str, CourseCategory]
+
+@dataclass
+class StudyPlan:
+    event_type: str
+    years: Dict[str, Dict[str, Any]]
+
+@dataclass
+class RegularFee:
+    amount: float
+    currency: str
+    period: str
+
+@dataclass
+class LatePaymentFee:
+    amount: float
+    currency: str
+
+@dataclass
+class TuitionFee:
+    event_type: str
+    regular_fee: RegularFee
+    late_payment_fee: LatePaymentFee
+    
+    
 class OpenAIDateParser:
     """Uses OpenAI to parse complex Thai date formats"""
     
@@ -64,19 +157,14 @@ class OpenAIDateParser:
     async def parse_date(self, date_str: str) -> Dict[str, Union[str, bool]]:
         """Parse complex Thai date format using OpenAI"""
         try:
-            # Build prompt
             result = self.prompt_builder.run(date=date_str)
-            
-            # Get OpenAI response
             response = await self.generator.arun(prompt=result["prompt"])
             
             if not response or not response.get("replies"):
                 raise ValueError("Empty response from OpenAI")
             
-            # Parse JSON response
             parsed = json.loads(response["replies"][0])
             
-            # Validate the parsed dates
             for date_field in ['start_date', 'end_date']:
                 if parsed.get(date_field):
                     datetime.strptime(parsed[date_field], '%Y-%m-%d')
@@ -87,25 +175,10 @@ class OpenAIDateParser:
             logger.error(f"OpenAI date parsing failed for '{date_str}': {str(e)}")
             raise ValueError(f"Could not parse date: {date_str}")
 
-@dataclass
-class ValidationResult:
-    """Stores the result of a validation check"""
-    is_valid: bool
-    errors: List[str]
-    warnings: List[str]
-    normalized_data: Dict[str, str]
-
 class ThaiTextPreprocessor:
     """Handles Thai text preprocessing and normalization"""
     
-    # Thai character normalization mappings
-    CHAR_MAP = {
-        'ํา': 'ำ',  # Normalize sara am
-        '์': '',    # Remove yamakkan
-        '–': '-',   # Normalize dashes
-        '—': '-',
-        '٫': ',',   # Normalize separators
-    }
+    CHAR_MAP = {'ํา': 'ำ','์': '','–': '-','—': '-','٫': ',',}
     
     @classmethod
     def normalize_thai_text(cls, text: str) -> str:
@@ -113,14 +186,11 @@ class ThaiTextPreprocessor:
         if not text:
             return text
             
-        # Apply character mappings
         for old, new in cls.CHAR_MAP.items():
             text = text.replace(old, new)
         
-        # Normalize whitespace
         text = re.sub(r'\s+', ' ', text.strip())
         
-        # Normalize Thai numerals if present
         thai_digits = '๐๑๒๓๔๕๖๗๘๙'
         arabic_digits = '0123456789'
         
@@ -142,13 +212,11 @@ class CalendarEventValidator:
         warnings = []
         normalized_data = {}
         
-        # Validate and normalize date using OpenAI
         if event.date:
             try:
                 parsed_date = await self.date_parser.parse_date(event.date)
                 normalized_data['date'] = parsed_date['start_date']
                 
-                # If it's a date range, store it in the note
                 if parsed_date['is_range'] and parsed_date['end_date']:
                     range_note = f"ถึงวันที่ {parsed_date['end_date']}"
                     if event.note:
@@ -161,14 +229,12 @@ class CalendarEventValidator:
         else:
             errors.append("Date is required")
         
-        # Validate time format if provided
         if event.time:
             time_pattern = r'^([01]?[0-9]|2[0-3]):([0-5][0-9])$'
             if not re.match(time_pattern, event.time):
                 errors.append(f"Invalid time format: {event.time}")
             normalized_data['time'] = event.time
         
-        # Validate and normalize activity
         if event.activity:
             normalized_activity = self.preprocessor.normalize_thai_text(event.activity)
             if len(normalized_activity) < 3:
@@ -177,7 +243,6 @@ class CalendarEventValidator:
         else:
             errors.append("Activity is required")
         
-        # Validate semester
         valid_semesters = {'ภาคต้น', 'ภาคปลาย', 'ภาคฤดูร้อน'}
         if event.semester:
             normalized_semester = self.preprocessor.normalize_thai_text(event.semester)
@@ -187,17 +252,14 @@ class CalendarEventValidator:
         else:
             errors.append("Semester is required")
         
-        # Validate event type
         valid_types = {'registration', 'deadline', 'examination', 'academic', 'holiday'}
         if event.event_type not in valid_types:
             errors.append(f"Invalid event type: {event.event_type}")
         normalized_data['event_type'] = event.event_type
         
-        # Normalize note if present and not already set by date range
         if event.note and 'note' not in normalized_data:
             normalized_data['note'] = self.preprocessor.normalize_thai_text(event.note)
         
-        # Normalize section if present
         if event.section:
             normalized_data['section'] = self.preprocessor.normalize_thai_text(event.section)
         
@@ -208,11 +270,17 @@ class CalendarEventValidator:
             normalized_data=normalized_data
         )
 
-# Update CalendarEvent class to include async validation
 @dataclass
 class CalendarEvent:
     """Structured representation of a calendar event with validation"""
-
+    date: str
+    time: str
+    activity: str
+    note: str
+    semester: str
+    event_type: str
+    section: Optional[str] = None
+    
     @staticmethod
     def classify_event_type(activity: str) -> str:
         """Classify event type based on activity description"""
@@ -229,13 +297,6 @@ class CalendarEvent:
             if any(term in activity_lower for term in terms):
                 return event_type
         return 'academic'
-    date: str
-    time: str
-    activity: str
-    note: str
-    semester: str
-    event_type: str
-    section: Optional[str] = None
     
     async def initialize(self, openai_api_key: str):
         """Asynchronously validate and normalize the event"""
@@ -245,11 +306,9 @@ class CalendarEvent:
         if not result.is_valid:
             raise ValueError(f"Invalid calendar event: {', '.join(result.errors)}")
         
-        # Update with normalized data
         for field, value in result.normalized_data.items():
             setattr(self, field, value)
-        
-        # Log any warnings
+
         if result.warnings:
             logger.warning(f"Calendar event warnings: {', '.join(result.warnings)}")
     
@@ -259,22 +318,18 @@ class CalendarEvent:
         ภาคการศึกษา: {self.semester}
         ประเภท: {self.event_type}
         วันที่: {self.date}
-        เวลา: {self.time}
+        เวลา: {self.time or '-'}
         กิจกรรม: {self.activity}
         หมวดหมู่: {self.section or '-'}
-        หมายเหตุ: {self.note}
+        หมายเหตุ: {self.note or '-'}
         """.strip()
-
+    
 class CacheManager:
     """Manages caching for different components of the RAG pipeline"""
     
     def __init__(self, cache_dir: Path, ttl: int = 3600):
         """
         Initialize CacheManager
-        
-        Args:
-            cache_dir: Directory to store cache files
-            ttl: Time-to-live in seconds for cache entries (default: 1 hour)
         """
         self.cache_dir = cache_dir
         self.ttl = ttl
@@ -297,7 +352,6 @@ class CacheManager:
             try:
                 with open(cache_path, 'rb') as f:
                     cache = pickle.load(f)
-                # Clean expired entries
                 self._clean_expired_entries(cache)
                 return cache
             except Exception as e:
@@ -354,259 +408,435 @@ class CacheManager:
         self.query_cache[key] = (result, datetime.now())
         self._save_cache("queries", self.query_cache)
     
-    def get_document_cache(self, doc_id: str) -> Optional[Any]:
-        """Get cached document"""
-        if doc_id in self.document_cache:
-            doc, timestamp = self.document_cache[doc_id]
-            if datetime.now() - timestamp <= timedelta(seconds=self.ttl):
-                return doc
-        return None
-    
     def set_document_cache(self, doc_id: str, document: Any):
         """Cache document"""
         self.document_cache[doc_id] = (document, datetime.now())
         self._save_cache("documents", self.document_cache)
-    
-    def clear_cache(self, cache_type: Optional[str] = None):
-        """Clear specific or all caches"""
-        if cache_type == "embeddings":
-            self.embeddings_cache.clear()
-            self._save_cache("embeddings", self.embeddings_cache)
-        elif cache_type == "queries":
-            self.query_cache.clear()
-            self._save_cache("queries", self.query_cache)
-        elif cache_type == "documents":
-            self.document_cache.clear()
-            self._save_cache("documents", self.document_cache)
-        else:
-            self.embeddings_cache.clear()
-            self.query_cache.clear()
-            self.document_cache.clear()
-            for cache_type in ["embeddings", "queries", "documents"]:
-                self._save_cache(cache_type, {})
 
 @dataclass
 class ModelConfig:
-    """Configuration for language models and embeddings"""
     openai_api_key: str
     embedder_model: str = "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"
     openai_model: str = "gpt-4o"
     temperature: float = 0.7
-    max_tokens: int = 2000
-    top_p: float = 0.95
-    frequency_penalty: float = 0.0
-    presence_penalty: float = 0.0
 
 @dataclass
 class RetrieverConfig:
-    """Configuration for document retrieval"""
     top_k: int = 5
-    similarity_threshold: float = 0.7
-    reranking_enabled: bool = False
-    reranking_model: Optional[str] = None
-    filter_duplicates: bool = True
-    min_document_length: int = 10
 
 @dataclass
 class CacheConfig:
-    """Configuration for caching behavior"""
     enabled: bool = True
-    cache_dir: Path = field(default_factory=lambda: Path("./cache"))
-    embeddings_cache_ttl: int = 86400  # 24 hours
-    query_cache_ttl: int = 3600  # 1 hour
-    max_cache_size: int = 1000  # entries
-    cache_cleanup_interval: int = 3600  # 1 hour
+    cache_dir: Path = Path("./cache")
+    ttl: int = 86400  # 24 hours
 
 @dataclass
 class ProcessingConfig:
-    """Configuration for data processing"""
     batch_size: int = 32
-    max_retries: int = 3
-    timeout: int = 30
-    max_concurrent_requests: int = 5
-    chunk_size: int = 512
-    chunk_overlap: int = 50
-    preprocessing_workers: int = 4
-
-@dataclass
-class MonitoringConfig:
-    """Configuration for monitoring and logging"""
-    enable_monitoring: bool = True
-    log_level: str = "INFO"
-    metrics_enabled: bool = True
-    trace_enabled: bool = True
-    performance_logging: bool = True
-    slow_query_threshold: float = 5.0  # seconds
-    health_check_interval: int = 300  # 5 minutes
 
 @dataclass
 class LocalizationConfig:
-    """Configuration for Thai language handling"""
-    thai_tokenizer_model: str = "thai-tokenizer"
     enable_thai_normalization: bool = True
-    remove_thai_tones: bool = False
-    keep_english: bool = True
-    custom_stopwords: List[str] = field(default_factory=list)
-    custom_synonyms: Dict[str, List[str]] = field(default_factory=dict)
-          
+
 @dataclass
 class PipelineConfig:
-    """Main configuration for the RAG pipeline"""
-    # Model configurations
     model: ModelConfig
-    
-    # Retriever settings
     retriever: RetrieverConfig = field(default_factory=RetrieverConfig)
-    
-    # Cache settings
     cache: CacheConfig = field(default_factory=CacheConfig)
-    
-    # Processing settings
     processing: ProcessingConfig = field(default_factory=ProcessingConfig)
-    
-    # Monitoring settings
-    monitoring: MonitoringConfig = field(default_factory=MonitoringConfig)
-    
-    # Localization settings
     localization: LocalizationConfig = field(default_factory=LocalizationConfig)
-    
-    # Rate limiting
-    rate_limit_enabled: bool = True
-    requests_per_minute: int = 60
-    
-    # System settings
-    debug_mode: bool = False
-    development_mode: bool = False
-    
-    def __post_init__(self):
-        """Validate configuration and create necessary directories"""
-        if not self.model.openai_api_key:
-            raise ValueError("OpenAI API key is required")
-        
-        if self.cache.enabled:
-            self.cache.cache_dir.mkdir(parents=True, exist_ok=True)
-    
-    def to_dict(self) -> Dict[str, Any]:
-        """Convert configuration to dictionary format"""
-        return {
-            "model_config": {
-                "embedder_model": self.model.embedder_model,
-                "openai_model": self.model.openai_model,
-                "temperature": self.model.temperature,
-                # Add other relevant fields
-            },
-            "retriever_config": {
-                "top_k": self.retriever.top_k,
-                "similarity_threshold": self.retriever.similarity_threshold,
-                # Add other relevant fields
-            },
-            # Add other configuration sections
-        }
-    
-    @classmethod
-    def from_dict(cls, config_dict: Dict[str, Any]) -> 'PipelineConfig':
-        """Create configuration from dictionary"""
-        model_config = ModelConfig(**config_dict.get("model_config", {}))
-        retriever_config = RetrieverConfig(**config_dict.get("retriever_config", {}))
-        # Create other config objects
-        
-        return cls(
-            model=model_config,
-            retriever=retriever_config,
-            # Add other configuration objects
-        )
 
 def create_default_config(api_key: str) -> PipelineConfig:
-    """Create a default configuration with the given API key"""
-    model_config = ModelConfig(
-        openai_api_key=api_key,
-        embedder_model="sentence-transformers/paraphrase-multilingual-mpnet-base-v2"
-    )
-    return PipelineConfig(
-        model=model_config,
-        retriever=RetrieverConfig(),
-        cache=CacheConfig(),
-        processing=ProcessingConfig(),
-        monitoring=MonitoringConfig(),
-        localization=LocalizationConfig()
-    )
+    return PipelineConfig(model=ModelConfig(openai_api_key=api_key))
 
 class CalendarDataProcessor:
-    """Process and structure calendar data"""
+    """Process and structure calendar data from the new raw-data.json format"""
     
     @staticmethod
-    def parse_calendar_json(json_data: List[Dict]) -> List[CalendarEvent]:
+    def parse_calendar_json(json_data: Dict) -> List[CalendarEvent]:
+        """Parse the new calendar JSON format into CalendarEvent objects"""
         events = []
         
-        for semester_data in json_data:
-            semester = semester_data['education']
+        # Extract academic calendar data - handle direct dictionary input
+        calendar_data = json_data.get('academic_calendar', []) if isinstance(json_data, dict) else json_data
+        
+        for semester_block in calendar_data:
+            semester = semester_block.get('education', '')
+            schedule = semester_block.get('schedule', [])
             
-            # Process regular schedule events
-            for event in semester_data.get('schedule', []):
-                # Check if this is a regular event or a section with details
+            # Handle regular schedule events
+            for event in schedule:
                 if 'section' in event and 'details' in event:
-                    # This is a section with details
+                    # Process section-based events (thesis deadlines, etc.)
                     section = event['section']
                     for detail in event['details']:
-                        # Extract semester-specific information if available
                         if 'ภาคต้น' in detail and 'ภาคปลาย' in detail:
-                            # Handle both semesters
-                            semesters = ['ภาคต้น', 'ภาคปลาย']
-                            for sem in semesters:
-                                events.append(CalendarEvent(
-                                    date=detail.get(sem, ''),
-                                    time='',
-                                    activity=detail.get('title', ''),
-                                    note=section,
-                                    semester=sem,
-                                    event_type='deadline',
-                                    section=section
-                                ))
+                            # Handle dual-semester events
+                            for sem_key in ['ภาคต้น', 'ภาคปลาย']:
+                                if detail.get(sem_key):
+                                    events.append(CalendarEvent(
+                                        date=detail[sem_key],
+                                        time='',
+                                        activity=detail['title'],
+                                        note=section,
+                                        semester=sem_key,
+                                        event_type='deadline',
+                                        section=section
+                                    ))
                         else:
-                            # Single event
+                            # Single semester event
                             events.append(CalendarEvent(
                                 date=detail.get('date', ''),
                                 time='',
                                 activity=detail.get('title', ''),
                                 note=section,
-                                semester=semester,
+                                semester=ThaiTextPreprocessor.normalize_thai_text(semester),
                                 event_type='deadline',
                                 section=section
                             ))
                 else:
-                    # This is a regular event
+                    # Regular calendar event
                     event_type = CalendarEvent.classify_event_type(event.get('activity', ''))
+                    
+                    # Clean semester string
+                    cleaned_semester = semester
+                    if '(' in semester:
+                        match = re.search(r'\((.*?)\)', semester)
+                        if match:
+                            cleaned_semester = match.group(1)
+                    cleaned_semester = ThaiTextPreprocessor.normalize_thai_text(cleaned_semester)
+                    
                     events.append(CalendarEvent(
                         date=event.get('date', ''),
                         time=event.get('time', ''),
                         activity=event.get('activity', ''),
                         note=event.get('note', ''),
-                        semester=semester,
+                        semester=cleaned_semester,
                         event_type=event_type
                     ))
         
         return events
 
-# Update the EnhancedDocumentStore class to use caching
-class EnhancedDocumentStore:
-    """Enhanced document store with caching capabilities"""
+    @staticmethod
+    def extract_program_details(json_data: Dict) -> ProgramDetailInfo:
+        """Extract and structure program details into ProgramDetailInfo object"""
+        raw_details = json_data.get('program_details', {})
+        
+        # Process application info
+        app_info_data = raw_details.get('application_info', {})
+        app_info = ApplicationInfo(
+            application_portal=app_info_data.get('application_portal', ''),
+            program_email=app_info_data.get('program_email', '')
+        )
+        
+        # Process required documents
+        req_docs = {}
+        raw_docs = raw_details.get('required_documents', {})
+        
+        # Process mandatory documents
+        mandatory_docs = {}
+        for doc_key, doc_value in raw_docs.get('mandatory', {}).items():
+            mandatory_docs[doc_key] = RequiredDocument(
+                name=doc_key,
+                description=doc_value
+            )
+        req_docs['mandatory'] = mandatory_docs
+        
+        # Process optional documents
+        optional_docs = {}
+        for doc_key, doc_data in raw_docs.get('optional', {}).items():
+            if doc_key == 'english_proficiency':
+                ep_data = doc_data
+                optional_docs[doc_key] = RequiredDocument(
+                    name=ep_data.get('name', ''),
+                    description=str(ep_data.get('accepted_tests', {})),
+                    conditions=f"Validity: {ep_data.get('validity', '')}, Benefits: {ep_data.get('benefits', '')}, Exemptions: {ep_data.get('exemptions', '')}"
+                )
+            else:
+                optional_docs[doc_key] = RequiredDocument(
+                    name=doc_data.get('name', ''),
+                    description='',
+                    conditions=doc_data.get('condition', '')
+                )
+        req_docs['optional'] = optional_docs
+        
+        # Process selection steps
+        selection_steps = []
+        for step_data in raw_details.get('selection_process', {}).get('steps', []):
+            for step_num, description in step_data.items():
+                selection_steps.append(SelectionStep(
+                    step_number=step_num,
+                    description=description
+                ))
+        
+        return [ProgramDetailInfo(
+            application_info=app_info,
+            required_documents=req_docs,
+            submission_process=raw_details.get('submission_process', ''),
+            selection_process=selection_steps
+        )]
+
+    @staticmethod
+    def extract_contact_details(json_data: Dict) -> List[ContactDetail]:
+        """Extract and structure contact details into ContactDetail objects"""
+        raw_contacts = json_data.get('contact_details', [])
+        contact_details = []
+        
+        # Handle the case where raw_contacts might be a single object instead of a list
+        if not isinstance(raw_contacts, list):
+            raw_contacts = [raw_contacts]
+            
+        for contact_data in raw_contacts:
+            # Skip if contact_data is not a dictionary
+            if not isinstance(contact_data, dict):
+                continue
+                
+            try:
+                # Process transportation data
+                transportation_data = contact_data.get('transportation', {})
+                transportation = Transportation(
+                    boat=transportation_data.get('boat', ''),
+                    bts=transportation_data.get('bts', ''),
+                    mrt=transportation_data.get('mrt', ''),
+                    airport_link=transportation_data.get('airport_link', ''),
+                    bus=transportation_data.get('bus', {})
+                )
+                
+                # Process contact information
+                contact_info = Contact(
+                    email=contact_data.get('email', ''),
+                    facebook=contact_data.get('facebook', {})
+                )
+                
+                # Create ContactDetail object
+                contact_details.append(ContactDetail(
+                    event_type=contact_data.get('event_type', ''),
+                    department=contact_data.get('department', ''),
+                    faculty=contact_data.get('faculty', ''),
+                    university=contact_data.get('university', ''),
+                    location=contact_data.get('location', ''),
+                    contact=contact_info,
+                    transportation=transportation
+                ))
+            except Exception as e:
+                print(f"Error processing contact data: {e}")
+                continue
+        
+        return contact_details
+
+    
+    @staticmethod
+    def extract_course_structure(json_data: Dict) -> List[CourseStructure]:
+        """Extract and structure course information into CourseStructure objects"""
+        course_structures = []
+        
+        # Get course structure data
+        course_data = json_data.get('course_structure', {})
+        program_metadata = course_data.get('program_metadata', {})
+        curriculum = course_data.get('curriculum_structure', {})
+        
+        # Process foundation courses
+        foundation_data = curriculum.get('foundation_courses', {})
+        foundation_courses = []
+        for course in foundation_data.get('courses', []):
+            foundation_courses.append(Course(
+                code=course.get('code', ''),
+                title_th=course.get('title', {}).get('th', ''),
+                title_en=course.get('title', {}).get('en', ''),
+                credits=course.get('credits', 0)
+            ))
+        
+        # Process core courses
+        core_data = curriculum.get('core_courses', {})
+        core_courses = []
+        for course in core_data.get('modules', []):
+            core_courses.append(Course(
+                code=course.get('code', ''),
+                title_th=course.get('title', {}).get('th', ''),
+                title_en=course.get('title', {}).get('en', ''),
+                credits=course.get('credits', 0)
+            ))
+        
+        # Process elective courses
+        elective_data = curriculum.get('electives', {})
+        elective_courses = []
+        for course in elective_data.get('course_groups', []):
+            elective_courses.append(Course(
+                code=course.get('code', ''),
+                title_th=course.get('title', {}).get('th', ''),
+                title_en=course.get('title', {}).get('en', ''),
+                credits=course.get('credits', 0)
+            ))
+        
+        # Process research courses
+        research_data = curriculum.get('research', {})
+        research_courses = []
+        for course in research_data.get('course', []):
+            research_courses.append(Course(
+                code=course.get('code', ''),
+                title_th=course.get('title', {}).get('th', ''),
+                title_en=course.get('title', {}).get('en', ''),
+                credits=course.get('credits', 0)
+            ))
+        
+        # Create course categories
+        structure = {
+            'หมวดวิชาปรับพื้นฐาน': CourseCategory(  # Previously foundation_courses
+                description=foundation_data.get('metadata', {}).get('description'),
+                credits=foundation_data.get('metadata', {}).get('credits', 'non-credit'),
+                minimum_credits=None,
+                courses=foundation_courses
+            ),
+            'หมวดวิชาบังคับ': CourseCategory(  # Previously core_courses
+                description=None,
+                credits=0,
+                minimum_credits=core_data.get('minimum_requirement_credits'),
+                courses=core_courses
+            ),
+            'หมวดวิชาเลือก': CourseCategory(  # Previously elective_courses
+                description=None,
+                credits=0,
+                minimum_credits=elective_data.get('minimum_requirement_credits'),
+                courses=elective_courses
+            ),
+            'หมวดวิชาการค้นคว้าอิสระ': CourseCategory(  # Previously research_courses
+                description=None,
+                credits=0,
+                minimum_credits=research_data.get('minimum_requirement_credits'),
+                courses=research_courses
+            )
+        }
+        
+        # Create course structure
+        course_structure = CourseStructure(
+            event_type='curriculum_structure',
+            program_name=program_metadata.get('name', ''),
+            department=program_metadata.get('department', ''),
+            total_credits=program_metadata.get('total_credits', 0),
+            degree_level=program_metadata.get('degree_level', ''),
+            structure=structure
+        )
+        
+        return [course_structure]
+    
+    @staticmethod
+    def extract_program_study_plan(json_data: Dict) -> List[StudyPlan]:
+        """Extract and structure study plan information into StudyPlan objects"""
+        study_plan_data = json_data.get('program_study_plan', {})
+        
+        # Initialize the years dictionary to store all year/semester data
+        years_dict = {}
+        
+        for year_key, year_data in study_plan_data.items():
+            years_dict[year_key] = {}
+            
+            for semester_key, semester_data in year_data.items():
+                # Get metadata
+                metadata = semester_data.get('metadata', {})
+                
+                # Initialize semester structure
+                semester_struct = {
+                    'metadata': metadata,
+                    'courses': []
+                }
+                
+                # Handle both 'modules' and 'courses' keys
+                course_data = semester_data.get('modules', []) or semester_data.get('courses', [])
+                
+                # Add courses to semester
+                for course in course_data:
+                    course_info = {
+                        'code': course.get('code', ''),
+                        'title': course.get('title', {'th': '', 'en': ''}),
+                        'credits': course.get('credits', 0)
+                    }
+                    semester_struct['courses'].append(course_info)
+                
+                # Add semester data to year
+                years_dict[year_key][semester_key] = semester_struct
+        
+        # Create StudyPlan object
+        study_plan = StudyPlan(
+            event_type='study_plan',
+            years=years_dict
+        )
+        
+        return [study_plan]
+
+    @staticmethod
+    def extract_fees(json_data: Dict) -> List[TuitionFee]:
+        """Extract and structure fee information into TuitionFee objects"""
+        fees_data = json_data.get('fees', {})
+        
+        # Parse regular tuition fee
+        regular_fee_str = fees_data.get('tuition', '')
+        regular_amount = float(regular_fee_str.split()[0]) if regular_fee_str else 0
+        
+        regular_fee = RegularFee(
+            amount=regular_amount,
+            currency='THB',
+            period='per semester'
+        )
+        
+        # Parse late payment fee
+        late_fee_str = fees_data.get('late_payment', '')
+        late_amount = float(late_fee_str.split()[0]) if late_fee_str else 0
+        
+        late_payment_fee = LatePaymentFee(
+            amount=late_amount,
+            currency='THB'
+        )
+        
+        # Create TuitionFee object
+        tuition_fee = TuitionFee(
+            event_type='tuition_fee',
+            regular_fee=regular_fee,
+            late_payment_fee=late_payment_fee
+        )
+        
+        return [tuition_fee]
+
+class HybridDocumentStore:
+    """Enhanced document store with hybrid retrieval capabilities"""
     
     def __init__(self, config: PipelineConfig):
         self.store = InMemoryDocumentStore()
         self.embedder = SentenceTransformersDocumentEmbedder(
             model=config.model.embedder_model
         )
+        # Initialize BM25 retriever
+        self.bm25_retriever = InMemoryBM25Retriever(
+            document_store=self.store,
+            top_k=config.retriever.top_k
+        )
+        # Initialize embedding retriever
+        self.embedding_retriever = InMemoryEmbeddingRetriever(
+            document_store=self.store,
+            top_k=config.retriever.top_k
+        )
         self.cache_manager = CacheManager(
             cache_dir=config.cache.cache_dir,
-            ttl=config.cache.embeddings_cache_ttl
+            ttl=config.cache.ttl
         )
         
-        # Configure for Thai text
         self.embedder.warm_up()
         
+        # Initialize containers
         self.events = []
         self.event_type_index = {}
         self.semester_index = {}
+        self._document_counter = 0
+        
+        # Additional data containers
+        self.course_data = []
+        self.contact_data = []
+        self.study_plan_data = []
+
+    def _generate_unique_id(self) -> str:
+        """Generate a unique document ID"""
+        self._document_counter += 1
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        return f"doc_{timestamp}_{self._document_counter}"
     
     def _compute_embedding(self, text: str) -> Any:
         """Compute embedding with caching"""
@@ -618,89 +848,269 @@ class EnhancedDocumentStore:
         embedding = self.embedder.run(documents=[doc])["documents"][0].embedding
         self.cache_manager.set_embedding_cache(text, embedding)
         return embedding
-    
-    def add_events(self, events: List[CalendarEvent]):
-        """Add events with caching"""
-        documents = []
-        
-        for event in events:
-            # Store event
-            self.events.append(event)
-            event_idx = len(self.events) - 1
-            
-            # Update indices
-            if event.event_type not in self.event_type_index:
-                self.event_type_index[event.event_type] = []
-            self.event_type_index[event.event_type].append(event_idx)
-            
-            if event.semester not in self.semester_index:
-                self.semester_index[event.semester] = []
-            self.semester_index[event.semester].append(event_idx)
-            
-            # Create document with cached embedding
-            text = event.to_searchable_text()
+
+    def add_document(self, text: str, event_type: str):
+        """Add a single document to the store"""
+        try:
+            # Compute embedding
             embedding = self._compute_embedding(text)
             
+            # Create document with unique ID
             doc = Document(
+                id=self._generate_unique_id(),
                 content=text,
                 embedding=embedding,
-                meta={
-                    'event_type': event.event_type,
-                    'semester': event.semester,
-                    'date': event.date
-                }
+                meta={'event_type': event_type}
             )
-            documents.append(doc)
+            
+            # Write document
+            self.store.write_documents([doc])
             
             # Cache document
-            self.cache_manager.set_document_cache(str(event_idx), doc)
-        
-        # Store documents
-        self.store.write_documents(documents)
-    
-    def search(self, 
-              query: str, 
-              event_type: Optional[str] = None,
-              semester: Optional[str] = None,
-              top_k: int = 5) -> List[Document]:
-        """Search with query caching"""
-        # Check cache first
+            self.cache_manager.set_document_cache(doc.id, doc)
+            
+        except Exception as e:
+            logger.error(f"Error adding document: {str(e)}")
+            raise
+
+    def add_events(self, events: List[CalendarEvent], contact_details: Optional[List[ContactDetail]] = None,
+                  course_structure: Optional[List[CourseStructure]] = None,
+                  study_plans: Optional[List[StudyPlan]] = None):
+        """Add events and additional data with caching"""
+        documents = []
+        added_events = set()  # Track added events to prevent duplicates
+        
+        # Process calendar events
+        for event in events:
+            event_key = f"{event.date}_{event.activity}_{event.semester}"
+            if event_key not in added_events:
+                added_events.add(event_key)
+                self.events.append(event)
+                event_idx = len(self.events) - 1
+                
+                # Update indices
+                if event.event_type not in self.event_type_index:
+                    self.event_type_index[event.event_type] = []
+                self.event_type_index[event.event_type].append(event_idx)
+                
+                if event.semester not in self.semester_index:
+                    self.semester_index[event.semester] = []
+                self.semester_index[event.semester].append(event_idx)
+                
+                # Create document
+                text = event.to_searchable_text()
+                embedding = self._compute_embedding(text)
+                doc = Document(
+                    id=self._generate_unique_id(),
+                    content=text,
+                    embedding=embedding,
+                    meta={
+                        'event_type': event.event_type,
+                        'semester': event.semester,
+                        'date': event.date,
+                        'event_idx': event_idx
+                    }
+                )
+                documents.append(doc)
+                self.cache_manager.set_document_cache(str(event_idx), doc)
+        
+        # Process contact details
+        if contact_details:
+            for contact in contact_details:
+                self.contact_data.append(contact)
+                text = f"""
+                ข้อมูลการติดต่อ:
+                คณะ: {contact.faculty}
+                ภาควิชา: {contact.department}
+                มหาวิทยาลัย: {contact.university}
+                สถานที่: {contact.location}
+                
+                การติดต่อ:
+                อีเมล: {contact.contact.email}
+                Facebook: {json.dumps(contact.contact.facebook, ensure_ascii=False)}
+                
+                การเดินทาง:
+                เรือ: {contact.transportation.boat}
+                BTS: {contact.transportation.bts}
+                MRT: {contact.transportation.mrt}
+                Airport Link: {contact.transportation.airport_link}
+                รถประจำทาง: {json.dumps(contact.transportation.bus, ensure_ascii=False)}
+                """
+                embedding = self._compute_embedding(text)
+                doc = Document(
+                    id=self._generate_unique_id(),
+                    content=text,
+                    embedding=embedding,
+                    meta={'event_type': 'contact'}
+                )
+                documents.append(doc)
+        
+        # Process course structure
+        if course_structure:
+            for course in course_structure:
+                self.course_data.append(course)
+                text = f"""
+                โครงสร้างหลักสูตร:
+                ชื่อหลักสูตร: {course.program_name}
+                ภาควิชา: {course.department}
+                หน่วยกิตรวม: {course.total_credits}
+                ระดับการศึกษา: {course.degree_level}
+                
+                รายละเอียดโครงสร้าง:
+                """
+                for category_name, category in course.structure.items():
+                    text += f"\n{category_name}:\n"
+                    if category.description:
+                        text += f"คำอธิบาย: {category.description}\n"
+                    text += f"หน่วยกิต: {category.credits}\n"
+                    if category.minimum_credits:
+                        text += f"หน่วยกิตขั้นต่ำ: {category.minimum_credits}\n"
+                    text += "รายวิชา:\n"
+                    for course_item in category.courses:
+                        text += f"- {course_item.code}: {course_item.title_th} ({course_item.title_en}) - {course_item.credits} หน่วยกิต\n"
+                
+                embedding = self._compute_embedding(text)
+                doc = Document(
+                    id=self._generate_unique_id(),
+                    content=text,
+                    embedding=embedding,
+                    meta={'event_type': 'curriculum'}
+                )
+                documents.append(doc)
+        
+        # Process study plans
+        if study_plans:
+            for plan in study_plans:
+                self.study_plan_data.append(plan)
+                text = "แผนการศึกษา:\n"
+                for year, semesters in plan.years.items():
+                    text += f"\nปีที่ {year}:\n"
+                    for semester, data in semesters.items():
+                        text += f"\n{semester}:\n"
+                        if 'metadata' in data and data['metadata']:
+                            text += f"ข้อมูลเพิ่มเติม: {json.dumps(data['metadata'], ensure_ascii=False)}\n"
+                        if 'courses' in data:
+                            for course in data['courses']:
+                                text += f"- {course['code']}: {course['title'].get('th', '')} ({course['title'].get('en', '')}) - {course['credits']} หน่วยกิต\n"
+                
+                embedding = self._compute_embedding(text)
+                doc = Document(
+                    id=self._generate_unique_id(),
+                    content=text,
+                    embedding=embedding,
+                    meta={'event_type': 'study_plan'}
+                )
+                documents.append(doc)
+        
+        batch_size = 10
+        for i in range(0, len(documents), batch_size):
+            batch = documents[i:i + batch_size]
+            try:
+                self.store.write_documents(batch)
+            except Exception as e:
+                logger.error(f"Error writing document batch {i//batch_size + 1}: {str(e)}")
+                for doc in batch:
+                    try:
+                        self.store.write_documents([doc])
+                    except Exception as e2:
+                        logger.error(f"Failed to write document {doc.id}: {str(e2)}")
+    
+    def hybrid_search(self, 
+                     query: str, 
+                     event_type: Optional[str] = None,
+                     semester: Optional[str] = None,
+                     top_k: int = 10,
+                     weight_semantic: float = 0.5) -> List[Document]:
+        """Hybrid search combining semantic and lexical search results"""
+        
         cache_key = json.dumps({
             'query': query,
             'event_type': event_type,
             'semester': semester,
-            'top_k': top_k
+            'top_k': top_k,
+            'weight_semantic': weight_semantic
         })
+        
         cached_results = self.cache_manager.get_query_cache(cache_key)
         if cached_results is not None:
             return cached_results
-            
-        # Compute query embedding
+        
+        # Get semantic search results
         query_embedding = self._compute_embedding(query)
+        semantic_results = self.embedding_retriever.run(
+            query_embedding=query_embedding
+        )["documents"]
         
-        # Perform search
-        retriever = InMemoryEmbeddingRetriever(
-            document_store=self.store,
-            top_k=top_k * 2
-        )
+        # Get BM25 results
+        bm25_results = self.bm25_retriever.run(
+            query=query
+        )["documents"]
         
-        results = retriever.run(query_embedding=query_embedding)["documents"]
+        # Combine results using score fusion
+        combined_results = self._merge_results(
+            semantic_results=semantic_results,
+            bm25_results=bm25_results,
+            weight_semantic=weight_semantic,
+            top_k=top_k
+        )
         
-        # Filter results
+        # Filter results based on metadata
         filtered_results = []
-        for doc in results:
-            if event_type and doc.meta['event_type'] != event_type:
+        for doc in combined_results:
+            if event_type and doc.meta.get('event_type') != event_type:
                 continue
-            if semester and doc.meta['semester'] != semester:
+            if semester and doc.meta.get('semester') != semester:
                 continue
             filtered_results.append(doc)
         
         final_results = filtered_results[:top_k]
-        
-        # Cache results
         self.cache_manager.set_query_cache(cache_key, final_results)
         
         return final_results
+    
+    def _merge_results(self,
+                      semantic_results: List[Document],
+                      bm25_results: List[Document],
+                      weight_semantic: float,
+                      top_k: int) -> List[Document]:
+        """Merge semantic and BM25 results using weighted score fusion"""
+        
+        # Create dictionaries to store normalized scores
+        semantic_scores = {}
+        bm25_scores = {}
+        
+        # Normalize semantic scores
+        max_semantic_score = max(doc.score for doc in semantic_results) if semantic_results else 1.0
+        for doc in semantic_results:
+            semantic_scores[doc.id] = doc.score / max_semantic_score if max_semantic_score > 0 else 0
+        
+        # Normalize BM25 scores
+        max_bm25_score = max(doc.score for doc in bm25_results) if bm25_results else 1.0
+        for doc in bm25_results:
+            bm25_scores[doc.id] = doc.score / max_bm25_score if max_bm25_score > 0 else 0
+        
+        # Combine scores
+        combined_scores = {}
+        all_docs = {doc.id: doc for doc in semantic_results + bm25_results}
+        
+        for doc_id in all_docs:
+            semantic_score = semantic_scores.get(doc_id, 0)
+            bm25_score = bm25_scores.get(doc_id, 0)
+            
+            # Weighted combination
+            combined_scores[doc_id] = (
+                weight_semantic * semantic_score +
+                (1 - weight_semantic) * bm25_score
+            )
+        
+        # Sort by combined score and return top_k results
+        sorted_docs = sorted(
+            all_docs.values(),
+            key=lambda x: combined_scores[x.id],
+            reverse=True
+        )
+        
+        return sorted_docs[:top_k]
 
 class AdvancedQueryProcessor:
     """Process queries with better understanding"""
@@ -712,58 +1122,23 @@ class AdvancedQueryProcessor:
         )
         self.prompt_builder = PromptBuilder(
             template="""
-            Analyze this academic calendar query (in Thai):
-            Query: {{query}}
+            วิเคราะห์คำถามที่เกี่ยวข้องกับปฏิทินการศึกษา (ภาษาไทย):
+            คำถาม: {{query}}
             
-            Determine:
-            1. The type of information being requested
-            2. Any specific semester mentioned
-            3. Key terms to look for
+            ระบุ:
+            1. ประเภทของข้อมูลที่ต้องการค้นหา
+            2. ภาคการศึกษาที่ระบุไว้ (ถ้ามี)
+            3. คำสำคัญที่เกี่ยวข้อง
             
-            Return as JSON:
+            ให้ผลลัพธ์ในรูปแบบ JSON:
             {
-                "event_type": "registration|deadline|examination|academic|holiday",
-                "semester": "term mentioned or null",
-                "key_terms": ["up to 3 most important terms"],
-                "response_format": "list|single|detailed"
+                "event_type": "ลงทะเบียน|กำหนดเวลา|การสอบ|วิชาการ|วันหยุด",
+                "semester": "ภาคการศึกษาที่ระบุ หรือ null",
+                "key_terms": ["คำสำคัญ 3 คำที่สำคัญที่สุด"],
+                "response_format": "รายการ|คำตอบเดียว|คำตอบละเอียด"
             }
-            """)
-        
-    def process_query(self, query: str) -> Dict[str, Any]:
-        """Process and analyze query"""
-        try:
-            # Get analysis
-            result = self.prompt_builder.run(query=query)
-            response = self.generator.run(prompt=result["prompt"])
-            
-            # Add validation for empty response
-            if not response or not response.get("replies") or not response["replies"][0]:
-                logger.warning("Received empty response from generator")
-                return self._get_default_analysis(query)
-            
-            try:
-                # Parse response with error handling
-                analysis = json.loads(response["replies"][0])
-                
-                # Validate required fields
-                required_fields = ["event_type", "semester", "key_terms", "response_format"]
-                for field in required_fields:
-                    if field not in analysis:
-                        logger.warning(f"Missing required field: {field}")
-                        return self._get_default_analysis(query)
-                
-                return {
-                    "original_query": query,
-                    **analysis
-                }
-                
-            except json.JSONDecodeError as je:
-                logger.error(f"JSON parsing failed: {str(je)}")
-                return self._get_default_analysis(query)
-                
-        except Exception as e:
-            logger.error(f"Query processing failed: {str(e)}")
-            return self._get_default_analysis(query)
+            """
+        )
     
     def _get_default_analysis(self, query: str) -> Dict[str, Any]:
         """Return default analysis when processing fails"""
@@ -775,98 +1150,41 @@ class AdvancedQueryProcessor:
             "key_terms": [],
             "response_format": "detailed"
         }
-
-@dataclass
-class RateLimitConfig:
-    """Configuration for rate limiting"""
-    requests_per_minute: int = 60
-    max_retries: int = 3
-    base_delay: float = 1.0
-    max_delay: float = 60.0
-    timeout: float = 30.0
-    concurrent_requests: int = 5
-
-class APIError(Exception):
-    """Base class for API related errors"""
-    def __init__(self, message: str, status_code: Optional[int] = None, response: Optional[Dict] = None):
-        super().__init__(message)
-        self.status_code = status_code
-        self.response = response
-
-class RateLimitExceededError(APIError):
-    """Raised when rate limit is exceeded"""
-    pass
-
-class OpenAIRateLimiter:
-    """Rate limiter with advanced error handling for OpenAI API"""
-    
-    def __init__(self, config: RateLimitConfig):
-        self.config = config
-        self.requests = deque(maxlen=config.requests_per_minute)
-        self.semaphore = asyncio.Semaphore(config.concurrent_requests)
-        self.total_requests = 0
-        self.errors = deque(maxlen=1000)  # Store recent errors
-        self.start_time = datetime.now()
-    
-    async def acquire(self):
-        """Acquire permission to make a request"""
-        now = time.time()
-        
-        # Clean old requests
-        while self.requests and self.requests[0] < now - 60:
-            self.requests.popleft()
-        
-        # Check if we're at the limit
-        if len(self.requests) >= self.config.requests_per_minute:
-            wait_time = 60 - (now - self.requests[0])
-            logger.warning(f"Rate limit reached. Waiting {wait_time:.2f} seconds")
-            await asyncio.sleep(wait_time)
-        
-        # Add new request timestamp
-        self.requests.append(now)
-        self.total_requests += 1
-    
-    def get_usage_stats(self) -> Dict[str, Any]:
-        """Get current usage statistics"""
-        return {
-            "total_requests": self.total_requests,
-            "current_rpm": len(self.requests),
-            "uptime": (datetime.now() - self.start_time).total_seconds(),
-            "error_rate": len(self.errors) / self.total_requests if self.total_requests > 0 else 0
-        }
-    
-    @retry(
-        stop=stop_after_attempt(3),
-        wait=wait_exponential(multiplier=1, min=4, max=60),
-        reraise=True
-    )
-    async def execute_with_retry(self, func, *args, **kwargs):
-        """Execute API call with retry logic"""
+         
+    def process_query(self, query: str) -> Dict[str, Any]:
+        """Enhanced query processing with better error handling."""
+        
         try:
-            async with self.semaphore:
-                await self.acquire()
-                return await func(*args, **kwargs)
-                
-        except Exception as e:
-            error_info = {
-                "timestamp": datetime.now(),
-                "error_type": type(e).__name__,
-                "message": str(e)
+            result = self.prompt_builder.run(query=query)
+            response = self.generator.run(prompt=result["prompt"])
+
+            if not response or not response.get("replies") or not response["replies"][0]:
+                logger.warning("Received empty response from OpenAI")
+                return self._get_default_analysis(query)
+
+            try:
+                analysis = json.loads(response["replies"][0])
+            except json.JSONDecodeError as je:
+                return self._get_default_analysis(query)
+
+            # **Ensure course-related queries retrieve study plans & curricula**
+            course_keywords = ['หน่วยกิต', 'วิชา', 'หลักสูตร', 'แผนการเรียน', 'วิชาเลือก', 'วิชาบังคับ', 'วิชาการค้นคว้า', 'วิชาหลัก']
+            if any(keyword in query for keyword in course_keywords):
+                analysis['event_type'] = 'curriculum'
+
+            # **Ensure fee-related queries retrieve tuition fee documents**
+            fee_keywords = ['ค่าเทอม', 'ค่าธรรมเนียม', 'ค่าเรียน', 'ค่าปรับ']
+            if any(keyword in query for keyword in fee_keywords):
+                analysis['event_type'] = 'fees'
+
+            return {
+                "original_query": query,
+                **analysis
             }
-            self.errors.append(error_info)
-            
-            if isinstance(e, RateLimitExceededError):
-                logger.warning("Rate limit exceeded, backing off...")
-                await asyncio.sleep(self.config.base_delay)
-                raise
-                
-            elif "timeout" in str(e).lower():
-                logger.error(f"Timeout error: {str(e)}")
-                raise APIError(f"Request timed out after {self.config.timeout} seconds")
-                
-            else:
-                logger.error(f"API error: {str(e)}")
-                raise
+
+        except Exception as e:
+            logger.error(f"Query processing failed: {str(e)}")
+            return self._get_default_analysis(query)
              
 class ResponseGenerator:
     """Generate responses with better context utilization"""
@@ -878,27 +1196,24 @@ class ResponseGenerator:
         )
         self.prompt_builder = PromptBuilder(
             template="""
-            You are a helpful academic advisor. Answer the following query using the provided calendar information.
-            
-            Query: {{query}}
-            
-            Relevant Calendar Information:
+            คุณเป็นที่ปรึกษาทางวิชาการ กรุณาตอบคำถามต่อไปนี้โดยใช้ข้อมูลจากปฏิทินการศึกษาที่ให้มา
+
+            คำถาม: {{query}}
+
+            ข้อมูลที่เกี่ยวข้องจากปฏิทินการศึกษา:
             {% for doc in context %}
-            ---
+            --- 
             {{doc.content}}
             {% endfor %}
             
-            Format: {{format}}
-            
-            Guidelines:
-            1. Answer in Thai language
-            2. Be specific about dates and requirements
-            3. Include relevant notes or conditions
-            4. Format the response according to the specified format
-            
-            Provide your response:
-            """)
-        
+            **ห้ามเดาข้อมูลเอง ถ้าไม่มีข้อมูลให้ตอบว่า "ไม่มีข้อมูลที่ตรงกับคำถาม"**
+
+            กรุณาตอบเป็นภาษาไทย:
+
+            ต้องบอกเสมอว่า **หากมีข้อสงสัยเพิ่มเติมสามารถสอบถามได้**
+            """
+        )
+   
     def generate_response(self, 
                          query: str,
                          documents: List[Document],
@@ -919,34 +1234,164 @@ class ResponseGenerator:
             return "ขออภัย ไม่สามารถประมวลผลคำตอบได้ในขณะนี้"
 
 class AcademicCalendarRAG:
-    """Main RAG pipeline for academic calendar queries"""
+    """Enhanced RAG system for academic calendar and program information"""
     
     def __init__(self, config: PipelineConfig):
         self.config = config
-        self.document_store = EnhancedDocumentStore(config)
+        self.document_store = HybridDocumentStore(config)  # Use the new hybrid store
         self.query_processor = AdvancedQueryProcessor(config)
         self.response_generator = ResponseGenerator(config)
+        self.data_processor = CalendarDataProcessor()
+        
+        # Initialize data containers
+        self.calendar_events = []
+        self.program_details = []
+        self.contact_details = []
+        self.course_structure = []
+        self.study_plans = []
+        self.tuition_fees = []
+    
+    def load_data(self, json_data: Dict):
+        """Load and process all data sources"""
+        try:
+            raw_events = self.data_processor.parse_calendar_json(json_data)
+            for event in raw_events:
+                if not event.event_type:
+                    event.event_type = CalendarEvent.classify_event_type(event.activity)
+                self.calendar_events.append(event)
+            
+            # Process other data types
+            self.program_details = self.data_processor.extract_program_details(json_data)
+            self.contact_details = self.data_processor.extract_contact_details(json_data)
+            self.course_structure = self.data_processor.extract_course_structure(json_data)
+            self.study_plans = self.data_processor.extract_program_study_plan(json_data)
+            self.tuition_fees = self.data_processor.extract_fees(json_data)
+            
+            self._add_calendar_events()
+            self._add_program_info()
+            
+        except Exception as e:
+            logger.error(f"Error loading data: {str(e)}")
+            raise
+    
+    def _add_calendar_events(self):
+        """Add calendar events and other data to document store"""
+        if self.calendar_events:
+            self.document_store.add_events(
+                events=self.calendar_events,
+                contact_details=self.contact_details,
+                course_structure=self.course_structure,
+                study_plans=self.study_plans
+            )
+    
+    def _add_program_info(self):
+        """Enhanced method to add program-related information to document store"""
+        if self.program_details:
+            for detail in self.program_details:
+                text = f"""
+                ข้อมูลการสมัคร:
+                เว็บไซต์รับสมัคร: {detail.application_info.application_portal}
+                อีเมล: {detail.application_info.program_email}
+                
+                เอกสารที่ต้องใช้:
+                {self._format_required_docs(detail.required_documents)}
+                
+                ขั้นตอนการส่งเอกสาร:
+                {detail.submission_process}
+                
+                ขั้นตอนการคัดเลือก:
+                {self._format_selection_steps(detail.selection_process)}
+                """
+                self.document_store.add_document(text, "program_details")
         
-    def load_data(self, json_data: List[Dict]):
-        """Load and process calendar data"""
-        processor = CalendarDataProcessor()
-        events = processor.parse_calendar_json(json_data)
-        self.document_store.add_events(events)
+        if self.tuition_fees:
+            for fee in self.tuition_fees:
+                text = f"""
+                ค่าธรรมเนียมการศึกษา:
+                ค่าเล่าเรียนปกติ: {fee.regular_fee.amount:,.2f} {fee.regular_fee.currency} {fee.regular_fee.period}
+                ค่าปรับชำระล่าช้า: {fee.late_payment_fee.amount:,.2f} {fee.late_payment_fee.currency}
+                """
+                self.document_store.add_document(text, "fees")
+    
+    def _format_required_docs(self, docs: Dict) -> str:
+        """Format required documents information with detailed English proficiency requirements"""
+        result = []
         
-    def process_query(self, query: str) -> Dict[str, Any]:
-        """Process query and generate response"""
+        if 'mandatory' in docs:
+            result.append("เอกสารที่ต้องใช้:")
+            for doc in docs['mandatory'].values():
+                result.append(f"- {doc.name}: {doc.description}")
+        
+        if 'optional' in docs:
+            result.append("\nเอกสารเพิ่มเติม:")
+            for doc_key, doc in docs['optional'].items():
+                if doc_key == 'english_proficiency':
+                    result.append(f"- {doc.name}")
+                    # Parse and format the accepted tests
+                    try:
+                        accepted_tests = eval(doc.description)
+                        result.append("  เกณฑ์คะแนนที่ยอมรับ:")
+                        for test, requirement in accepted_tests.items():
+                            result.append(f"    * {test}: {requirement}")
+                    except:
+                        result.append(f"  {doc.description}")
+                    
+                    if doc.conditions:
+                        conditions = doc.conditions.split(', ')
+                        for condition in conditions:
+                            result.append(f"  {condition}")
+                else:
+                    desc = f"- {doc.name}"
+                    if doc.conditions:
+                        desc += f" ({doc.conditions})"
+                    result.append(desc)
+        
+        return "\n".join(result)
+    
+    def _format_selection_steps(self, steps: List[SelectionStep]) -> str:
+        """Format selection process steps"""
+        return "\n".join(f"{step.step_number}. {step.description}" for step in steps)
+    
+    def _get_fee_documents(self) -> List[Document]:
+        """Get fee-related documents"""
+        if not self.tuition_fees:
+            return []
+            
+        documents = []
+        for fee in self.tuition_fees:
+            text = f"""
+            ค่าธรรมเนียมการศึกษา:
+            - ค่าเล่าเรียน: {fee.regular_fee.amount:,.2f} {fee.regular_fee.currency} {fee.regular_fee.period}
+            - ค่าปรับชำระล่าช้า: {fee.late_payment_fee.amount:,.2f} {fee.late_payment_fee.currency}
+            """
+            doc = Document(
+                content=text,
+                meta={"event_type": "fees"}
+            )
+            documents.append(doc)
+        
+        return documents
+    
+    def process_query(self, query: str, weight_semantic: float = 0.5) -> Dict[str, Any]:
+        """Process user query using hybrid retrieval"""
         try:
             # Analyze query
             query_info = self.query_processor.process_query(query)
             
-            # Retrieve relevant documents
-            documents = self.document_store.search(
+            # Get relevant documents using hybrid search
+            documents = self.document_store.hybrid_search(
                 query=query,
-                event_type=query_info["event_type"],
-                semester=query_info["semester"],
-                top_k=self.config.retriever.top_k
+                event_type=query_info.get("event_type"),
+                semester=query_info.get("semester"),
+                top_k=self.config.retriever.top_k,
+                weight_semantic=weight_semantic
             )
             
+            # Add fee information for fee-related queries
+            if query_info.get("event_type") == "fees" and self.tuition_fees:
+                fee_docs = self._get_fee_documents()
+                documents.extend(fee_docs)
+            
             # Generate response
             response = self.response_generator.generate_response(
                 query=query,
@@ -955,65 +1400,50 @@ class AcademicCalendarRAG:
             )
             
             return {
+                "query": query,
                 "answer": response,
-                "documents": documents,
+                "relevant_docs": documents,
                 "query_info": query_info
             }
             
         except Exception as e:
-            logger.error(f"Query processing failed: {str(e)}")
+            logger.error(f"Error processing query: {str(e)}")
             return {
-                "answer": "ขออภัย ไม่สามารถประมวลผลคำถามได้ในขณะนี้",
-                "documents": [],
-                "query_info": {}
+                "query": query,
+                "answer": "ขออภัย ไม่สามารถประมวลผลคำตอบได้ในขณะนี้",
+                "error": str(e)
             }
-    
 # def main():
-#     """Main function for processing real calendar queries"""
+#     """Main function demonstrating hybrid retrieval"""
 #     try:
 #         # Load API key
 #         with open("key.txt", "r") as f:
 #             openai_api_key = f.read().strip()
         
-#         # Use create_default_config instead of direct PipelineConfig initialization
+#         # Create config with hybrid retrieval settings
 #         config = create_default_config(openai_api_key)
-    
-#         # Customize config for Thai academic calendar use case
 #         config.localization.enable_thai_normalization = True
-#         config.retriever.top_k = 5  # Adjust based on your needs
-#         config.model.temperature = 0.3  # Lower temperature for more focused responses
+#         config.retriever.top_k = 5
+#         config.model.temperature = 0.3
         
-#         # Initialize pipeline with enhanced config
 #         pipeline = AcademicCalendarRAG(config)
         
-#         # Load calendar data
-#         with open("calendar.json", "r", encoding="utf-8") as f:
-#             calendar_data = json.load(f)
-#         pipeline.load_data(calendar_data)
+#         # Load and process data
+#         with open("raw-data.json", "r", encoding="utf-8") as f:
+#             raw_data = json.load(f)
         
-#         # Real queries to process
-#         queries = ["นิสิตที่เข้าศึกษาในภาคเรียนที่ 1 ปีการศึกษา 2567 สามารถถอนรายวิชาได้หรือไม���? เพราะเหตุใด?"]
+#         pipeline.load_data(raw_data)
+        
+#         # Test queries with different semantic weights
+#         queries = ["เปิดเทอมวันเเรกวันไหน"]
         
-#         print("Processing calendar queries...")
 #         print("=" * 80)
         
 #         for query in queries:
-#             result = pipeline.process_query(query)
 #             print(f"\nQuery: {query}")
+#             result = pipeline.process_query(query, weight_semantic=0.3)
 #             print(f"Answer: {result['answer']}")
-            
-#             # # Print retrieved documents for verification
-#             # print("\nRetrieved Documents:")
-#             # for i, doc in enumerate(result['documents'], 1):
-#             #     print(f"\nDocument {i}:")
-#             #     print(doc.content)
-            
-#             # # Print query understanding info
-#             # print("\nQuery Understanding:")
-#             # for key, value in result['query_info'].items():
-#             #     print(f"{key}: {value}")
-                
-#             print("=" * 80)
+#             print("-" * 40)
             
 #     except Exception as e:
 #         logger.error(f"Pipeline execution failed: {str(e)}")