Update app.py

app.py

@@ -1,305 +1,227 @@
 import gradio as gr
-import re
-
-# Load language detection model only (smaller model)
-from transformers import pipeline
-language_detector = pipeline("text-classification", model="papluca/xlm-roberta-base-language-detection")
-
-# Omdurman National Bank-specific guidelines in Arabic
-ONB_GUIDELINES_AR = {
-    "balance": "يمكنك التحقق من رصيدك عبر الإنترنت أو عبر تطبيق الهاتف الخاص ببنك الوطني.",
-    "lost_card": "في حالة فقدان البطاقة، اتصل بالرقم 249-123-456-789 فورًا.",
-    "loan": "شروط القرض تشمل الحد الأدنى للدخل (5000 جنيه سوداني) وتاريخ ائتماني جيد.",
-    "transfer": "لتحويل الأموال، استخدم تطبيق الهاتف أو الخدمة المصرفية عبر الإنترنت.",
-    "new_account": "لفتح حساب جديد، قم بزيارة أقرب فرع مع جواز سفرك أو هويتك الوطنية.",
-    "interest_rates": "أسعار الفائدة على الودائع تتراوح بين 5% إلى 10% سنويًا.",
-    "branches": "فروعنا موجودة في أم درمان، الخرطوم، وبورتسودان. زيارة موقعنا للتفاصيل.",
-    "working_hours": "ساعات العمل من 8 صباحًا إلى 3 مساءً من الأحد إلى الخميس.",
-    "contact": "الاتصال بنا على الرقم 249-123-456-789 أو عبر البريد الإلكتروني [email protected]."
-}
-
-# Omdurman National Bank-specific guidelines in English
-ONB_GUIDELINES_EN = {
-    "balance": "You can check your balance online or via the ONB mobile app.",
-    "lost_card": "In case of a lost card, call 249-123-456-789 immediately.",
-    "loan": "Loan requirements include minimum income (5000 SDG) and good credit history.",
-    "transfer": "To transfer funds, use the mobile app or online banking service.",
-    "new_account": "To open a new account, visit your nearest branch with your passport or national ID.",
-    "interest_rates": "Interest rates on deposits range from 5% to 10% annually.",
-    "branches": "Our branches are located in Omdurman, Khartoum, and Port Sudan. Visit our website for details.",
-    "working_hours": "Working hours are from 8 AM to 3 PM, Sunday to Thursday.",
-    "contact": "Contact us at 249-123-456-789 or via email at [email protected]."
-}
-
-# Menu options in both languages
-MENU_AR = """
-قائمة الخدمات المصرفية:
-1. رصيد - استعلام عن رصيد حسابك
-2. بطاقة - الإبلاغ عن بطاقة مفقودة
-3. قرض - معلومات عن القروض
-4. تحويل - تحويل الأموال
-5. حساب - فتح حساب جديد
-6. فائدة - أسعار الفائدة
-7. فرع - مواقع الفروع
-8. ساعات - ساعات العمل
-9. اتصال - معلومات الاتصال
-"""
-
-MENU_EN = """
-Banking Services Menu:
-1. balance - Check your account balance
-2. card - Report a lost card
-3. loan - Information about loans
-4. transfer - Transfer funds
-5. account - Open a new account
-6. interest - Interest rates
-7. branch - Branch locations
-8. hours - Working hours
-9. contact - Contact information
-"""
-
-# Map intents to keywords (enhanced)
-INTENT_KEYWORDS = {
-    "balance": ["balance", "check balance", "account balance", "how much", "رصيد", "حساب", "كم المبلغ", "1"],
-    "lost_card": ["lost", "card", "stolen", "missing", "فقدت", "بطاقة", "مسروقة", "ضائعة", "2"],
-    "loan": ["loan", "borrow", "borrowing", "credit", "قرض", "استدانة", "إئتمان", "3"],
-    "transfer": ["transfer", "send money", "payment", "تحويل", "ارسال", "دفع", "4"],
-    "new_account": ["account", "open", "create", "new", "حساب", "فتح", "جديد", "إنشاء", "5"],
-    "interest_rates": ["interest", "rate", "rates", "return", "فائدة", "نسبة", "عائد", "6"],
-    "branches": ["branch", "location", "where", "office", "فرع", "موقع", "أين", "مكتب", "7"],
-    "working_hours": ["hours", "time", "open", "close", "ساعات", "وقت", "مفتوح", "مغلق", "8"],
-    "contact": ["contact", "phone", "email", "call", "اتصال", "هاتف", "بريد", "اتصل", "9"]
-}
-
-def detect_language(text):
-    # Use Hugging Face language detection model
-    result = language_detector(text)
-    language = result[0]['label']
-    return language
-
-def classify_intent(message: str):
-    # Check for menu request
-    menu_keywords = ["menu", "options", "help", "قائمة", "خيارات", "مساعدة"]
-    message_lower = message.lower()
-    
-    for keyword in menu_keywords:
-        if keyword in message_lower:
-            return "menu"
-    
-    # Use keyword matching for intent classification
-    for intent_key, keywords in INTENT_KEYWORDS.items():
-        for keyword in keywords:
-            if keyword.lower() in message_lower:
-                return intent_key
-                
-    return "unknown"
-
-def respond(message: str):
-    if not message.strip():
-        return {
-            "ar": "الرجاء كتابة سؤالك.",
-            "en": "Please type your question."
+import torch
+import faiss
+import numpy as np
+from sentence_transformers import SentenceTransformer
+from langdetect import detect
+from typing import List, Dict, Tuple, Any
+from datetime import datetime
+
+class MultilingualVectorChatbot:
+    def __init__(self, 
+                 embedding_model_name: str = 'paraphrase-multilingual-MiniLM-L12-v2',
+                 similarity_threshold: float = 0.7):
+        """
+        Initialize the multilingual chatbot with enhanced features
+        
+        :param embedding_model_name: Multilingual sentence embedding model
+        :param similarity_threshold: Minimum similarity score for valid responses
+        """
+        # Initialize models and databases
+        self.embedding_model = SentenceTransformer(embedding_model_name)
+        self.embedding_dimension = self.embedding_model.get_sentence_embedding_dimension()
+        self.index = faiss.IndexFlatL2(self.embedding_dimension)
+        self.knowledge_base = []
+        self.similarity_threshold = similarity_threshold
+        
+        # Language-specific configurations
+        self.FALLBACK_RESPONSES = {
+            'ar': "عذرًا، لا أملك إجابة محددة لهذا السؤال. هل يمكنك إعادة الصياغة؟",
+            'en': "I'm sorry, I don't have a specific answer. Could you rephrase?",
+            'fr': "Désolé, je n'ai pas de réponse spécifique. Pouvez-vous reformuler?",
+            'es': "Lo siento, no tengo una respuesta específica. ¿Podrías reformular?"
         }
-    
-    # Detect language
-    language = detect_language(message)
-    
-    # If the language is neither Arabic nor English, default to English
-    if language != "ar" and language != "en":
-        language = "en"
-    
-    # Classify the user's intent using keyword matching
-    intent = classify_intent(message)
-    
-    # Prepare responses in both languages
-    responses = {
-        "ar": "",
-        "en": ""
-    }
-    
-    # Special handling for menu request
-    if intent == "menu":
-        responses["ar"] = MENU_AR
-        responses["en"] = MENU_EN
-        return responses
-    
-    # If intent is recognized, return the corresponding response
-    if intent != "unknown":
-        responses["ar"] = ONB_GUIDELINES_AR.get(intent, "عذرًا، لم يتم التعرف على الخيار المحدد.")
-        responses["en"] = ONB_GUIDELINES_EN.get(intent, "Sorry, the selected option was not recognized.")
-    else:
-        # Default response if no intent is matched - show menu
-        responses["ar"] = "عذرًا، لم أفهم سؤالك. إليك قائمة بالخدمات المتاحة:" + MENU_AR
-        responses["en"] = "Sorry, I didn't understand your question. Here's a menu of available services:" + MENU_EN
-    
-    return responses
-
-# Custom CSS for better UI
-custom_css = """
-.gradio-container {
-    font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
-}
-
-.chat-message {
-    padding: 1rem;
-    border-radius: 10px;
-    margin-bottom: 1rem;
-    max-width: 80%;
-}
-
-.user-message {
-    background-color: #e6f7ff;
-    margin-left: auto;
-    text-align: right;
-}
-
-.bot-message {
-    background-color: #f0f0f0;
-    margin-right: auto;
-    text-align: left;
-}
-
-.bot-message-ar {
-    background-color: #f0f0f0;
-    margin-left: auto;
-    text-align: right;
-}
-
-.header-section {
-    background-color: #1a5276;
-    color: white;
-    padding: 1rem;
-    border-radius: 10px;
-    margin-bottom: 1rem;
-    text-align: center;
-}
-
-.footer-section {
-    font-size: 0.8rem;
-    text-align: center;
-    margin-top: 2rem;
-    color: #666;
-}
-
-.lang-selector {
-    text-align: right;
-    margin-bottom: 1rem;
-}
-
-.menu-button {
-    margin-top: 0.5rem;
-}
-"""
+        
+        # Conversation history
+        self.conversation_history = []
+        
+        # Preload knowledge
+        self._preload_knowledge()
+
+    def _preload_knowledge(self):
+        """Preload initial multilingual knowledge base"""
+        knowledge_pairs = [
+            # Arabic Knowledge
+            {
+                'questions': [
+                    "ما هي عاصمة مصر؟",
+                    "أين تقع القاهرة؟",
+                    "ما أهمية القاهرة؟"
+                ],
+                'answer': "القاهرة هي عاصمة جمهورية مصر العربية، وتقع على ضفاف نهر النيل. وهي أكبر مدن مصر وأهم مركز سياسي وثقافي واقتصادي في البلاد.",
+                'language': 'ar'
+            },
+            {
+                'questions': [
+                    "كيف يمكنني تعلم البرمجة؟",
+                    "ما هي أفضل طرق تعلم البرمجة؟"
+                ],
+                'answer': "يمكنك تعلم البرمجة من خ��ال عدة طرق: دورات مجانية عبر الإنترنت مثل Coursera وfreeCodeCamp، منصات التعلم التفاعلية مثل Codecademy، ومشاريع عملية على GitHub. ابدأ بتعلم لغة برمجة أساسية مثل Python، وركز على الممارسة العملية.",
+                'language': 'ar'
+            },
+            # English Knowledge
+            {
+                'questions': [
+                    "What is the capital of France?",
+                    "Where is Paris located?",
+                    "Tell me about Paris"
+                ],
+                'answer': "Paris is the capital of France, located in the north-central part of the country on the Seine River. It's known for its art, fashion, gastronomy and culture, and is home to landmarks like the Eiffel Tower and Louvre Museum.",
+                'language': 'en'
+            },
+            {
+                'questions': [
+                    "How can I learn programming?",
+                    "What are the best ways to learn coding?"
+                ],
+                'answer': "You can learn programming through various methods: free online courses like Coursera and freeCodeCamp, interactive learning platforms like Codecademy, and practical projects on GitHub. Start by learning a foundational programming language like Python, and focus on hands-on practice.",
+                'language': 'en'
+            }
+        ]
+        
+        # Batch add knowledge for better performance
+        all_questions = []
+        all_answers = []
+        all_languages = []
+        
+        for knowledge in knowledge_pairs:
+            all_questions.extend(knowledge['questions'])
+            all_answers.extend([knowledge['answer']] * len(knowledge['questions']))
+            all_languages.extend([knowledge['language']] * len(knowledge['questions']))
+        
+        self.add_knowledge_batch(all_questions, all_answers, all_languages)
 
-# Chat interface with enhanced UI
-with gr.Blocks(css=custom_css) as demo:
-    # Store conversation history
-    state = gr.State(value=[])
-    # Store selected language
-    selected_lang = gr.State(value="ar")
-    
-    with gr.Row(elem_classes="header-section"):
-        with gr.Column():
-            gr.Markdown("# Omdurman National Bank | بنك أم درمان الوطني")
-            gr.Markdown("### Virtual Banking Assistant | المساعد المصرفي الافتراضي")
-    
-    with gr.Row():
-        with gr.Column(elem_classes="lang-selector"):
-            language_btn = gr.Radio(
-                ["العربية", "English"], 
-                value="العربية", 
-                label="Language | اللغة"
-            )
-    
-    with gr.Row():
-        chat_box = gr.Chatbot(elem_id="chatbox", height=400)
-    
-    with gr.Row():
-        with gr.Column(scale=8):
-            text_input = gr.Textbox(
-                placeholder="Type your question here | اكتب سؤالك هنا", 
-                label="", 
-                elem_id="chat-input"
-            )
-        with gr.Column(scale=1):
-            submit_btn = gr.Button("Send | إرسال", variant="primary")
-    
-    with gr.Row():
-        with gr.Column(elem_classes="menu-button"):
-            menu_btn = gr.Button("Show Menu | إظهار القائمة")
-    
-    with gr.Row(elem_classes="footer-section"):
-        gr.Markdown("© 2025 Omdurman National Bank. All Rights Reserved. | جميع الحقوق محفوظة لبنك أم درمان الوطني ٢٠٢٥ ©")
-    
-    # Update language state when language is changed
-    def update_language(lang):
-        if lang == "العربية":
-            return "ar"
-        else:
-            return "en"
-    
-    language_btn.change(
-        fn=update_language,
-        inputs=language_btn,
-        outputs=selected_lang
-    )
-    
-    # Handle message submission
-    def on_submit(message, chat_history, lang):
-        if not message.strip():
-            return "", chat_history
+    def add_knowledge_batch(self, questions: List[str], answers: List[str], languages: List[str] = None):
+        """
+        Add knowledge in batch for better performance
         
-        # Add user message to chat history
-        chat_history.append([message, None])
+        :param questions: List of questions
+        :param answers: List of corresponding answers
+        :param languages: List of language codes
+        """
+        if len(questions) != len(answers):
+            raise ValueError("Questions and answers must have the same length")
+            
+        if languages and len(questions) != len(languages):
+            raise ValueError("Languages list must match questions length")
         
-        # Get response
-        responses = respond(message)
+        # Detect languages if not provided
+        if not languages:
+            languages = []
+            for q in questions:
+                try:
+                    languages.append(detect(q))
+                except:
+                    languages.append('en')  # default to English
         
-        # Select response based on language
-        response = responses[lang]
+        # Batch embed questions
+        question_embeddings = self.embedding_model.encode(questions)
         
-        # Update bot response in chat history
-        chat_history[-1][1] = response
+        # Add to FAISS index
+        if len(questions) > 0:
+            self.index.add(np.array(question_embeddings))
+            
+            # Store in knowledge base
+            for q, a, lang in zip(questions, answers, languages):
+                self.knowledge_base.append({
+                    'question': q,
+                    'answer': a,
+                    'language': lang
+                })
+
+    def find_similar_question(self, query: str, top_k: int = 3) -> List[Dict]:
+        """
+        Enhanced similarity search with confidence scores
         
-        return "", chat_history
-    
-    # Handle menu button click
-    def show_menu(chat_history, lang):
-        menu_responses = {
-            "ar": MENU_AR,
-            "en": MENU_EN
-        }
+        :param query: Input query
+        :param top_k: Number of results to return
+        :return: List of results with similarity scores
+        """
+        query_embedding = self.embedding_model.encode(query)
+        distances, indices = self.index.search(np.array([query_embedding]), top_k)
+        
+        results = []
+        for dist, idx in zip(distances[0], indices[0]):
+            if idx < len(self.knowledge_base):
+                similarity = 1 / (1 + dist)  # Convert distance to similarity
+                result = self.knowledge_base[idx].copy()
+                result.update({
+                    'similarity_score': similarity,
+                    'distance': dist
+                })
+                results.append(result)
+        
+        return sorted(results, key=lambda x: x['similarity_score'], reverse=True)
+
+    def generate_response(self, query: str, include_confidence: bool = False) -> str:
+        """
+        Generate response with confidence scoring and language detection
         
-        # Add system message showing the menu
-        menu_text = menu_responses[lang]
-        chat_history.append([None, menu_text])
+        :param query: User query
+        :param include_confidence: Whether to include confidence score
+        :return: Generated response
+        """
+        try:
+            # Detect language
+            lang = detect(query)
+            
+            # Find similar questions
+            similar_results = self.find_similar_question(query, top_k=1)
+            
+            # Prepare response
+            if similar_results and similar_results[0]['similarity_score'] >= self.similarity_threshold:
+                response = similar_results[0]['answer']
+                if include_confidence:
+                    confidence = similar_results[0]['similarity_score']
+                    if lang == 'ar':
+                        response += f"\n(ثقة الإجابة: {confidence:.2f})"
+                    else:
+                        response += f"\n(Answer confidence: {confidence:.2f})"
+            else:
+                response = self.FALLBACK_RESPONSES.get(lang, self.FALLBACK_RESPONSES['en'])
+            
+            # Update conversation history
+            self.conversation_history.append({
+                'query': query,
+                'response': response,
+                'language': lang,
+                'timestamp': str(datetime.now())
+            })
+            
+            return response
         
-        return chat_history
-    
-    # Link inputs and button to response function
-    submit_btn.click(
-        fn=on_submit,
-        inputs=[text_input, chat_box, selected_lang],
-        outputs=[text_input, chat_box]
-    )
-    
-    # Link menu button to show menu function
-    menu_btn.click(
-        fn=show_menu,
-        inputs=[chat_box, selected_lang],
-        outputs=[chat_box]
-    )
-    
-    # Also trigger on Enter key
-    text_input.submit(
-        fn=on_submit,
-        inputs=[text_input, chat_box, selected_lang],
-        outputs=[text_input, chat_box]
-    )
+        except Exception as e:
+            print(f"Error generating response: {str(e)}")
+            return self.FALLBACK_RESPONSES['en']
+
+# Initialize the chatbot
+chatbot = MultilingualVectorChatbot()
+
+def chat_interface(message: str, history: List[List[str]]) -> Tuple[str, Any]:
+    """
+    Gradio chat interface that properly handles state
+    """
+    try:
+        response = chatbot.generate_response(message, include_confidence=True)
+        return response
+    except Exception as e:
+        print(f"Interface error: {e}")
+        return "An error occurred. Please try again."
+
+# Create and launch the interface
+iface = gr.ChatInterface(
+    fn=chat_interface,
+    title="🌍 Multilingual Vector Chatbot",
+    description="Ask me questions in Arabic, English, French, or Spanish about various topics!",
+    theme="soft",
+    examples=[
+        ["What is the capital of France?"],
+        ["كيف أتعلم البرمجة؟"],
+        ["Comment ça va?"],
+        ["¿Dónde está el baño?"]
+    ],
+    cache_examples=True
+)
 
 if __name__ == "__main__":
-    demo.launch(
-        server_name="0.0.0.0",
-        server_port=7860,
-        share=True  # Enable public link
-    ) 
+    iface.launch()