Update app.py

app.py

@@ -20,6 +20,7 @@ import os
 import requests
 import time
 import tempfile
+from langdetect import detect
 
 API_KEY = os.environ.get("OPENROUTER_API_KEY")
 
@@ -82,18 +83,31 @@ qa_questions = list(qa_data.keys())
 qa_answers = list(qa_data.values())
 qa_embeddings = semantic_model.encode(qa_questions, convert_to_tensor=True)
 #-------------------------bm25---------------------------------
-from rank_bm25 import BM25Okapi
-from nltk.tokenize import word_tokenize
-import nltk
-nltk.download('punkt')
 
+def detect_language(text):
+    try:
+        lang = detect(text)
+        return 'french' if lang.startswith('fr') else 'english'
+    except:
+        return 'english'  # default fallback
+
+def clean_and_tokenize(text, lang):
+    tokens = word_tokenize(text.lower(), language=lang)
+    try:
+        stop_words = set(stopwords.words(lang))
+        return [t for t in tokens if t not in stop_words]
+    except:
+        return tokens  # fallback if stopwords not found
 
 def rerank_with_bm25(docs, query):
-    tokenized_docs = [word_tokenize(doc['content'].lower()) for doc in docs]
+    lang = detect_language(query)
+    
+    tokenized_docs = [clean_and_tokenize(doc['content'], lang) for doc in docs]
     bm25 = BM25Okapi(tokenized_docs)
-    tokenized_query = word_tokenize(query.lower())
-
+    
+    tokenized_query = clean_and_tokenize(query, lang)
     scores = bm25.get_scores(tokenized_query)
+    
     top_indices = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True)[:3]
     return [docs[i] for i in top_indices]