Update appStore/tfidf_extraction.py

appStore/tfidf_extraction.py

@@ -1,19 +1,14 @@
 import re
 import langdetect
-from stopwordsiso import stopwords
+from stopwordsiso import stopwords, stopwords_json
 from sklearn.feature_extraction.text import TfidfVectorizer
 
-
 def detect_language(text):
-    """
-    Detect language using langdetect; returns a language code (e.g. 'en', 'de', 'es').
-    If detection fails or is uncertain, fallback to 'en'.
-    """
+    """Detect language using langdetect; returns a language code (e.g. 'en', 'de', 'es')."""
     try:
         return langdetect.detect(text)
     except:
-        return 'en'  # fallback
-
+        return 'en'  # fallback to English
 
 def get_stopwords_for_language(lang_code):
     """
@@ -21,36 +16,31 @@ def get_stopwords_for_language(lang_code):
     If not available, fallback to empty set.
     """
     lang_code = lang_code.lower()
-    if lang_code in stopwords.langdict:
-        return stopwords.lang(lang_code)
+    # stopwords_json is a dict of { 'en': [...], 'de': [...], ...}
+    if lang_code in stopwords_json:
+        # call stopwords(lang_code) to retrieve that language’s stopwords as a set
+        return stopwords(lang_code)
     else:
         return set()  # fallback to empty set
 
-
 def extract_top_keywords(text, top_n=5):
     """
-    Extract top_n keywords from 'text' using a simple TF-IDF approach with
-    language detection and language-specific stopwords.
+    Extract top_n keywords from 'text' using a simple TF-IDF approach
+    with language detection and language-specific stopwords.
     """
-    # Clean the text (remove punctuation, lower the case, etc.)
     cleaned_text = re.sub(r"[^\w\s]", " ", text.lower())
-
-    # Detect language
     lang_code = detect_language(cleaned_text)
-
-    # Get the relevant stopwords
+    
     language_stopwords = get_stopwords_for_language(lang_code)
 
-    # Initialize TF-IDF with the custom language stop words
+    # Pass your custom language_stopwords into TF-IDF:
     vectorizer = TfidfVectorizer(stop_words=language_stopwords)
-
-    # We pass in a list of one "document" to TF-IDF
     tfidf_matrix = vectorizer.fit_transform([cleaned_text])
 
     feature_names = vectorizer.get_feature_names_out()
-    scores = tfidf_matrix.toarray()[0]  # row 0 since we only have one doc
+    scores = tfidf_matrix.toarray()[0]
 
-    # Pair (word, score), then sort descending by score
+    # Pair (word, score), sort descending
     word_score_pairs = list(zip(feature_names, scores))
     word_score_pairs.sort(key=lambda x: x[1], reverse=True)