Update appStore/tfidf_extraction.py

appStore/tfidf_extraction.py

@@ -3,33 +3,41 @@ import langdetect
 from stopwordsiso import stopwords, has_lang
 from sklearn.feature_extraction.text import TfidfVectorizer
 
-def detect_language(text: str) -> str:
+def detect_language(text):
     try:
         return langdetect.detect(text)
     except:
-        return "en"  # fallback if detection fails
+        return "en"
 
-def get_stopwords_for_language(lang_code: str):
+def get_stopwords_for_language(lang_code):
     lang_code = lang_code.lower()
     if has_lang(lang_code):
-        return stopwords(lang_code)  # returns a *set* of stopwords
-    else:
-        return set()
+        return stopwords(lang_code)  # returns a set of stopwords
+    return set()
 
-def extract_top_keywords(text: str, top_n: int = 5) -> list[str]:
+def extract_top_keywords(text, top_n=5):
+    # Basic cleanup
     cleaned_text = re.sub(r"[^\w\s]", " ", text.lower())
-
     lang_code = detect_language(cleaned_text)
     language_stopwords = get_stopwords_for_language(lang_code)
 
-    # Convert the set to a list here!
-    vectorizer = TfidfVectorizer(stop_words=list(language_stopwords))
+    # Convert stopwords set to list because TfidfVectorizer needs list/None/'english'
+    stopwords_list = list(language_stopwords)
+
+    vectorizer = TfidfVectorizer(stop_words=stopwords_list)
+
+    try:
+        tfidf_matrix = vectorizer.fit_transform([cleaned_text])
+    except ValueError as e:
+        # If there's nothing left after removing stopwords/punctuation
+        if "empty vocabulary" in str(e).lower():
+            return []  # Return an empty list -> no keywords
+        else:
+            raise e  # Something else went wrong
 
-    tfidf_matrix = vectorizer.fit_transform([cleaned_text])
     feature_names = vectorizer.get_feature_names_out()
     scores = tfidf_matrix.toarray()[0]
 
-    # Pair up (word, score), then sort descending
     word_score_pairs = list(zip(feature_names, scores))
     word_score_pairs.sort(key=lambda x: x[1], reverse=True)