appStore/tfidf_extraction.py

import re
import langdetect
from stopwordsiso import stopwords, has_lang
from sklearn.feature_extraction.text import TfidfVectorizer

def detect_language(text):
    try:
        return langdetect.detect(text)
    except:
        return "en"

def get_stopwords_for_language(lang_code):
    lang_code = lang_code.lower()
    if has_lang(lang_code):
        return stopwords(lang_code)  # returns a set of stopwords
    return set()

def extract_top_keywords(text, top_n=5):
    # Basic cleanup
    cleaned_text = re.sub(r"[^\w\s]", " ", text.lower())
    lang_code = detect_language(cleaned_text)
    language_stopwords = get_stopwords_for_language(lang_code)

    # Convert stopwords set to list because TfidfVectorizer needs list/None/'english'
    stopwords_list = list(language_stopwords)

    vectorizer = TfidfVectorizer(stop_words=stopwords_list)

    try:
        tfidf_matrix = vectorizer.fit_transform([cleaned_text])
    except ValueError as e:
        # If there's nothing left after removing stopwords/punctuation
        if "empty vocabulary" in str(e).lower():
            return []  # Return an empty list -> no keywords
        else:
            raise e  # Something else went wrong

    feature_names = vectorizer.get_feature_names_out()
    scores = tfidf_matrix.toarray()[0]

    word_score_pairs = list(zip(feature_names, scores))
    word_score_pairs.sort(key=lambda x: x[1], reverse=True)

    return [w for (w, _) in word_score_pairs[:top_n]]