yanrin/topic_modeling.py
def ret_top_model(corpus, dictionary, train_texts, num_times):
    """
    Since LDAmodel is a probabilistic model, it comes up different topics each time we run it. To control the
    quality of the topic model we produce, we can see what the interpretability of the best topic is and keep
    evaluating the topic model until a certian threshold is crossed.

    Returns:
    -------
    lm: Final evaluated topic model
    top_topics: ranked topics in decreasing order. List of tuples
    """
    top_topics = [(0, 0)]
    rounds = 1
    high = 0.0
    out_lm = None
    print('dict size:',len(dictionary))
    num_topics = int(len(dictionary)*0.1) #10% of all dictionary
    print('num_topics:',num_topics)
    while True:
        lm = LdaModel(corpus=corpus, num_topics=num_topics, id2word=dictionary, minimum_probability=0)
        coherence_values = {}
        for n, topic in lm.show_topics(num_topics=-1, formatted=False):
            topic = [word for word, _ in topic]
            cm = CoherenceModel(topics=[topic], texts=train_texts, dictionary=dictionary, window_size=10)
            coherence_values[n] = cm.get_coherence()
        top_topics = sorted(coherence_values.items(), key=operator.itemgetter(1), reverse=True)
        if high < top_topics[0][1]:
            high = top_topics[0][1]
            out_lm = lm
        print('round ',rounds,':',top_topics[0][1])
        if rounds > num_times-1:
            break
        rounds+=1
    return out_lm, top_topics, high

    Since LDAmodel is a probabilistic model, it comes up different topics each time we run it. To control the
    quality of the topic model we produce, we can see what the interpretability of the best topic is and keep
    evaluating the topic model until a certian threshold is crossed.

    Returns:
    -------
    lm: Final evaluated topic model
    top_topics: ranked topics in decreasing order. List of tuples
    
Since LDAmodel is a probabilistic model, it comes up different topics each time we run it. To control the
quality of the topic model we produce, we can see what the interpretability of the best topic is and keep
evaluating the topic model until a certian threshold is crossed.
Since LDAmodel is a probabilistic model it comes up different topics each time we run it . To control the quality of the topic model we produce we can see what the interpretability of the best topic is and keep evaluating the topic model until a certian threshold is crossed .
def ret_top_model(corpus, dictionary, train_texts, num_times):
    top_topics = [(0, 0)]
    rounds = 1
    high = 0.0
    out_lm = None
    print('dict size:',len(dictionary))
    num_topics = int(len(dictionary)*0.1) 
    print('num_topics:',num_topics)
    while True:
        lm = LdaModel(corpus=corpus, num_topics=num_topics, id2word=dictionary, minimum_probability=0)
        coherence_values = {}
        for n, topic in lm.show_topics(num_topics=-1, formatted=False):
            topic = [word for word, _ in topic]
            cm = CoherenceModel(topics=[topic], texts=train_texts, dictionary=dictionary, window_size=10)
            coherence_values[n] = cm.get_coherence()
        top_topics = sorted(coherence_values.items(), key=operator.itemgetter(1), reverse=True)
        if high < top_topics[0][1]:
            high = top_topics[0][1]
            out_lm = lm
        print('round ',rounds,':',top_topics[0][1])
        if rounds > num_times-1:
            break
        rounds+=1
    return out_lm, top_topics, high
def ret_top_model ( corpus , dictionary , train_texts , num_times ) : top_topics = [ ( 0 , 0 ) ] rounds = 1 high = 0.0 out_lm = None print ( 'dict size:' , len ( dictionary ) ) num_topics = int ( len ( dictionary ) * 0.1 ) print ( 'num_topics:' , num_topics ) while True : lm = LdaModel ( corpus = corpus , num_topics = num_topics , id2word = dictionary , minimum_probability = 0 ) coherence_values = { } for n , topic in lm . show_topics ( num_topics = - 1 , formatted = False ) : topic = [ word for word , _ in topic ] cm = CoherenceModel ( topics = [ topic ] , texts = train_texts , dictionary = dictionary , window_size = 10 ) coherence_values [ n ] = cm . get_coherence ( ) top_topics = sorted ( coherence_values . items ( ) , key = operator . itemgetter ( 1 ) , reverse = True ) if high < top_topics [ 0 ] [ 1 ] : high = top_topics [ 0 ] [ 1 ] out_lm = lm print ( 'round ' , rounds , ':' , top_topics [ 0 ] [ 1 ] ) if rounds > num_times - 1 : break rounds += 1 return out_lm , top_topics , high
Since LDAmodel is a probabilistic model, it comes up different topics each time we run it.
Since LDAmodel is a probabilistic model it comes up different topics each time we run it .
"""
    Since LDAmodel is a probabilistic model, it comes up different topics each time we run it. To control the
    quality of the topic model we produce, we can see what the interpretability of the best topic is and keep
    evaluating the topic model until a certian threshold is crossed.

    Returns:
    -------
    lm: Final evaluated topic model
    top_topics: ranked topics in decreasing order. List of tuples
    """ #10% of all dictionary
None
corpus
dictionary
train_texts
num_times