import graphlab
graphlab.canvas.set_target("ipynb")

"""
Explanation: Topic Modeling Using Graphlab


王成军
wangchengjun@nju.edu.cn
计算传播网 http://computational-communication.com
End of explanation
"""


sf = graphlab.SFrame.read_csv("/Users/chengjun/bigdata/w15", header=False)

sf

"""
Explanation: Download Data: http://select.cs.cmu.edu/code/graphlab/datasets/wikipedia/wikipedia_raw/w15
End of explanation
"""


dir(sf['X1'])

bow = sf['X1']._count_words()

type(sf['X1'])

type(bow)

bow.dict_has_any_keys(['limited'])

bow.dict_values()[0][:20]

sf['bow'] = bow

type(sf['bow'])

len(sf['bow'])

sf['bow'][0].items()[:5]

sf['tfidf'] = graphlab.text_analytics.tf_idf(sf['X1'])

sf['tfidf'][0].items()[:5]

sf.show()

sf

"""
Explanation: Transformations
https://dato.com/learn/userguide/text/analysis.html
End of explanation
"""


docs = sf['bow'].dict_trim_by_values(2)

docs = docs.dict_trim_by_keys(graphlab.text_analytics.stopwords(), exclude=True)

"""
Explanation: Text cleaning
End of explanation
"""


m = graphlab.topic_model.create(docs)

m

m.get_topics()

topics = m.get_topics().unstack(['word','score'], new_column_name='topic_words')['topic_words'].apply(lambda x: x.keys())
for topic in topics:
    print topic

pred = m.predict(docs)

pred.show()

pred = m.predict(docs, output_type='probabilities')

m['vocabulary']

m['topics']

def print_topics(m):
    topics = m.get_topics(num_words=5)
    topics = topics.unstack(['word','score'], new_column_name='topic_words')['topic_words']
    topics = topics.apply(lambda x: x.keys())
    for topic in topics:
        print topic
print_topics(m)

"""
Explanation: Topic modeling
End of explanation
"""


m2 = graphlab.topic_model.create(docs,
                                 num_topics=20,
                                 initial_topics=m['topics'])

"""
Explanation: Initializing from other models
End of explanation
"""


associations = graphlab.SFrame()
associations['word'] = ['recognition']
associations['topic'] = [0]

m2 = graphlab.topic_model.create(docs,
                                 num_topics=20,
                                 num_iterations=50,
                                 associations=associations, 
                                 verbose=False)

m2.get_topics(num_words=10)

print_topics(m2)

"""
Explanation: Seeding the model with prior knowledge
End of explanation
"""


