from __future__ import print_function

import os
import logging
import json
import warnings

try:
    raise ImportError
    import pyLDAvis.gensim
    CAN_VISUALIZE = True
    pyLDAvis.enable_notebook()
    from IPython.display import display
except ImportError:
    ValueError("SKIP: please install pyLDAvis")
    CAN_VISUALIZE = False

import numpy as np

from gensim.models import CoherenceModel, LdaModel, HdpModel
from gensim.models.wrappers import LdaVowpalWabbit, LdaMallet
from gensim.corpora import Dictionary

warnings.filterwarnings('ignore')  # To ignore all warnings that arise here to enhance clarity

"""
Explanation: Demonstration of the topic coherence pipeline in Gensim
Introduction
We will be using the u_mass and c_v coherence for two different LDA models: a "good" and a "bad" LDA model. The good LDA model will be trained over 50 iterations and the bad one for 1 iteration. Hence in theory, the good LDA model will be able come up with better or more human-understandable topics. Therefore the coherence measure output for the good LDA model should be more (better) than that for the bad LDA model. This is because, simply, the good LDA model usually comes up with better topics that are more human interpretable.
End of explanation
"""


texts = [['human', 'interface', 'computer'],
         ['survey', 'user', 'computer', 'system', 'response', 'time'],
         ['eps', 'user', 'interface', 'system'],
         ['system', 'human', 'system', 'eps'],
         ['user', 'response', 'time'],
         ['trees'],
         ['graph', 'trees'],
         ['graph', 'minors', 'trees'],
         ['graph', 'minors', 'survey']]

dictionary = Dictionary(texts)
corpus = [dictionary.doc2bow(text) for text in texts]

"""
Explanation: Set up corpus
As stated in table 2 from this paper, this corpus essentially has two classes of documents. First five are about human-computer interaction and the other four are about graphs. We will be setting up two LDA models. One with 50 iterations of training and the other with just 1. Hence the one with 50 iterations ("better" model) should be able to capture this underlying pattern of the corpus better than the "bad" LDA model. Therefore, in theory, our topic coherence for the good LDA model should be greater than the one for the bad LDA model.
End of explanation
"""


goodLdaModel = LdaModel(corpus=corpus, id2word=dictionary, iterations=50, num_topics=2)
badLdaModel = LdaModel(corpus=corpus, id2word=dictionary, iterations=1, num_topics=2)

"""
Explanation: Set up two topic models
We'll be setting up two different LDA Topic models. A good one and bad one. To build a "good" topic model, we'll simply train it using more iterations than the bad one. Therefore the u_mass coherence should in theory be better for the good model than the bad one since it would be producing more "human-interpretable" topics.
End of explanation
"""


goodcm = CoherenceModel(model=goodLdaModel, corpus=corpus, dictionary=dictionary, coherence='u_mass')
badcm = CoherenceModel(model=badLdaModel, corpus=corpus, dictionary=dictionary, coherence='u_mass')

"""
Explanation: Using U_Mass Coherence
End of explanation
"""


print(goodcm)

"""
Explanation: View the pipeline parameters for one coherence model
Following are the pipeline parameters for u_mass coherence. By pipeline parameters, we mean the functions being used to calculate segmentation, probability estimation, confirmation measure and aggregation as shown in figure 1 in this paper.
End of explanation
"""


if CAN_VISUALIZE:
    prepared = pyLDAvis.gensim.prepare(goodLdaModel, corpus, dictionary)
    display(pyLDAvis.display(prepared))

if CAN_VISUALIZE:
    prepared = pyLDAvis.gensim.prepare(badLdaModel, corpus, dictionary)
    display(pyLDAvis.display(prepared))

print(goodcm.get_coherence())
print(badcm.get_coherence())

"""
Explanation: Interpreting the topics
As we will see below using LDA visualization, the better model comes up with two topics composed of the following words:
1. goodLdaModel:
    - Topic 1: More weightage assigned to words such as "system", "user", "eps", "interface" etc which captures the first set of documents.
    - Topic 2: More weightage assigned to words such as "graph", "trees", "survey" which captures the topic in the second set of documents.
2. badLdaModel:
    - Topic 1: More weightage assigned to words such as "system", "user", "trees", "graph" which doesn't make the topic clear enough.
    - Topic 2: More weightage assigned to words such as "system", "trees", "graph", "user" which is similar to the first topic. Hence both topics are not human-interpretable.
Therefore, the topic coherence for the goodLdaModel should be greater for this than the badLdaModel since the topics it comes up with are more human-interpretable. We will see this using u_mass and c_v topic coherence measures.
Visualize topic models
End of explanation
"""


goodcm = CoherenceModel(model=goodLdaModel, texts=texts, dictionary=dictionary, coherence='c_v')
badcm = CoherenceModel(model=badLdaModel, texts=texts, dictionary=dictionary, coherence='c_v')

"""
Explanation: Using C_V coherence
End of explanation
"""


print(goodcm)

"""
Explanation: Pipeline parameters for C_V coherence
End of explanation
"""


print(goodcm.get_coherence())
print(badcm.get_coherence())

"""
Explanation: Print coherence values
End of explanation
"""


# Replace with path to your Vowpal Wabbit installation
vw_path = '/usr/local/bin/vw'

# Replace with path to your Mallet installation
home = os.path.expanduser('~')
mallet_path = os.path.join(home, 'mallet-2.0.8', 'bin', 'mallet')

model1 = LdaVowpalWabbit(vw_path, corpus=corpus, num_topics=2, id2word=dictionary, passes=50)
model2 = LdaVowpalWabbit(vw_path, corpus=corpus, num_topics=2, id2word=dictionary, passes=1)

cm1 = CoherenceModel(model=model1, corpus=corpus, coherence='u_mass')
cm2 = CoherenceModel(model=model2, corpus=corpus, coherence='u_mass')
print(cm1.get_coherence())
print(cm2.get_coherence())

model1 = LdaMallet(mallet_path, corpus=corpus, num_topics=2, id2word=dictionary, iterations=50)
model2 = LdaMallet(mallet_path, corpus=corpus, num_topics=2, id2word=dictionary, iterations=1)

cm1 = CoherenceModel(model=model1, texts=texts, coherence='c_v')
cm2 = CoherenceModel(model=model2, texts=texts, coherence='c_v')
print(cm1.get_coherence())
print(cm2.get_coherence())

"""
Explanation: Support for wrappers
This API supports gensim's ldavowpalwabbit and ldamallet wrappers as input parameter to model.
End of explanation
"""


hm = HdpModel(corpus=corpus, id2word=dictionary)

# To get the topic words from the model
topics = []
for topic_id, topic in hm.show_topics(num_topics=10, formatted=False):
    topic = [word for word, _ in topic]
    topics.append(topic)
topics[:2]

# Initialize CoherenceModel using `topics` parameter
cm = CoherenceModel(topics=topics, corpus=corpus, dictionary=dictionary, coherence='u_mass')
cm.get_coherence()

"""
Explanation: Support for other topic models
The gensim topics coherence pipeline can be used with other topics models too. Only the tokenized topics should be made available for the pipeline. Eg. with the gensim HDP model
End of explanation
"""


