#We have a pre-loaded file with all the questions.
file = open(utils.data_path+'lda/questions.txt', 'r')
questions = file.readlines()
file.close()
len(questions)

"""
Explanation: Exploration of data with LDA
In this exercise we will infer the different topics of questions using LDA topic modelling.
Reading the data
End of explanation
"""


questions[0]

"""
Explanation: An example of a question:
End of explanation
"""


norm_texts = [utils.tokenize_and_stem(question) for question in questions]

norm_texts[0]

"""
Explanation: We create a list of tokenized and stemmed questions, also removing stop-words.
End of explanation
"""


#Turn our tokenized documents into a id <-> term dictionary
dictionary = corpora.Dictionary(norm_texts)
type(dictionary)

#Saving the dictionary object to be used later.
dictionary.save(utils.data_path+'lda/lda_dictionary')

#Loading existing disctionary object.
dictionary = corpora.Dictionary.load(utils.data_path+'lda/lda_dictionary')

"""
Explanation: Create Dictionary object
LDA needs a dictionary object, a mapping between words and numeric ids.
End of explanation
"""


# convert tokenized documents into a document-term matrix
bows = [dictionary.doc2bow(text) for text in norm_texts]
len(bows)

"""
Explanation: Create BOW object
Now with our dictionary, we have to create a bag-of-words object.
End of explanation
"""


bows[0]

"""
Explanation: An example of a bag-of-words element (id of word, frequency in sentence).
End of explanation
"""


ldamodel = gensim.models.ldamodel.LdaModel(bows, num_topics=10, id2word = dictionary, passes=20)
ldamodel.print_topics()

ldamodel.save(utils.data_path+'lda/lda_model')

ldamodel = gensim.models.LdaModel.load(utils.data_path+'lda/lda_model')

"""
Explanation: Train the LDA model
Trying different parameters, we found that using 10 topics gives us results that make sense.
End of explanation
"""


def create_topic_pars(pars, ldamodel, word_dictionary):
    norm_pars = [utils.tokenize_and_stem(par) for par in pars]
    print('created normalized paragraphs object of length %d' % len(norm_pars))
    bows = [word_dictionary.doc2bow(text) for text in norm_pars]
    print('created bag-of-words object of length %d' % len(bows))
    topic_pars = []
    for idx, val in enumerate(bows):
        lda_vector = ldamodel[val]
        #original LDA model topic (most relevant) and paragraph:
        topic_pars.append([ldamodel.print_topic(max(lda_vector, key=lambda item: item[1])[0]), pars[idx]]) #we attach the original paragraph here, not the cleaned version that we used for LDA.
    return(topic_pars)

topic_pars = create_topic_pars(questions, ldamodel, dictionary)

"""
Explanation: Running the model in old and new data
We now create a list of (topic, question) for each question in our data set, using the most probable topic. We also use this to infer the topic of a complete new question
End of explanation
"""


topic_pars[23000]

"""
Explanation: An example of the topic of an existing question
End of explanation
"""


new_questions = ['hey, i really think we should get more time to answer the questions in our quizzes, this sucks!!! :((()))', 'I am really sad, dont know how im going to be happy again, anyone knows?', 'is there anyway i can improve my grammar online?', 'I really did not understand the difference between mean and median, anyone who can help me!!!']

new_topic_pars = create_topic_pars(new_questions, ldamodel, dictionary)

new_topic_pars

"""
Explanation: Now we try with some complete new questions:
End of explanation
"""


new_topic_pars[2]

"""
Explanation: This is clearly an english grammar question:
End of explanation
"""


grammar_answers = [topic_par for topic_par in topic_pars if topic_par[0] == new_topic_pars[2][0]]

len(grammar_answers)

grammar_answers[502:505]

"""
Explanation: Lets check the answers to questions on our dataset with the same topic:
End of explanation
"""


