# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3.6 (tensorflow)
#     language: python
#     name: tensorflow
# ---

# # Applying otimal LDA model on sentences of reviews

# Goal: Finding the best topic modeling model with optimum value of number of topics 'k', to generate most optimal topics on reviews

#Importing Libraries
import gensim
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
from stop_words import get_stop_words
from gensim import corpora, models
from gensim.models import Phrases
from nltk.stem.porter import *
import numpy as np
import operator
import nltk
# Compute bigrams.
from gensim.models import Phrases
from pprint import pprint
# Plotting tools
import pyLDAvis
import pyLDAvis.gensim  # don't skip this

import pickle

import pandas as pd

#Loading data. This data is obtained by grouping the product IDs with respect number reviews and merging their reviews together.
df = pd.read_csv('/Users/harika_pradeep/Downloads/df_grouped.csv', index_col=0)

#Loading metadata that has product titles for each product.
df_m = pd.read_csv('/Users/harika_pradeep/Downloads/Title_prod.csv', index_col=0)

df.shape

df_m.shape

df.head()

df_m.head()

#Merging both the dataframes with product ID column "asin"
df_merged = pd.merge (df_m,df,left_on='asin',right_on='asin')

df_merged.shape 

df_merged[df_merged['title'].isnull()].shape

df_m['asin'].unique().shape[0]

df_merged.head()

# Sorting the rows in decreasing order
df_merged.sort_values(by=['count'],inplace=True,ascending=False)

df_merged.head ()

df_merged.reset_index(drop = True, inplace = True)

df_merged.head ().shape

#Considering only the top 25 products for further calculations (for memory constraints)
df_req = df_merged.head(25)

df_req.shape

df_req

df_req.to_csv("/Users/harika_pradeep/Downloads/top_25.csv")

#splitting each review for these 25 products into sentences. This will be used while appying sentiment analysis on each sentence
df_req_copy = df_req
df_sentences = pd.DataFrame(df_req_copy.reviewText.str.split('.').tolist(), index=df_req_copy.asin).stack()
df_sentences =df_sentences.reset_index([0, 'asin'])
df_sentences.columns = ['asin', 'sentences']

#Total number of sentences on which we will be applying topic modeling and sentiment analysis is 300677
df_sentences.shape

df_sentences.head()

df_sentences.to_csv("/Users/harika_pradeep/Downloads/top_25_sents.csv")

# # Tokenization : Tokenizing each sentence into words

tokenizer = RegexpTokenizer(r'\w+')

#Applying this on one sentence
doc = df_sentences.sentences[0]

tokens = tokenizer.tokenize(doc.lower())
print('{} characters in string vs {} words in a list'.format(len(doc), len(tokens)))

docs = [token for token in tokens if not token.isnumeric()]
print('{} words in a list after removing numbers'.format(len(docs)))

# Remove words that are less than 4 characters only
docs = [token for token in docs if len(token) > 3]
print('{} words in a list after words that are less than 4 chars'.format(len(docs)))

# # Preparing stopwords and removing them

# +
#create a merged list of stop words
nltk_stpwd = stopwords.words('english')
#Extend stopwords with commonly found tokens in review texts
nltk_stpwd.extend(['generally', 'used', 'personally', 'review', 'honestly','truly','whatever','done','star','one','two','three','four','five','since','ever','even','much','thing','also','go','come','must'])
stop_words_stpwd = get_stop_words('en')
merged_stopwords = list(set(nltk_stpwd + stop_words_stpwd))

print(len(set(merged_stopwords)))
print(merged_stopwords[:10])
# -

stopped_tokens = [token for token in docs if not token in merged_stopwords]
print('{} words in a list after removing stop words'.format(len(stopped_tokens)))

# # Lemmatizing

# "keeping only noun, adj, vb, adv"

# Instantiate a WordNetLemmatizer
lemmatizer = WordNetLemmatizer()

lemm_tokens = [lemmatizer.lemmatize(token) for token in stopped_tokens]
print(lemm_tokens[:10])

# +
#Applying all the above steps on entire data
num_sentences = df_sentences.shape[0]

doc_set = [df_sentences.sentences[i] for i in range(num_sentences)]

texts = []

for doc in doc_set:
    # putting our 5 steps together
    tokens = tokenizer.tokenize(doc.lower())
    tokens_alp = [token for token in tokens if not token.isnumeric()]
    token_gr_3 = [token for token in tokens_alp if len(token) > 3]
    stopped_tokens = [token for token in token_gr_3 if not token in merged_stopwords]
    lemm_tokens = [lemmatizer.lemmatize(token) for token in stopped_tokens]
    # add tokens to list
    texts.append(lemm_tokens)
# -

bigram = Phrases(texts, min_count=30)
for idx in range(len(texts)):
    for token in bigram[texts[idx]]:
        if '_' in token:
            # Token is a bigram, add to document.
            texts[idx].append(token)

#Saving file for later use
with open('texts_s.pkl', 'wb') as f:
    pickle.dump(texts, f)

# Gensim's Dictionary encapsulates the mapping between normalized words and their integer ids.
texts_dict = corpora.Dictionary(texts)
# saving for later use
texts_dict.save('auto_review_s.dict') 
# Examine each token’s unique id
print(texts_dict)

print("IDs 1 through 10: {}".format(sorted(texts_dict.token2id.items(), key=operator.itemgetter(1), reverse = False)[:10]))

#Filter tokens that appear too rare are too frequent
texts_dict.filter_extremes(no_below = 20, no_above = 0.15) # inplace filter
print(texts_dict)
print("top terms:")
print(sorted(texts_dict.token2id.items(), key=operator.itemgetter(1), reverse = False)[:100])

#Create bag of words corpus for all the sentences
corpus = [texts_dict.doc2bow(text) for text in texts]
len(corpus)

# # Applying LDA Model using Gensim Libray

#Save corpus for later use
gensim.corpora.MmCorpus.serialize('amzn_h_k_sentences.mm', corpus)

# In order to generate most accurate topics selcting the number of topics while applying the model is important. We can evaluate the model performance based coherence coefficient. The more the coherence value the more accurate the topics are.

#Build LDA model with num_topics = 5
lda_model = gensim.models.LdaModel(corpus=corpus, alpha='auto',eta='auto',id2word=texts_dict,num_topics=5,
                                   chunksize=10000,passes=10)

lda_model.save('LDA_Model_sent.lda')

pprint(lda_model.print_topics())
doc_lda = lda_model[corpus]

top_topics = lda_model.top_topics(corpus, topn=20)

# Compute Perplexity
print('\nPerplexity: ', lda_model.log_perplexity(corpus))

#Find the average topic coherence
avg_topic_coherence = sum([t[1] for t in top_topics]) / 10
print('Average topic coherence: %.4f.' % avg_topic_coherence)

#Print topics and word distribution for each topic
counter = 0
for topic in top_topics:
    print('Topic {}:'.format(counter))
    counter += 1
    pprint(topic)

# Visualize the topics
pyLDAvis.enable_notebook()
#Prepare topic visualization
vis = pyLDAvis.gensim.prepare(lda_model, corpus, texts_dict,sort_topics = False)

vis

pyLDAvis.save_html(vis,"vis_sent.html")

# # Applying LDA Mallet model

mallet_path = '/Users/harika_pradeep/Downloads/mallet-2.0.8/bin/mallet' # update this path
ldamallet = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=5, id2word=texts_dict)

pprint(ldamallet.show_topics(formatted=False))

#Coeherenc coefficient calculation for Mallet LDA
coherence_model_ldamallet = CoherenceModel(model=ldamallet, texts=texts, dictionary=texts_dict, coherence='c_v')
coherence_ldamallet = coherence_model_ldamallet.get_coherence()
print('\nCoherence Score: ', coherence_ldamallet)


# The coherence coefficient is better for lda mallet rather than lda model. Now we can check for which value of 'k' (the number of topics) we get better coheherence

def compute_coherence_values(dictionary, corpus, texts, limit, start=2, step=3):
    coherence_values = []
    model_list = []
    for num_topics in range(start, limit, step):
        model = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=num_topics, id2word=texts_dict)
        model_list.append(model)
        coherencemodel = CoherenceModel(model=model, texts=texts, dictionary=dictionary, coherence='c_v')
        coherence_values.append(coherencemodel.get_coherence())

    return model_list, coherence_values


model_list, coherence_values = compute_coherence_values(dictionary=texts_dict, corpus=corpus, texts=texts, start=2, limit=40, step=6)

import matplotlib.pyplot as plt

#Plotting the coherence for LDAmallet model for differen k values
limit=40; start=2; step=6;
x = range(start, limit, step)
plt.plot(x, coherence_values)
plt.xlabel("Num Topics")
plt.ylabel("Coherence score")
plt.legend(("coherence_values"), loc='best')
plt.show()

#best LDA model at topic number = 20 
ldamallet = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=20, id2word=texts_dict)

pprint(ldamallet.show_topics(formatted=False))

#Coeherenc coefficient calculation for Mallet LDA
coherence_model_ldamallet = CoherenceModel(model=ldamallet, texts=texts, dictionary=texts_dict, coherence='c_v')
coherence_ldamallet = coherence_model_ldamallet.get_coherence()
print('\nCoherence Score: ', coherence_ldamallet)

ldamallet.save('ldamallet.lda')

# #### Reference Links:
# https://blog.insightdatascience.com/topic-modeling-and-sentiment-analysis-to-pinpoint-the-perfect-doctor-6a8fdd4a3904 <br>
# https://towardsdatascience.com/end-to-end-topic-modeling-in-python-latent-dirichlet-allocation-lda-35ce4ed6b3e0 <br>
# https://www.machinelearningplus.com/nlp/topic-modeling-gensim-python/#18dominanttopicineachsentence <br>
# https://kldavenport.com/topic-modeling-amazon-reviews/#what-can-we-do-with-this-in-production



StarcoderdataJupyterScriptsDedupFiltered