import os
from gensim import corpora, models
%load_ext memory_profiler

import scipy
scipy.show_config()

MODELS_DIR = "../Data/models/lda_standard"
num_topics = 10
dictionary = corpora.Dictionary.load(os.path.join(MODELS_DIR,'twentyNewsGroup.dict'))
corpus = corpora.MmCorpus(os.path.join(MODELS_DIR, 'corpora.mm'))

"""
Explanation: Benchmarking LDA in gensim
This is a benchmarking done on gensim by runninng LDA over the 20 newsgroup dataset.
Benchmarking was done on a : i7 system with 2 cores @2.40Ghz 4 threads and 8 Gb RAM.
Gensim version 0.13.4.1
End of explanation
"""


%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=num_topics)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=num_topics)

%timeit lda = models.ldamulticore.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=num_topics, workers = 1)

%timeit lda = models.ldamulticore.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=num_topics, workers = 2)

%timeit lda = models.ldamulticore.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=num_topics, workers = 3)

%timeit lda = models.ldamulticore.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=num_topics, workers = 4)

%memit lda = models.ldamulticore.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=num_topics, workers = 1)

%memit lda = models.ldamulticore.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=num_topics, workers = 2)

%memit lda = models.ldamulticore.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=num_topics, workers = 3)

%memit lda = models.ldamulticore.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=num_topics, workers = 4)

"""
Explanation: Default settings
First off I will start by testing with default settings
End of explanation
"""


%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=10, iterations = 100)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=10, iterations = 100)

%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=10, iterations = 300)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=10, iterations = 300)

%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=10, iterations = 1000)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=10, iterations = 1000)

"""
Explanation: Additional parameters
Now to simulate a more useful use-case scenario when performing topic modelling, I will now start putting more parameters and tinkering with them, and see how exection time and memory consumption would vary.
Iterations
End of explanation
"""


%timeit lda = models.ldamulticore.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=10, workers = 3, iterations = 100)

%memit lda = models.ldamulticore.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=10, workers = 3, iterations = 100)

%timeit lda = models.ldamulticore.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=10, workers = 3, iterations = 300)

%memit lda = models.ldamulticore.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=10, workers = 3, iterations = 300)

%timeit lda = models.ldamulticore.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=10, workers = 3, iterations = 1000)

%memit lda = models.ldamulticore.LdaMulticore(corpus=corpus, id2word=dictionary, num_topics=10, workers = 3, iterations = 1000)

"""
Explanation: Testing LDA with iterations with 3 workers
End of explanation
"""


%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, iterations=100)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, iterations=100)

%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=20, iterations=100)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=20, iterations=100)

%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=40, iterations=100)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=40, iterations=100)

%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=60, iterations=100)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=60, iterations=100)

%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=80, iterations=100)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=80, iterations=100)

"""
Explanation: Number of topics
End of explanation
"""


#default number of passes is 1
%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, passes=1)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, passes=1)

%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, passes=2)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, passes=2)

"""
Explanation: Number of passes
End of explanation
"""


%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, alpha='symmetric', iterations=1000)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, alpha='symmetric', iterations=1000)

%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, alpha='asymmetric', iterations=1000)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, alpha='asymmetric', iterations=1000)

%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, alpha='auto', iterations=1000)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, alpha='auto', iterations=1000)

"""
Explanation: Alpha
End of explanation
"""


%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, chunksize=2000, iterations=1000)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, chunksize=2000, iterations=1000)

%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, chunksize=3000, iterations=1000)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, chunksize=3000, iterations=1000)

%timeit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, chunksize=4000, iterations=1000)

%memit lda = models.LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, chunksize=4000, iterations=1000)

"""
Explanation: Chunksize
End of explanation
"""


import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt
import pandas as pd
%matplotlib inline
sns.set(style="white")

f, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 4))
d = { 'time' : pd.Series([74, 63, 53.2, 51.7, 50.8], index=xrange(5)),
    'peak_memory' : pd.Series([310.88, 462.82, 416.68, 465.73, 502.21], index=xrange(5)),
     'memory_increment' : pd.Series([65.73, 198.18, 127.80, 195.93, 92.29], index=xrange(5)),
     'algo' : pd.Series(['lda', 'lda-m(1)', 'lda-m(2)', 'lda-m(3)', 'lda-m(4)'], index=xrange(5))}
data = pd.DataFrame(d)
plt.suptitle('Default settings; # of topics 10', fontsize=20)
ax1.set_title('algo vs execution time')
ax1.set_xlabel('Algo')
ax1.set_ylabel('Time in sec')
data = pd.DataFrame(d)
sns.barplot(x='algo', y='time', data=data, palette='PuBu', ax=ax1)
ax2.set_title('algo vs memory usage')
ax2.set_xlabel('Algo')
ax2.set_ylabel('Memory in MB')
sns.barplot(x='algo', y='peak_memory', data=data, palette='Greens', ax=ax2)

f1, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 4))
d1 = { 'time' : pd.Series([84, 91, 96, 63, 91, 127], index=xrange(6)),
    'memory' : pd.Series([469.88, 330.81, 331.52, 488.07, 595.02, 542.75], index=xrange(6)),
     'iterations' : pd.Series([100, 300, 1000, 100, 300, 1000], index=xrange(6)),
     'algo' : pd.Series(['lda', 'lda', 'lda', 'lda-m(3)', 'lda-m(3)', 'lda-m(3)'], index=xrange(6))}
data1 = pd.DataFrame(d1)
plt.suptitle('Number of iterations', fontsize=20)
ax1.set_title('algo vs execution time')
ax1.set_xlabel('Algo')
ax1.set_ylabel('Time in sec')
sns.barplot(x='algo', y='time', hue='iterations', data=data1, palette='Purples', ax=ax1)
ax2.set_title('algo vs memory usage')
ax2.set_xlabel('Algo')
ax2.set_ylabel('Memory in MB')
sns.barplot(x='algo', y='memory', hue='iterations', data=data1, palette='pastel', ax=ax2)

f, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 4))
d = { 'time' : pd.Series([68, 113, 172, 226, 273], index=xrange(5)),
    'memory' : pd.Series([448.07, 491.55, 563.61, 706.84, 889.00], index=xrange(5)),
     'topics' : pd.Series([5, 20, 40, 60, 80], index=xrange(5))}
data = pd.DataFrame(d)
plt.suptitle('Number of topics', fontsize=20)
ax1.set_title('Execution time')
ax1.set_xlabel('Algo')
ax1.set_ylabel('Time in sec')
sns.barplot(x='topics', y='time', data=data, palette=sns.cubehelix_palette(5)
, ax=ax1)
ax2.set_title('Memory usage')
ax2.set_xlabel('Algo')
ax2.set_ylabel('Memory in MB')
sns.barplot(x='topics', y='memory', data=data, ax=ax2)

f, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 4))
d = { 'time' : pd.Series([57.8, 98], index=xrange(2)),
    'memory' : pd.Series([425.43, 282.23], index=xrange(2)),
     'passes' : pd.Series([1, 2], index=xrange(2))}
data = pd.DataFrame(d)
plt.suptitle('Number of passes', fontsize=20)
ax1.set_title('Execution time')
ax1.set_xlabel('Algo')
ax1.set_ylabel('Time in sec')
sns.barplot(x='passes', y='time', data=data, palette='pastel', ax=ax1)
ax2.set_title('Memory usage')
ax2.set_xlabel('Algo')
ax2.set_ylabel('Memory in MB')
sns.barplot(x='passes', y='memory', data=data, ax=ax2)

f, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 4))
d = { 'time' : pd.Series([81, 80, 78], index=xrange(3)),
    'memory' : pd.Series([282.17, 282.23, 282.47], index=xrange(3)),
     'alpha' : pd.Series(['symmetric', 'asymmetric', 'auto'], index=xrange(3))}
data = pd.DataFrame(d)
plt.suptitle('Alpha', fontsize=20)
ax1.set_title('Execution time')
ax1.set_xlabel('Algo')
ax1.set_ylabel('Time in sec')
sns.barplot(x='alpha', y='time', data=data, palette='Reds', ax=ax1)
ax2.set_title('Memory usage')
ax2.set_xlabel('Algo')
ax2.set_ylabel('Memory in MB')
sns.barplot(x='alpha', y='memory', data=data, ax=ax2)

f, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 4))
d = { 'time' : pd.Series([81, 94, 112], index=xrange(3)),
    'memory' : pd.Series([282.43, 294.43, 305.25], index=xrange(3)),
     'chunk size' : pd.Series([2000, 3000, 4000], index=xrange(3))}
data = pd.DataFrame(d)
plt.suptitle('Chunk size', fontsize=20)
ax1.set_title('Execution time')
ax1.set_xlabel('Algo')
ax1.set_ylabel('Time in sec')
sns.barplot(x='chunk size', y='time', data=data, palette='Blues', ax=ax1)
ax2.set_title('Memory usage')
ax2.set_xlabel('Algo')
ax2.set_ylabel('Memory in MB')
sns.barplot(x='chunk size', y='memory', data=data, palette=sns.cubehelix_palette(3, start=.5, rot=-.75), ax=ax2)

"""
Explanation: Visualization
End of explanation
"""


