# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# Import some stuff

from lib import MongoBackend
from matplotlib import pyplot as plt
from matplotlib.pyplot import show
backend = MongoBackend('master')

res = backend.query("mahout_kmeans_text", ({'minDF':110, 'k':10}, {'input_size':1, 'output_size':1}))
print len(res)


# ## Simple Example: input vs output size

query = ({'minDF':110, 'k':10}, {'input_size':1, 'output_size':1})
backend.plot_query("mahout_kmeans_text", query, title="Input vs Output size", ylabel='size(bytes)', xlabel='size(bytes)')

# ### Mahout K-means, multi-K

plt.title("Mahout Kmeans text"); plt.ylabel('time(sec)'), plt.xlabel('#documents')
query = (({'minDF':110, 'k':-1}, {'documents':1, 'time':1}))
for k in range(5,21,5):
    query[0]['k']=k
    backend.plot_query("mahout_kmeans_text", query,label='k=%d'%k, show_plot=False)
show()



query = (({'minDF':110, 'k':10}, {'input_size':1, 'time':1}))
plt.title("Mahout Kmeans text"); plt.ylabel('time(sec)'), plt.xlabel('size (bytes)')
for k in range(5,21,5):
    query[0]['k']=k
    backend.plot_query("mahout_kmeans_text", query, label='k=%d'%k, show_plot=False)
plt.show()

query = (({'k':-1}, {'documents':1, 'time':1}))
plt.title("Weka Kmeans text"); plt.ylabel('time(sec)'), plt.xlabel('# documents')
for k in [2, 5,10,20]:
    query[0]['k']=k
    backend.plot_query("weka_kmeans_comp", query, label='k=%d'%k, show_plot=False)
plt.show()

query = (({'k':-1}, {'documents':1, 'time':1}))
plt.title("Cilk Kmeans text"); plt.ylabel('time(sec)'), plt.xlabel('# documents')
for k in [2, 5,10,20]:
    query[0]['k']=k
    backend.plot_query("cilk_kmeans_comp", query, label='k=%d'%k, show_plot=False)
plt.show()

# +
### Spark K-means, multi-K
# -

plt.title("Spark Kmeans text"); plt.ylabel('time(sec)'), plt.xlabel('#documents')
query = (({'minDF':10, 'k':-1}, {'documents':1, 'time':1}))
for k in range(5,21,5):
    query[0]['k']=k
    backend.plot_query("spark_kmeans_text", query,label='k=%d'%k, show_plot=False)
show()

# +
### Cilk
# -

plt.title("Cilk K-Means (mindDF:110)"); plt.ylabel('time(sec)'), plt.xlabel('#documents')
query = (({'minDF':160, 'k':-1}, {'documents':1, 'time':1}))
for k in range(5,21,5):
    query[0]['k']=k
    backend.plot_query("cilk_kmeans", query,label='k=%d'%k, show_plot=False)
show()

plt.title("Cilk K-Means (k:10)"); plt.ylabel('time(sec)'), plt.xlabel('#documents')
query = (({'minDF':1, 'k':10}, {'documents':1, 'time':1}))
for minDF in range(10,161,50):
    query[0]['minDF']=minDF
    backend.plot_query("cilk_kmeans", query,label='minDF=%d'%minDF, show_plot=False)
show()

# ## Streaming K-Means

plt.title("Spark Stream. K-Means"); plt.ylabel('time(sec)'), plt.xlabel('#entries')
for k in (2,5,10,15,20):
    query = (({'k':k}, {'lines':1, 'time':1}))
    backend.plot_query("streaming_kmeans", query,label='K=%d'%k, show_plot=False)
show()

lt.title("Spark Stream. K-Means"); plt.ylabel('time(sec)'), plt.xlabel('#entries')
for k in (2,5,10,15,20):
    query = (({'k':k}, {'lines':1, 'time':1}))
    backend.plot_query("streaming_kmeans", query,label='K=%d'%k, show_plot=False)
show()

StarcoderdataJupyterScriptsDedupFiltered