%matplotlib inline

# Load this library to make the graphs interactive for smaller samples
#import mpld3
#mpld3.enable_notebook()

# Turns out, multiple interactive scattergraphs with 170,000+ points each is a bit too much for a browser
# Who knew?! 

# Get the dataset:
from clustering import create_cluster_dataset, NewspaperArchive
DBFILE = "1745_1756_no_drift.db"
n = NewspaperArchive()
ds = create_cluster_dataset(n, daterange = [1745, 1756], dbfile = DBFILE)


"""
Explanation: Clustering experiment #1
Plan:

Derive a vector for left and righthand alignment variance for each contiguous block of text in each article (newspaper has pages, which hold articles, which consist of blocks).
Create a k=8 kmeans clustering using data from all newspapers from 1745 to 1756.
Visualise the clustering
Given the set of found poems, see into which clusters the poems get assigned.
Report on the spread of these and if a cluster is found which just has poems, report on all of the references within that cluster.
End of explanation
"""


data, transform, id_list = ds
print(data.toarray())

print(transform.get_feature_names())

"""
Explanation: What do these 'vectors' look like? What do the columns refer to?
End of explanation
"""


from clustering import ClusterDB
db = ClusterDB(DBFILE)
item = dict(db.vecidtoitem(id_list[-1]))
print(item)

from burney_data import BurneyDB
bdb = BurneyDB("burney.db")

titlemd = bdb.get_title_row(titleAbbreviation=item['newspaper'])
entry = bdb.get_entry_row(year=item['year'], month=item['month'], day=item['day'], title_id= titlemd['id'])
issue = bdb.get_issue_row(id=entry['issue_id'])

print(titlemd)
print(issue)
print(entry)

import numpy as np
from matplotlib import pyplot as plt

# Mask off leaving just the left and right variance columns
npdata = data.toarray()
mask = np.ones((8), dtype=bool)
mask[[0,1,2,4,5,7]] = False

marray = npdata[:,mask]

"""
Explanation: Going from a vector back to the metadata reference:
By keeping an 'id_list', we can look up the identifier for any vector in the list from the database we've made for this clustering attemp. This lets us look up what the reference for that is, and where we can find it:
End of explanation
"""


plt.scatter(marray[:,0], marray[:,1])
plt.show()

"""
Explanation: x1 vs x2 varience?
What is the rough shape of this data? The varience of x1 and x2 are equivalent to the left and right alignment of the text varies in a given block of text.
End of explanation
"""


# Build the clustering and show the individual clusters as best we can:
from sklearn.cluster import KMeans

estimator = KMeans(n_clusters=8)

clusters = estimator.fit(marray)
labels = estimator.labels_

def isol(label, labels):
    for l in labels.astype(np.float):
        if l != label:
            yield "#444444"
        else:
            yield "#FF3355"

#for label in set(labels):
#    print(label)
#    plt.scatter(marray[:,0], marray[:,1], c=list(isol(label, labels)))
#    plt.show()

"""
Explanation: Attempting K-Means
What sort of clustering algorithm to employ is actually a good question. K-means can give fairly meaningless responses if the data is of a given sort. Generally, it can be useful but cannot be used blindly. 
Given the data above, it might be a good start however.
End of explanation
"""


import csv

def get_info(item_id):
    record = dict(db.vecidtoitem(item_id))
    vect = dict(db.vector(item_id))
    titlemd = bdb.get_title_row(titleAbbreviation=record['newspaper'])
    entry = bdb.get_entry_row(year=record['year'], month=record['month'], day=record['day'], title_id= titlemd['id'])
    issue = bdb.get_issue_row(id=entry['issue_id'])
    record.update(titlemd)
    record.update(entry)
    record.update(issue)
    record.update(vect)
    return record

for label in set(labels):
    print("Saving label {0}".format(label))
    with open("exp1_cluster{0}.csv".format(label), "w") as cfn:
        fields = ["title", "titleAbbreviation", "year", "month", "day", 
              "issueNumber", "printedDate", "page", "article", "block_number", "filepath", "x1_var1", "x2_var1", "ltcount"]
        csvdoc = csv.DictWriter(cfn, fieldnames = fields)
        csvdoc.writerow(dict([(x,x) for x in fields]))
        count = 0
        for idx, vlabel in enumerate(list(labels)):
            if idx % 1000 == 0:
                print("Tackling line {0} - saved {1} lines for this label".format(idx, count))
            if vlabel == label:
                record = get_info(id_list[idx])
                csvdoc.writerow(dict([(x,record[x]) for x in fields]))
                count += 1

"""
Explanation: It looks like cluster 5 here is the one that should contain more complete poems as a block of text than the rest if our assumptions are correct.
Lets export this as a list of references to explore further - "cluster5.csv"
End of explanation
"""


