import nltk
from tethne.readers import zotero
import matplotlib.pyplot as plt
from helpers import normalize_token, filter_token
import gensim
from itertools import groupby, islice

text_root = '../data/EmbryoProjectTexts/files'
zotero_export_path = '../data/EmbryoProjectTexts'

corpus = nltk.corpus.PlaintextCorpusReader(text_root, 'https.+')
metadata = zotero.read(zotero_export_path, index_by='link', follow_links=False)

years = []
documents = []
for fileid in corpus.fileids():
    for sentence in corpus.sents(fileids=[fileid]):
        years.append(metadata[fileid].date)
        documents.append([normalize_token(token) 
                          for token in sentence 
                          if filter_token(token)])

len(documents)

model = gensim.models.Word2Vec(documents, size=200, window=5)

"""
Explanation: 1.8 CBOW/Skip-gram model
Literature: here and here.
End of explanation
"""


model.most_similar(positive=['cell'])

model.similarity('egg', 'ovum'), model.similarity('egg', 'embryo'), model.similarity('egg', 'brain')

model.syn0.shape     # Words x Dimensions.

len(model.index2word)

"""
Explanation: Similary is measured using cosine distance in the low-dimensional embedding/vector space.
End of explanation
"""


from sklearn import decomposition

focal_words = ['egg', 'ontogeny', 'ovum', 'fetus', 'ivf', 'uterus', 'brain', 
               'neuron', 'cell', 'transplantation', 'culture', 'stem', 'nucleus']

focal_indices = [model.index2word.index(w) for w in focal_words]

pca = decomposition.PCA(n_components=2)

t = pca.fit_transform(model.syn0[focal_indices, :])

plt.scatter(t[:, 0], t[:, 1])
for i in xrange(len(focal_indices)):
    w = focal_words[i]
    plt.text(t[i, 0], t[i, 1], w)

"""
Explanation: PCA on lower-dimensional projection
End of explanation
"""


