documents = nltk.corpus.PlaintextCorpusReader('../data/EmbryoProjectTexts/files', 'https.+')
metadata = zotero.read('../data/EmbryoProjectTexts', index_by='link', follow_links=False)

"""
Explanation: Feature selection: keywords
A major problem-area in text mining is determining the thematic or topical content of texts. One of the most basic problems in this area is to identify the terms in a text -- "keywords" -- that most accurately represents its distinctive thematic characteristics.
In this notebook, we will use Dunning's log-likelihood statistic to identify keywords for individual documents in a collection of texts. It is fairly typical that methods used for statistical analysis are also used for information extraction and classification. 
We'll use the Embryo Project corpus from earlier notebooks. Recall that the plain text documents are stored separately from their metadata -- this is the format that you would expect from a Zotero RDF export.
End of explanation
"""


wordcounts_per_document = nltk.ConditionalFreqDist([
    (fileid, normalize_token(token))
     for fileid in documents.fileids()
     for token in documents.words(fileids=[fileid])
     if filter_token(token)
])

from scipy import sparse

# We pick a single "focal" document that we want to characterize.
focal_fileid = documents.fileids()[3]

# Since this procedure will involve numerical matrices, we
#  need to map documents and words onto row and column indices.
#  These "dictionaries" will help us to keep track of those
#  mappings.
document_index = {}    # Maps int -> fileid (str).
vocabulary = {}        # Maps int -> word (str).
lookup = {}            # Reverse map for vocabulary (word (str) -> int).

# Containers for sparse data.
I = []          # Document vector.
J = []          # Word vector.
data = []       # Word count vector.
labels = []     # Vector of labels; either the URI of interest, or "Other".

# Here we transform the ConditionalFrequencyDist into three vectors (I, J, data)
#  that sparsely describe the document-word count matrix.
for i, (fileid, counts) in enumerate(wordcounts_per_document.iteritems()):
    document_index[i] = fileid
    for token, count in counts.iteritems():
        # Removing low-frequency terms is optional, but speeds things up
        #  quite a bit for this demonstration.
        if count < 3:
            continue
        
        # get() lets us 
        j = lookup.get(token, len(vocabulary))
        vocabulary[j] = token
        lookup[token] = j
        
        I.append(i)
        J.append(j)
        data.append(count)
    labels.append(fileid if fileid == focal_fileid else 'Other')
    print '\r', i,

sparse_matrix = sparse.coo_matrix((data, (I, J)))

sparse_matrix.shape

from sklearn.feature_selection import chi2
from sklearn.feature_extraction.text import CountVectorizer


keyness, _ = chi2(sparse_matrix, labels)

ranking = np.argsort(keyness)[::-1]

_, words = zip(*sorted(vocabulary.items(), key=lambda i: i[0]))

words = np.array(words)

keywords = words[ranking]

zip(keywords[:20], keyness[ranking][:20])

def extract_keywords(fileid, n=20):
    print '\r', fileid,
    document_index = {}    # Maps int -> fileid (str).
    vocabulary = {}    # Maps int -> word (str).
    lookup = {}    # Reverse map for vocabulary (word (str) -> int).

    I = []
    J = []
    data = []
    labels = []

    for i, (key, counts) in enumerate(wordcounts_per_document.iteritems()):
        document_index[i] = key
        for token, count in counts.iteritems():
            if count < 3:
                continue
            j = lookup.get(token, len(vocabulary))
            vocabulary[j] = token
            lookup[token] = j

            I.append(i)
            J.append(j)
            data.append(count)
        labels.append(key if key == fileid else 'Other')
        
    sparse_matrix = sparse.coo_matrix((data, (I, J)))
    
    keyness, _ = chi2(sparse_matrix, labels)
    ranking = np.argsort(keyness)[::-1]
    _, words = zip(*sorted(vocabulary.items(), key=lambda i: i[0]))
    words = np.array(words)
    keywords = words[ranking]
    return keywords[:n]

keywords = [extract_keywords(fileid) for fileid in documents.fileids()]

"""
Explanation: Our first step is to count up all of the words in each of the documents. This conditional frequency distribution should look familiar by now.
End of explanation
"""


