
testarticles = [1957284403,1965754064,233465322,283147769,362778020,37793736,389321649,540607195,60134403,887344770, ]
all_articles = []
all_sets_sentences = []
all_article_sentences = []
all_comments_sentences = []
all_groups = [] # each comment is a group, the article counts as a comment
for art in testarticles:
    ext = Extractor("../test_data/{0}.ofs.gold.xml".format(art))
    article = ext.extract(verbose=True)
    all_articles.append(article)
    
    df_article = pd.DataFrame.from_dict(article['sentences'], orient='index')
    sorted_indexes = [ "s{0}".format(x) for x in range(len(article['sentences'].values()))]
    sentences = list(df_article.ix[sorted_indexes, 'text'])
    
    ordered_sentences = df_article.ix[sorted_indexes]
    article_sentences_ix = ordered_sentences.ix[:,'comment'].isnull()
    
    art_sentences = ordered_sentences[article_sentences_ix]
    com_sentences = ordered_sentences[article_sentences_ix == False]
    
    article_sentences = list(art_sentences['text'])
    comment_sentences = list(com_sentences['text'])
    
    groups_of_sentences = ordered_sentences.groupby(by='comment', sort=False)
    groupcount = len(groups_of_sentences.groups.keys())
    index = ["c{0}".format(i) for i in range(groupcount)]
    grouped_comments = []
    for g in index:
        com = " | ".join(list(groups_of_sentences.get_group(g)['text']))
        grouped_comments.append(com)

    grouped_comments.append(" | ".join(article_sentences))
    
    all_article_sentences.append(article_sentences)
    all_comments_sentences.append(comment_sentences)
    all_groups.append(grouped_comments)
    
    if df_article.ix['s2', 'text'] == sentences[2]:
        print "Extracted list of sentences is in a proper order."
        all_sets_sentences.append(sentences)
    else:
        print "Extracted list of sentences is unordered."


"""
Explanation: Get some data
End of explanation
"""


all_ranked_sentences = []
all_matrxs = []
all_ranked_art_sentences = []
all_art_matrxs = []
all_ranked_com_sentences = []
all_com_matrxs = []

def lexrank_all_lists_of_sentences(sentences_lists):
    all_ranked = []
    all_matrixes = []
    for sentences in sentences_lists:
        ranked, matrix = lexrank.lexrank(sentences)
        all_ranked_sentences.append(ranked)
        all_matrixes.append(matrix)
    return all_ranked, all_matrixes

all_ranked_sentences, all_matrxs = lexrank_all_lists_of_sentences(
    all_sets_sentences
)

all_ranked_art_sentences, all_art_matrxs = lexrank_all_lists_of_sentences(
    all_article_sentences
)

all_ranked_com_sentences, all_com_matrxs = lexrank_all_lists_of_sentences(
    all_comments_sentences
)


"""
Explanation: Feed data into lexrank
The summpy MIT licensed repo used for this task, returns a tuple with a dictionary with sentences index + score, and the similarity_matrix.
End of explanation
"""


all_ranked_dfs = []
all_similarity_dfs = []
for ranked_sentences, similarity_mtrx in zip(all_ranked_sentences, all_matrxs):
    df_ranked_sentences = pd.DataFrame.from_dict(ranked_sentences,
                                                 orient='index')
    df_similarity_mtrx = pd.DataFrame(similarity_mtrx)
    all_ranked_dfs.append(df_ranked_sentences)
    all_similarity_dfs.append(df_similarity_mtrx)

all_similarity_dfs[0]

"""
Explanation: See the data
End of explanation
"""


import numpy as np

from sklearn.cluster import DBSCAN
from sklearn import metrics

def print_results(db):
    core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
    core_samples_mask[db.core_sample_indices_] = True
    labels = db.labels_

    # Number of clusters in labels, ignoring noise if present.
    n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)

    print('Estimated number of clusters: %d' % n_clusters_)

for i, (article, comments) in enumerate(zip(all_art_matrxs, all_com_matrxs)):    
    db_a = DBSCAN(metric='precomputed', eps=0.3, min_samples=2).fit(article)
    db_c = DBSCAN(metric='precomputed', eps=0.3, min_samples=2).fit(comments)
    
    print "\nARTICLE {0}".format(i)
    print_results(db_a)
    print "\nCOMMENTS {0}".format(i)
    print_results(db_c)

"""
Explanation: Cluster Similarity Matrixes DBSCAN
It requires the matrix to be normalized between 0 to 1, but the cosine similarity is already that. Also, the similarity matrix needs to have distances, which is done by doing 1 - pairwise_similarity (this is already taken care off by the lexrank algorithm)
End of explanation
"""


from sklearn.feature_extraction.text import TfidfVectorizer
import nltk.stem

english_stemmer = nltk.stem.SnowballStemmer('english')

class StemmedTfidfVectorizer(TfidfVectorizer):
    def build_analyzer(self):
        analyzer=super(StemmedTfidfVectorizer,self).build_analyzer()
        return lambda doc:(english_stemmer.stem(w) for w in analyzer(doc))

vectorizer = StemmedTfidfVectorizer(min_df=1, stop_words='english')



article_s =  all_article_sentences
comment_s = all_comments_sentences
for i, (article, comments) in enumerate(zip(article_s, comment_s)): 
    X_article = vectorizer.fit_transform(article)
    X_comments = vectorizer.fit_transform(comments)
    db_a = DBSCAN(eps=0.3, min_samples=2).fit(X_article)
    db_c = DBSCAN(eps=0.3, min_samples=2).fit(X_comments)
    
    print "\nARTICLE {0}".format(i)
    print_results(db_a)
    print "\nCOMMENTS {0}".format(i)
    print_results(db_c)

"""
Explanation: Vectorize sentences just to check if it is better if I do this myself other than lexrank
End of explanation
"""


for i, group in enumerate(all_groups): 
    X_group = vectorizer.fit_transform(group)

    db_a = DBSCAN(eps=1, min_samples=2).fit(X_group)
    
    print "\nArticle {0}'s groups".format(i)
    print_results(db_a)

"""
Explanation: Don't cluster sentences, cluster comments
Count the article as a comment.
End of explanation
"""


from gensim import corpora, models, similarities
from nltk.tokenize import TweetTokenizer
import nltk.stem

def preprocess_docs(documents):
    tokenizer = TweetTokenizer()
    english_stemmer = nltk.stem.SnowballStemmer('english')

    texts = [tokenizer.tokenize(d) for d in documents]

    stemmed_texts = []
    for text in texts:
        stemmed_text = [english_stemmer.stem(t) for t in text]
        stemmed_texts.append(stemmed_text)
    return stemmed_texts

def query_all(lsi_queries, index):
    total_pairs = 0
    selected_pairs = 0
    strong_sentences = 0
    similarity_dict = {}
    for i, query in enumerate(lsi_queries):
        sims = index[query]
        strong_sims = [s for s in list(enumerate(sims)) if s[1] > 0.99]
        pairs = len(strong_sims)
        if pairs > 1 and pairs < 8:
#             print "\ns{0} similarities with comments".format(i+comment_start_index)
#             print "{0} strong similarity pairs".format(len(strong_sims))
#             print all_comments_sentences[0][i]
            strong_sentences += 1
            selected_pairs += pairs
            similarity_dict[i] = strong_sims

        total_pairs += pairs
    print "\n{0} strong sentences".format(strong_sentences)
    print "{0} total sentence-group pairs".format(selected_pairs)
    print total_pairs/float(len(lsi_queries)) 
    return similarity_dict

def find_links_between_in(documents, comments_sentences,
                          article_sentences):    
    stemmed_texts = preprocess_docs(documents)
    dictionary = corpora.Dictionary(stemmed_texts)
    dictionary.filter_extremes(no_below=1, no_above=0.8)
    corpus = [dictionary.doc2bow(text) for text in stemmed_texts]

    lsi = models.LsiModel(corpus, id2word=dictionary, num_topics=2)
    index = similarities.MatrixSimilarity(lsi[corpus])

#     comment_start_index = len(article_sentences)
    stemmed_queries = preprocess_docs(comments_sentences)
    query_dict = corpora.Dictionary(stemmed_queries)
    lsi_queries = [lsi[query_dict.doc2bow(text)] for text in stemmed_queries]
    similarity_dict = query_all(lsi_queries, index)
    return similarity_dict

documents = all_groups[0]
comments_s = all_comments_sentences[0]
article_s = all_article_sentences[0]

def find_links_in_all_articles():
    for i, docs in enumerate(all_groups):
        comments = all_comments_sentences[i]
        article = all_article_sentences[i]
        print "\n\nARTICLE {0}".format(i)
        find_links_between_in(docs, comments, article)


% time find_links_in_all_articles()

"""
Explanation: Skip the clustering for grouping, use comments and the article as groups
End of explanation
"""


def find_sentence_links_in_all_articles():
    all_similarity_dicts = []
    for i, docs in enumerate(all_sets_sentences):
        comments = all_comments_sentences[i]
        article = all_article_sentences[i]
        print "\n\nARTICLE {0}".format(i)
        s_dict = find_links_between_in(docs, comments, article)
        all_similarity_dicts.append(s_dict)
    return all_similarity_dicts

% time all_similarity_dicts = find_sentence_links_in_all_articles()

def output_top_sentence_pairs(s_dict, all_art_sentences,
                              all_sentences, 
                              all_comment_sentences):
    d = s_dict
    comment_start_index = len(all_art_sentences)
    sentences = all_sentences
    comment_sentences = all_comment_sentences
    
    for comment_sentence, links in zip(d.keys(),d.values()):

        s1_id = "s{0}".format(comment_sentence + comment_start_index)
        s2_id = "s{0}".format(links[0][0])
        print "\nLink found"
        print comment_sentence+comment_start_index, [l[0] for l in links]        
        print "s{0} is:\n{1}\nSimilar too:".format(comment_sentence+comment_start_index,
                                                  comment_sentences[comment_sentence].encode('utf8'))
        for i, (l_id, prob) in enumerate(links):
            print "S{0}: {1}".format(l_id,sentences[l_id].encode('utf8'))

for i, _ in enumerate(all_articles):
    print "\nARTICLE {0}==========".format(i)
    output_top_sentence_pairs(all_similarity_dicts[i],
                              all_article_sentences[i],
                              all_sets_sentences[i],
                              all_comments_sentences[i])

df_frame = pd.DataFrame.from_dict(all_articles[9]['sentences'], orient='index')
from textblob import TextBlob

s = all_articles[9]['sentences']['s153']
b = TextBlob(s['text'])
b.sentiment

"""
Explanation: Sentence-sentence pairs with gensim
End of explanation
"""


