import pandas as pd
import numpy as np

"""
Explanation: Cleaning the Data and creating a sparce matrix
I start by importing the basic libraries and the data from a file.
End of explanation
"""


legislatorsData = pd.read_csv("../data/legislators.csv")
legislatorsData.head()
legislatorsData.columns
legislators = pd.DataFrame(legislatorsData)
legislators.head()

from urllib2 import Request, urlopen
import json
from pandas.io.json import json_normalize

def requestWords( id ):
    id = str(id)
    url = "http://capitolwords.org/api/1/phrases.json?entity_type=legislator&entity_value="+id+"&apikey=0bf8e7eb6ce146f48217bfee767c998d"
    request=Request(url)
    response = urlopen(request)
    contents = response.read()
    len(contents)
    if len(contents) > 2:
        data = json.loads(contents)
        words = json_normalize(data)
        list_of_words = words.ngram.tolist()
        string_of_words ="|".join(list_of_words)
        return string_of_words
    else:
        return np.nan

legislators['favorite_words'] = legislators.apply(lambda row: requestWords(row['bioguide_id']),axis=1)

print legislators.favorite_words.head(3)
print "All entries before getting rid of entris with no words:", len(legislators.favorite_words)

legislators_words = legislators[legislators.favorite_words.notnull()]

print "Number of legislators with word record:", len(legislators_words.favorite_words)

favorite_words = legislators_words.favorite_words.str.get_dummies(sep = "|")
print favorite_words.head(3)
favorite_words.columns[:100]

favorite_words.shape

favorite_words.columns[260:300]

favorite_words.columns[760:800]

from sklearn.feature_extraction.text import CountVectorizer
vectorizer = CountVectorizer(min_df=1)
vectorizer

"""
Explanation: This is what the data looks like:
End of explanation
"""


corpus = favorite_words.columns.tolist()
corpus[:3]

X = vectorizer.fit_transform(corpus)

analyze = vectorizer.build_analyzer()
print analyze("economy a this")
vectorizer.get_feature_names()[910:920]

vectorizer.vocabulary_.get('document') #not seen in the training corpus will be completely ignored in future calls to the transform method

vectorizer.transform(['Something completely unrelated']).toarray()

"""
Explanation: Corpus
I used the whole must repited words to have more of a global tf-idf
End of explanation
"""


from sklearn.feature_extraction.text import TfidfTransformer

transformer = TfidfTransformer()
transformer

tfidf = transformer.fit_transform(favorite_words)
tfidf_array = tfidf.toarray()
tfidf_array.shape
tfidf_array[20].max()
transformer.idf_

"""
Explanation: Biagram
not sure if i shoud add Biagrams since the corpus is made up single words and a lot of of number dropping politicians love to do.
Transformer
End of explanation
"""


from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(min_df=1)
vectorizer.fit_transform(corpus)
vec_idf = vectorizer.idf_
print len(vec_idf)

words_weight = pd.DataFrame(tfidf_array, index=legislators_words.index , columns=corpus)
print legislators_words.index
print words_weight.index

capitol_words = legislators_words.merge(words_weight, right_index=True, left_index=True)

capitol_words.head()

del capitol_words["a"]
column_names_capitol = capitol_words.columns.tolist()
word_column_names = column_names_capitol[806:]
number_column_names = column_names_capitol[30:805]

capitol_words[word_column_names].head()

capitol_words[word_column_names].sum().max()

"""
Explanation: Vectorizer
End of explanation
"""


total_word_counts = capitol_words[word_column_names].sum()
max_sum_col = total_word_counts[total_word_counts==total_word_counts.max()]
max_sum_col.index[0]

print "Words before cleanse for maximizing variance:", len(word_column_names)

"""
Explanation: Now I will make another DataFrame removing the words with the max counts
End of explanation
"""


def percentage_global_mention(word_column_names):
    for word in word_column_names:
        not_mentioned_mask = capitol_words[word_column_names][word]==0.0
        not_mentioned_count = capitol_words[word_column_names][word].mask(not_mentioned_mask).count()
        index_count = capitol_words.count()[0]
        percentage = float(not_mentioned_count)/index_count
        if percentage > 0.95:
            print percentage , word
            del capitol_words[word]
        else:
            print "Nothing to worry about"

"""
Explanation: Getting rid of the words that 95% of the people said
Because they don't add much in to analyzing what makes legislators different from one another.
End of explanation
"""



word_frequencies = (capitol_words[word_column_names]>0).astype(int).sum(axis=0).astype(float)/capitol_words.shape[0]
most_frequent_words = word_frequencies[word_frequencies>.95].index
most_frequent_words

word_frequencies = (capitol_words[word_column_names]>0).astype(int).sum(axis=0)
word_frequencies.max()

capitol_words.party_x.unique()
party_mask = capitol_words.party_x!="I"
two_party_words = capitol_words[party_mask]
print "Entries before getting rid of independents:", capitol_words.shape[0]
print "Entries after getting rid of independents:", two_party_words.shape[0]
print "Number of independents:", (capitol_words.shape[0])-(two_party_words.shape[0])

two_party_words.to_csv(path_or_buf="../data/two.csv")

"""
Explanation: Or a better way to do it:
credits to Sergey
End of explanation
"""


