folder = 'hillary-clinton-emails/'

emails = pd.read_csv(folder + 'Emails.csv', index_col='Id')
emails.head(5)

"""
Explanation: Import data
End of explanation
"""


emails.head()

"""
Explanation: Analyse Emails
End of explanation
"""


emails.columns

print('Number of emails: ', len(emails))

bodyNaN = emails.ExtractedBodyText.isnull().sum()
print('Number of emails with ExtractedBodyText=NaN: {}, ({:.2f}%)'.format(emails.ExtractedBodyText.isnull().sum(), bodyNaN/ len(emails)))

"""
Explanation: The columns ExtractedBodyText is supposed to be the content of the mail but some of the mail have a ExtractedBodyText = NaN but the Rawtext seems to contains something
End of explanation
"""


bodyNaN = emails.ExtractedSubject.isnull().sum()
print('Number of emails with ExtractedSubject=NaN: {}, ({:.2f}%)'.format(emails.ExtractedBodyText.isnull().sum(), bodyNaN/ len(emails)))

"""
Explanation: We could also use the subject since it is usually a summary of the mail
End of explanation
"""


subBodyNan = emails[np.logical_and(emails.ExtractedBodyText.isnull(),emails.ExtractedSubject.isnull())]
print('Number of email where both subject and body is NaN: {}({:.2f})'.format(len(subBodyNan), len(subBodyNan)/ len(emails)))

"""
Explanation: Now let's try to combine the subject and the body and drop the mail that have both subject= NaN and body = Nan
End of explanation
"""


emails = emails[~ np.logical_and(emails.ExtractedBodyText.isnull(), emails.ExtractedSubject.isnull())]
len(emails)

emails.ExtractedBodyText.fillna('',inplace=True)
emails.ExtractedSubject.fillna('',inplace=True)
emails['SubjectBody'] = emails.ExtractedBodyText + emails.ExtractedSubject
emails.SubjectBody.head()

"""
Explanation: Well, that number is small enough to drop all email where both Extracted subject and Extracted body is NaN.
Let's drop them and create a new columns subjectBody that is the concatenation of the 2 columns ExtractedSubject and ExtractedBody. From now we will work with that columns
End of explanation
"""


print('Number of NaN in columns SubjectBody: ' ,emails.SubjectBody.isnull().sum())

"""
Explanation: Last check to be sur that our columns of interest don't have anymore NaN
End of explanation
"""


list(pycountry.countries)[0]

"""
Explanation: Keep only mail that mentions a country
Structure of a country in pycountry.countres
End of explanation
"""


emails.SubjectBody.head(100).apply(print)

"""
Explanation: First we create a dataframe with one line by countries and we count for each countries its occurences in the mail.
Since a country can be reference in many way (Switzerland, switzerland, CH), we need to consider all the possible form. 
We may have a problem with word that have many meaning like US(country) and us (pronoun) so we can't just take all the country name in loer case and all the mail in lower case and just compare.
We first try to use that technique:
    1. the country name can appear either in lower case, with the first letter in uppercase or all in uppercase
    2. alpha_2 and alpha_3 are always used in uppercase
But we still have a lot of problem. Indeed a lot of mail contain sentences all in upper cas (see below):
       - SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-SCB0045012
For example this sentence will match for Togo because of the TO and also for norway because of the NO. An other example is Andorra that appears in 55 mails thanks to AND
At first we also wanted to keep the upper case since it can be helpfull to do the sentiment analysis. Look at those 2 sentence and their corresponding score:
    - VADER is very smart, handsome, and funny.: compound: 0.8545, neg: 0.0, neu: 0.299, pos: 0.701,
    - VADER is VERY SMART, handsome, and FUNNY.: compound: 0.9227, neg: 0.0, neu: 0.246, pos: 0.754,
The score is not the same. But since we have a lot of information in Upper case and it nothing to do with sentiment, we will put all mails in lower case. And we will also remove the stopwords. 
We know that we remove the occurance of USA under 'us' but it will also remove the 'and', 'can', 'it',...
End of explanation
"""



from gensim import corpora, models, utils
from nltk.corpus import stopwords
sw = stopwords.words('english') + ['re', 'fw', 'fvv', 'fwd']
sw = sw + ['pm', "a", "about", "above", "above", "across", "after", "afterwards", "again", "against", "all", "almost", "alone", "along", "already", "also","although","always","am","among", "amongst", "amoungst", "amount",  "an", "and", "another", "any","anyhow","anyone","anything","anyway", "anywhere", "are", "around", "as",  "at", "back","be","became", "because","become","becomes", "becoming", "been", "before", "beforehand", "behind", "being", "below", "beside", "besides", "between", "beyond", "bill", "both", "bottom","but", "by", "call", "can", "cannot", "cant", "co", "con", "could", "couldnt", "cry", "de", "describe", "detail", "do", "done", "down", "due", "during", "each", "eg", "eight", "either", "eleven","else", "elsewhere", "empty", "enough", "etc", "even", "ever", "every", "everyone", "everything", "everywhere", "except", "few", "fifteen", "fify", "fill", "find", "fire", "first", "five", "for", "former", "formerly", "forty", "found", "four", "from", "front", "full", "further", "get", "give", "go", "had", "has", "hasnt", "have", "he", "hence", "her", "here", "hereafter", "hereby", "herein", "hereupon", "hers", "herself", "him", "himself", "his", "how", "however", "hundred", "ie", "if", "in", "inc", "indeed", "interest", "into", "is", "it", "its", "itself", "keep", "last", "latter", "latterly", "least", "less", "ltd", "made", "many", "may", "me", "meanwhile", "might", "mill", "mine", "more", "moreover", "most", "mostly", "move", "much", "must", "my", "myself", "name", "namely", "neither", "never", "nevertheless", "next", "nine", "no", "nobody", "none", "noone", "nor", "not", "nothing", "now", "nowhere", "of", "off", "often", "on", "once", "one", "only", "onto", "or", "other", "others", "otherwise", "our", "ours", "ourselves", "out", "over", "own","part", "per", "perhaps", "please", "put", "rather", "re", "same", "see", "seem", "seemed", "seeming", "seems", "serious", "several", "she", "should", "show", "side", "since", "sincere", "six", "sixty", "so", "some", "somehow", "someone", "something", "sometime", "sometimes", "somewhere", "still", "such", "system", "take", "ten", "than", "that", "the", "their", "them", "themselves", "then", "thence", "there", "thereafter", "thereby", "therefore", "therein", "thereupon", "these", "they", "thickv", "thin", "third", "this", "those", "though", "three", "through", "throughout", "thru", "thus", "to", "together", "too", "top", "toward", "towards", "twelve", "twenty", "two", "un", "under", "until", "up", "upon", "us", "very", "via", "was", "we", "well", "were", "what", "whatever", "when", "whence", "whenever", "where", "whereafter", "whereas", "whereby", "wherein", "whereupon", "wherever", "whether", "which", "while", "whither", "who", "whoever", "whole", "whom", "whose", "why", "will", "with", "within", "without", "would", "yet", "you", "your", "yours", "yourself", "yourselves", "the"]

def fil(row):
    t = utils.simple_preprocess(row.SubjectBody)
    filt = list(filter(lambda x: x not in sw, t))
    return ' '.join(filt)
emails['SubjectBody'] = emails.apply(fil, axis=1)
emails.head(10)

countries = np.array([[country.name.lower(), country.alpha_2.lower(), country.alpha_3.lower()] for country in list(pycountry.countries)])
countries[:5]

countries.shape

countries = pd.DataFrame(countries, columns=['Name', 'Alpha_2', 'Alpha_3'])
countries.head()



countries.shape

countries.isin(['aruba']).any().any()

def check_country(row):
    return countries.isin(row.SubjectBody.split()).any().any()
    

emails_country = emails[emails.apply(check_country, axis=1)]
len(emails_country)


"""
Explanation: Tokenize and remove stopwords
End of explanation
"""


sentiments = pd.DataFrame(emails_country.SubjectBody)
sentiments.head()

sentiments.shape

"""
Explanation: Sentiment analysis
We explain before our precessing. Now we will do the sentiment analysis only on the subject and the body
So we will only consider the subject and the body
End of explanation
"""


sentiments.head()


from nltk.sentiment.vader import SentimentIntensityAnalyzer
sid = SentimentIntensityAnalyzer()
def sentiment_analysis(row):
    score = sid.polarity_scores(row)
    return pd.Series({'Pos': score['pos'], 'Neg': score['neg'], 'Compound_':score['compound'] })

sentiments = pd.concat([sentiments, sentiments.SubjectBody.apply(sentiment_analysis)], axis=1)

sentiments.to_csv('mailScore.csv')
sentiments.head()

"""
Explanation: Analysis
We will do a sentiment analysis on each sentense and then compute a socre for each country
We will compare different module:
    - nltk.sentiment.vader that attribute a score to each sentence
    - liuhu that has a set of positive word and one of neg word. We count the positif word and neg word in each sentence and compute the mean
Vader (That part takes time)
End of explanation
"""


from nltk.corpus import opinion_lexicon
sentimentsLihuh = pd.read_csv('mailScore.csv', index_col='Id')
#transform the array of positiv and negatif word in dict
dicPosNeg =  dict()
for word in opinion_lexicon.positive():
    dicPosNeg[word] = 1
    
for word in opinion_lexicon.negative():
    dicPosNeg[word] = -1   
    

def sentiment_liuhu(sentence):
    counter = []
    for word in sentence.split():
        value = dicPosNeg.get(word, -999)
        if value != -999:
            counter.append(value)
            
    if len(counter) == 0 :
        return pd.Series({'Sum_': int(0), 'Mean_': int(0) })
    return pd.Series({'Sum_': np.sum(counter), 'Mean_': np.mean(counter) })

sentimentsLihuh = pd.concat([sentimentsLihuh, sentimentsLihuh.SubjectBody.apply(sentiment_liuhu)], axis=1)

sentimentsLihuh.to_csv('mailScore2.csv')
sentimentsLihuh

"""
Explanation: Liuhu
End of explanation
"""


sentiments = pd.read_csv('mailScore2.csv', index_col='Id')
sentiments.head()

def aggScoreByCountry(country):
    condition = sentiments.apply(lambda x: np.any(country.isin(x.SubjectBody.split())), axis=1)
    sent = sentiments[condition]
    if len(sent) == 0:
        print(country.Name, -999)
        return pd.Series({'Compound_':-999, 'Mean_':-999, 'Appearance': int(len(sent))})
    compound_ = np.mean(sent.Compound_)
    mean_ = np.mean(sent.Mean_)
    print(country.Name, compound_)
    return pd.Series({'Compound_': compound_, 'Mean_': mean_, 'Appearance': int(len(sent))})

countries = pd.concat([countries, countries.apply(lambda x: aggScoreByCountry(x), axis=1)],axis=1)
countries.to_csv('countriesScore.csv')

"""
Explanation: Aggregate by countries
We groupe by country and compute the mean of each score
End of explanation
"""


countries = countries[countries.Compound_ != -999]
len(countries)

"""
Explanation: Drop all country that have a score of -999 (they never appear in the mails)
End of explanation
"""


minimum_appearance = 15
countries_min = countries[countries.Appearance >= minimum_appearance]
len(countries_min)

"""
Explanation: It's a lot of country. We will also use a thresold for the appearance. We only keep mails that are mentioned in a minimum number of emails
End of explanation
"""


# Set up colors : red to green
countries_sorted = countries_min.sort(columns=['Compound_'])
plt.figure(figsize=(16, 6), dpi=80)


appearance = np.array(countries_sorted.Appearance)
colors = cm.RdYlGn((appearance / float(max(y))))
plot = plt.scatter(appearance, appearance, c=appearance, cmap = 'RdYlGn')
plt.clf()
colorBar = plt.colorbar(plot)
colorBar.ax.set_title("Appearance")


index = np.arange(len(countries_sorted))
bar_width = 0.95
plt.bar(range(countries_sorted.shape[0]), countries_sorted.Compound_, align='center', tick_label=countries_sorted.Name, color=colors)
plt.xticks(rotation=90, ha='center')
plt.title('Using Vader')
plt.xlabel('Countries')
plt.ylabel('Vador Score')


countries_sorted = countries_min.sort(columns=['Mean_'])

plt.figure(figsize=(16, 6), dpi=80)

appearance = np.array(countries_sorted.Appearance)
colors = cm.RdYlGn((appearance / float(max(y))))
plot = plt.scatter(appearance, appearance, c=appearance, cmap = 'RdYlGn')
plt.clf()
colorBar = plt.colorbar(plot)
colorBar.ax.set_title("Appearance")


index = np.arange(len(countries_sorted))
bar_width = 0.95
plt.bar(range(countries_sorted.shape[0]), countries_sorted.Mean_, align='center', tick_label=countries_sorted.Name, color=colors)
plt.xticks(rotation=90, ha='center')
plt.title('Liuhu Score')
plt.xlabel('Countries')
plt.ylabel('Liuhu Score')




"""
Explanation: Plot
We plot the 2 analysis. The first plot show an historgram with the vador score and in color the appearance in the mail.
In the second plot the histogram shows the liuhu score and in color the appearance in the mail
we only consider countries that are at least mention 15 times. Otherwise we end up with to much countries
End of explanation
"""


