import nltk
import numpy as np
import pandas as pd
%matplotlib inline

# pull in the spam dataset
spam = pd.read_csv("spambase.csv")

"""
Explanation: IS620 - Document Classification
Daina Bouquin
Spam Filtering and Classification
End of explanation
"""


# Summary stats
spam.describe()

# Variable types
print(spam.dtypes)

# Count spam and non-spam
count_spam = len(spam[spam.spamclass==1])
count_nonspam = len(spam[spam.spamclass==0])

print "Spam: %d" %count_spam
print "Non-spam: %d" %count_nonspam


"""
Explanation: Summaries
End of explanation
"""


# Split into test, train, validate
percTrain = 0.7
percVal = 0.15
percTest = 0.15
N = len(spam)
trainNum = int(percTrain * N)
valNum = int(percVal * N)
testNum = N - trainNum - valNum

# Check targets
print "Training target: %d" %trainNum
print "Validation target: %d" %valNum
print "Testing target: %d" %testNum
print "Total: %d" %(trainNum + valNum + testNum)

# Split it up
from sklearn.cross_validation import train_test_split
trainSet, testSet = train_test_split(spam, test_size=testNum, random_state=8)
trainSet, valSet = train_test_split(trainSet, test_size=valNum, random_state=88)

# Check lengths
print "Training set: %d" %len(trainSet)
print "Validation set: %d" %len(valSet)
print "Testing set: %d" %len(testSet)
print "Total: %d" %(len(trainSet) + len(valSet) + len(testSet))

"""
Explanation: Split data into two datasets: training & testing
End of explanation
"""


# Random forest - train
from sklearn import ensemble

rf = ensemble.RandomForestClassifier(criterion="entropy", random_state=88)
rf_fit = rf.fit(trainSetVars, trainSetClass)

rf_train = rf_fit.predict(trainSetVars)
model_summary(trainSetClass, rf_train)

# Random forest - test set
rf_test = rf_fit.predict(testSetVars)
model_summary(testSetClass, rf_test)

# Random forest feature importance
featImp(rf_fit, testSetVars)

"""
Explanation: Random forest
http://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html
End of explanation
"""


