# load the iris dataset as an example
from sklearn.datasets import load_iris
iris = load_iris()

# store the feature matrix (X) and response vectore (y)
X = iris.data
y = iris.target

"""
Explanation: Part 1: Model building in scikit-learn
End of explanation
"""


# check the shapes of X and y
print(X.shape)
print(y.shape)

"""
Explanation: "Features" are also known as predictors, inputs or attributes. The "reponse" is also known as the target, label or output.
End of explanation
"""


# examine the first 5 rows of the feature matrix
import pandas as pd
pd.DataFrame(X, columns=iris.feature_names).head()

y[:5]

pd.Series(y).value_counts()

"""
Explanation: "Observations" are also known as samples, instances, or records.
End of explanation
"""


# import the class
from sklearn.neighbors import KNeighborsClassifier

# instantiate the model (with the default parameters)
knn = KNeighborsClassifier()

# fit the model with data (occurs in-place)
knn.fit(X, y)

"""
Explanation: In order to build a model, the features must be numeric, and every observation must have the same features in the same order.
End of explanation
"""


# predict the response for a new observation
knn.predict([[3, 5, 4, 2]])

"""
Explanation: In order to make a prediction, the new observation must have the same features as the training observations, both in number and meaning.
End of explanation
"""


# example text for model training (SMS messages)
simple_train = ['call you tonight', 'Call me a cab', 'please call me... PLEASE!']

# import and instantiate CountVectorizer (with the default parameters)
from sklearn.feature_extraction.text import CountVectorizer
vect = CountVectorizer()

vect.fit(simple_train)

vect.get_feature_names()

# transform training data into a 'document-term matrix'
simple_train_dtm = vect.transform(simple_train)
simple_train_dtm

# convert sparse matrix to a dense matrix
simple_train_dtm.toarray()

# examine the vocabulary and document-term matrix together
pd.DataFrame(simple_train_dtm.toarray(), columns=vect.get_feature_names())

# examine the sparse matrix contents
print(simple_train_dtm)

"""
Explanation: Part 2: Representing text as numerical data
End of explanation
"""


# example text for model testing
simple_test = ["please don't call me"]

"""
Explanation: As most documents will typically use a very small subset of the words used in the corpus, the resulting matrix will have many feature values that are zeros (typically more than 99% of them).
For instance, a collection of 10,000 short text documents (such as emails) will use a vocabulary with a size in the order of 100,000 unique words in total while each document will use 100 to 1000 unique words individually.
In order to be able to store such a matrix in memory but also to speed up operations, implementations will typically use a sparse representation such as the implementations available in the  scipy.sparse package.
End of explanation
"""


# transform testing data into a document-term matrix (using existing vocabulary)
simple_test_dtm = vect.transform(simple_test)
simple_test_dtm.toarray()

"""
Explanation: In order to make a prediction, the new observation must have the same features as the training observations, both in number and meaning.
End of explanation
"""


# read file into pandas using a relative path
# alternative: read file into pandas from a URL
# url = 'https://raw.githubusercontent.com/justmarkham/pycon-2016-tutorial/master/data/sms.tsv'
path = 'sms.tsv'
sms = pd.read_table(path, header=None, names=['label', 'message'])

# examine the shape
sms.shape

# examine the first 10 rows
sms.head(10)

# examine the calss distribution
sms.label.value_counts()

# examine the calss distribution
sms.label.value_counts() * 100 / sms.shape[0]

# convert label to a numerical variable
sms['label_num'] = sms.label.map({
    'ham' : 0,
    'spam': 1
})

# check that the conversion worked
sms.head(10)

# define X and y from the SMS data for use with CountVectorizer
X = sms.message
y = sms.label_num
print(X.shape)
print(y.shape)

# split X, y into training and testing sets
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1087)

print(X_train.shape)
print(X_test.shape)
print(y_train.shape)
print(y_test.shape)

"""
Explanation: Summary:

vect.fit(train) learns the vocabulary of the training data
vect.transform(train) uses the fitted vocabulary to build a document-term matrix from the training data
vect.transform(test) uses the fitted vocabulary to build a document-term matrix from the testing data (and ignores tokens it hasn't seen before)

Part 3: Reading a text-based dataset into pandas
End of explanation
"""


# instantiate the vectorizer
vect = CountVectorizer()

# learn training data vocabulary, then use it to create a document-term matrix
vect.fit(X_train)
X_train_dtm = vect.transform(X_train)

# examine the document matrix
X_train_dtm

# transform testing data (using fitted vocabulary) into a document-term matrix
X_test_dtm = vect.transform(X_test)
X_test_dtm

"""
Explanation: Part 4: Vectorizing our dataset
End of explanation
"""


# import and instatiate a multinomial naive Bayes model
from sklearn.naive_bayes import MultinomialNB
nb = MultinomialNB()

# train the model using X_train_dtm (timing it)
%time nb.fit(X_train_dtm, y_train)

# make class prediction for X_test_dtm
y_pred_class = nb.predict(X_test_dtm)

# calculate accuracy of class predictions
from sklearn import metrics
metrics.accuracy_score(y_test, y_pred_class)

# print the confusion matrix
metrics.confusion_matrix(y_test, y_pred_class)

# print message text for the false positives (ham incorrectly classified as spam)
X_test[ y_test < y_pred_class]

# print message text for false negatives (spam incorrectly classified as ham)
X_test[y_test > y_pred_class]

# example false negative
X_test[5037]

# calculate predicted probabilities for X_test_dtm (poorly calibrated)
y_pred_prob = nb.predict_proba(X_test_dtm)[:, 1]
y_pred_prob

# calculate AUC
metrics.roc_auc_score(y_test, y_pred_prob)

"""
Explanation: Part 5: Building and evaluation of a model
End of explanation
"""


# import and instantiate a logistic regression model
from sklearn.linear_model import LogisticRegression
lreg = LogisticRegression()

# train the model using X_train_dtm
%time lreg.fit(X_train_dtm, y_train)

# make class predictions using X_test_dtm
y_pred_class = lreg.predict(X_test_dtm)

# calculate predicted probabilities for X_test_dtm (well calibrated)
y_pred_prob = lreg.predict_proba(X_test_dtm)[:, 1]
y_pred_prob 

# calculate accuracy
metrics.accuracy_score(y_test, y_pred_class)

# calculate AUC
metrics.roc_auc_score(y_test, y_pred_prob)

"""
Explanation: Part 6: Comparing models
We will compare multinomial naive Bayes with logistic regression
End of explanation
"""


