import numpy as np
import pandas as pd
import json
import matplotlib.pyplot as plt
%matplotlib inline

"""
Explanation: Exploring precision and recall
The goal of this second notebook is to understand precision-recall in the context of classifiers.

Use Amazon review data in its entirety.
Train a logistic regression model.
Explore various evaluation metrics: accuracy, confusion matrix, precision, recall.
Explore how various metrics can be combined to produce a cost of making an error.
Explore precision and recall curves.

Because we are using the full Amazon review dataset (not a subset of words or reviews), in this assignment we return to using GraphLab Create for its efficiency. As usual, let's start by firing up GraphLab Create.
Make sure you have the latest version of GraphLab Create (1.8.3 or later). If you don't find the decision tree module, then you would need to upgrade graphlab-create using
pip install graphlab-create --upgrade
See this page for detailed instructions on upgrading.
End of explanation
"""


products = pd.read_csv('amazon_baby.csv')
products.head(2)

print products.dtypes

print len(products)


#products2 = products.fillna({'review':''})  # fill in N/A's in the review column

#print products2
print products[products['review'].isnull()].head(3)
print '\n'
print products.iloc[38]

"""
Explanation: Load amazon review dataset
End of explanation
"""


products = products.fillna({'review':''})  # fill in N/A's in the review column

def remove_punctuation(text):
    import string
    #if type(text) == float:
        #print text
    return text.translate(None, string.punctuation) 


products['review_clean'] = products['review'].apply(remove_punctuation)

products = products[products['rating'] != 3]

products['sentiment'] = products['rating'].apply(lambda rating : +1 if rating > 3 else -1)


products.head(5)

"""
Explanation: Extract word counts and sentiments
As in the first assignment of this course, we compute the word counts for individual words and extract positive and negative sentiments from ratings. To summarize, we perform the following:

Remove punctuation.
Remove reviews with "neutral" sentiment (rating 3).
Set reviews with rating 4 or more to be positive and those with 2 or less to be negative.
End of explanation
"""


products.shape

"""
Explanation: Now, let's remember what the dataset looks like by taking a quick peek:
End of explanation
"""


with open('module-9-assignment-train-idx.json') as train_data_file:    
    train_data_idx = json.load(train_data_file)
with open('module-9-assignment-test-idx.json') as test_data_file:    
    test_data_idx = json.load(test_data_file)

print train_data_idx[:3]
print test_data_idx[:3]

train_data = products.iloc[train_data_idx]
train_data.head(2)

print len(train_data[train_data['sentiment'] == 1])
print len(train_data[train_data['sentiment'] == -1])
print len(train_data)

test_data = products.iloc[test_data_idx]
test_data.head(2)

print len(test_data[test_data['sentiment'] == 1])
print len(test_data[test_data['sentiment'] == -1])
print len(test_data)

print len(train_data) + len(test_data)

"""
Explanation: Split data into training and test sets
We split the data into a 80-20 split where 80% is in the training set and 20% is in the test set.
End of explanation
"""


from sklearn.feature_extraction.text import CountVectorizer

vectorizer = CountVectorizer(token_pattern=r'\b\w+\b')
     # Use this token pattern to keep single-letter words
# First, learn vocabulary from the training data and assign columns to words
# Then convert the training data into a sparse matrix
train_matrix = vectorizer.fit_transform(train_data['review_clean'])
# Second, convert the test data into a sparse matrix, using the same word-column mapping
test_matrix = vectorizer.transform(test_data['review_clean'])


#print vectorizer.vocabulary_

"""
Explanation: Build the word count vector for each review
We will now compute the word count for each word that appears in the reviews. A vector consisting of word counts is often referred to as bag-of-word features. Since most words occur in only a few reviews, word count vectors are sparse. For this reason, scikit-learn and many other tools use sparse matrices to store a collection of word count vectors. Refer to appropriate manuals to produce sparse word count vectors. General steps for extracting word count vectors are as follows:

Learn a vocabulary (set of all words) from the training data. Only the words that show up in the training data will be considered for feature extraction.
Compute the occurrences of the words in each review and collect them into a row vector.
Build a sparse matrix where each row is the word count vector for the corresponding review. Call this matrix train_matrix.
Using the same mapping between words and columns, convert the test data into a sparse matrix test_matrix.

The following cell uses CountVectorizer in scikit-learn. Notice the token_pattern argument in the constructor.
End of explanation
"""


from sklearn.linear_model import LogisticRegression

model = LogisticRegression()
model.fit(train_matrix, train_data['sentiment'])

model.classes_

"""
Explanation: Train a logistic regression classifier
We will now train a logistic regression classifier with sentiment as the target and word_count as the features. We will set validation_set=None to make sure everyone gets exactly the same results.  
Remember, even though we now know how to implement logistic regression, we will use GraphLab Create for its efficiency at processing this Amazon dataset in its entirety.  The focus of this assignment is instead on the topic of precision and recall.
End of explanation
"""


from sklearn.metrics import accuracy_score

accuracy = accuracy_score(y_true=test_data['sentiment'].as_matrix(), y_pred=model.predict(test_matrix))
print "Test Accuracy: %s" % accuracy

"""
Explanation: Model Evaluation
We will explore the advanced model evaluation concepts that were discussed in the lectures.
Accuracy
One performance metric we will use for our more advanced exploration is accuracy, which we have seen many times in past assignments.  Recall that the accuracy is given by
$$
\mbox{accuracy} = \frac{\mbox{# correctly classified data points}}{\mbox{# total data points}}
$$
To obtain the accuracy of our trained models using GraphLab Create, simply pass the option metric='accuracy' to the evaluate function. We compute the accuracy of our logistic regression model on the test_data as follows:
End of explanation
"""


baseline = len(test_data[test_data['sentiment'] == 1]) / float(len(test_data))
print "Baseline accuracy (majority class classifier): %s" % baseline

"""
Explanation: Baseline: Majority class prediction
Recall from an earlier assignment that we used the majority class classifier as a baseline (i.e reference) model for a point of comparison with a more sophisticated classifier. The majority classifier model predicts the majority class for all data points. 
Typically, a good model should beat the majority class classifier. Since the majority class in this dataset is the positive class (i.e., there are more positive than negative reviews), the accuracy of the majority class classifier can be computed as follows:
End of explanation
"""


from sklearn.metrics import confusion_matrix
cmat = confusion_matrix(y_true=test_data['sentiment'].as_matrix(),
                        y_pred=model.predict(test_matrix),
                        labels=model.classes_)    # use the same order of class as the LR model.
print ' target_label | predicted_label | count '
print '--------------+-----------------+-------'
# Print out the confusion matrix.
# NOTE: Your tool may arrange entries in a different order. Consult appropriate manuals.
for i, target_label in enumerate(model.classes_):
    for j, predicted_label in enumerate(model.classes_):
        print '{0:^13} | {1:^15} | {2:5d}'.format(target_label, predicted_label, cmat[i,j])


"""
Explanation: Quiz Question: Using accuracy as the evaluation metric, was our logistic regression model better than the baseline (majority class classifier)?
YES

Test Accuracy: 0.932235421166
Baseline accuracy (majority class classifier): 0.842782577394

Confusion Matrix
The accuracy, while convenient, does not tell the whole story. For a fuller picture, we turn to the confusion matrix. In the case of binary classification, the confusion matrix is a 2-by-2 matrix laying out correct and incorrect predictions made in each label as follows:
+---------------------------------------------+
              |                Predicted label              |
              +----------------------+----------------------+
              |          (+1)        |         (-1)         |
+-------+-----+----------------------+----------------------+
| True  |(+1) | # of true positives  | # of false negatives |
| label +-----+----------------------+----------------------+
|       |(-1) | # of false positives | # of true negatives  |
+-------+-----+----------------------+----------------------+
To print out the confusion matrix for a classifier, use metric='confusion_matrix':
End of explanation
"""


FP = 1451
FN = 808
print 100*FP +1*FN

"""
Explanation: Quiz Question: How many predicted values in the test set are false positives?
1451
Computing the cost of mistakes
Put yourself in the shoes of a manufacturer that sells a baby product on Amazon.com and you want to monitor your product's reviews in order to respond to complaints.  Even a few negative reviews may generate a lot of bad publicity about the product. So you don't want to miss any reviews with negative sentiments --- you'd rather put up with false alarms about potentially negative reviews instead of missing negative reviews entirely. In other words, false positives cost more than false negatives. (It may be the other way around for other scenarios, but let's stick with the manufacturer's scenario for now.)
Suppose you know the costs involved in each kind of mistake: 
1. \$100 for each false positive.
2. \$1 for each false negative.
3. Correctly classified reviews incur no cost.
Quiz Question: Given the stipulation, what is the cost associated with the logistic regression classifier's performance on the test set?
End of explanation
"""


from sklearn.metrics import precision_score
precision = precision_score(y_true=test_data['sentiment'].as_matrix(), 
                            y_pred=model.predict(test_matrix))
print "Precision on test data: %s" % precision

"""
Explanation: Precision and Recall
You may not have exact dollar amounts for each kind of mistake. Instead, you may simply prefer to reduce the percentage of false positives to be less than, say, 3.5% of all positive predictions. This is where precision comes in:
$$
[\text{precision}] = \frac{[\text{# positive data points with positive predicitions}]}{\text{[# all data points with positive predictions]}} = \frac{[\text{# true positives}]}{[\text{# true positives}] + [\text{# false positives}]}
$$
So to keep the percentage of false positives below 3.5% of positive predictions, we must raise the precision to 96.5% or higher. 
First, let us compute the precision of the logistic regression classifier on the test_data.
End of explanation
"""


print 1-precision

"""
Explanation: Quiz Question: Out of all reviews in the test set that are predicted to be positive, what fraction of them are false positives? (Round to the second decimal place e.g. 0.25)
End of explanation
"""


from sklearn.metrics import recall_score
recall = recall_score(y_true=test_data['sentiment'].as_matrix(),
                      y_pred=model.predict(test_matrix))
print "Recall on test data: %s" % recall

print 1

"""
Explanation: Quiz Question: Based on what we learned in lecture, if we wanted to reduce this fraction of false positives to be below 3.5%, we would: (see the quiz)
A complementary metric is recall, which measures the ratio between the number of true positives and that of (ground-truth) positive reviews:
$$
[\text{recall}] = \frac{[\text{# positive data points with positive predicitions}]}{\text{[# all positive data points]}} = \frac{[\text{# true positives}]}{[\text{# true positives}] + [\text{# false negatives}]}
$$
Let us compute the recall on the test_data.
End of explanation
"""


def apply_threshold(probabilities, threshold):
    ### YOUR CODE GOES HERE
    # +1 if >= threshold and -1 otherwise.
    result = np.ones(len(probabilities))
    result[probabilities < threshold] = -1
    
    return result

"""
Explanation: Quiz Question: What fraction of the positive reviews in the test_set were correctly predicted as positive by the classifier?
Quiz Question: What is the recall value for a classifier that predicts +1 for all data points in the test_data?
Precision-recall tradeoff
In this part, we will explore the trade-off between precision and recall discussed in the lecture.  We first examine what happens when we use a different threshold value for making class predictions.  We then explore a range of threshold values and plot the associated precision-recall curve.  
Varying the threshold
False positives are costly in our example, so we may want to be more conservative about making positive predictions. To achieve this, instead of thresholding class probabilities at 0.5, we can choose a higher threshold. 
Write a function called apply_threshold that accepts two things
* probabilities (an SArray of probability values)
* threshold (a float between 0 and 1).
The function should return an SArray, where each element is set to +1 or -1 depending whether the corresponding probability exceeds threshold.
End of explanation
"""


probabilities = model.predict_proba(test_matrix)[:,1]

predictions_with_default_threshold = apply_threshold(probabilities, 0.5)

predictions_with_high_threshold = apply_threshold(probabilities, 0.9)



print predictions_with_default_threshold
print predictions_with_high_threshold
print '\n'
print sum(probabilities >= 0.5)
print sum(probabilities >= 0.9)
print '\n'
print predictions_with_default_threshold * (predictions_with_default_threshold==-1)
print len(predictions_with_default_threshold * (predictions_with_default_threshold==-1))
print '\n'
print np.sum(predictions_with_default_threshold >0)
print np.sum(predictions_with_high_threshold>0)


print "Number of positive predicted reviews (threshold = 0.5): %s" % (predictions_with_default_threshold == 1).sum()

print "Number of positive predicted reviews (threshold = 0.9): %s" % (predictions_with_high_threshold == 1).sum()

"""
Explanation: Run prediction with output_type='probability' to get the list of probability values. Then use thresholds set at 0.5 (default) and 0.9 to make predictions from these probability values.
End of explanation
"""


# Threshold = 0.5
precision_with_default_threshold = precision_score(y_true=test_data['sentiment'].as_matrix(), y_pred=predictions_with_default_threshold)

recall_with_default_threshold = recall_score(y_true=test_data['sentiment'].as_matrix(), y_pred=predictions_with_default_threshold)

# Threshold = 0.9
precision_with_high_threshold = precision_score(y_true=test_data['sentiment'].as_matrix(), y_pred=predictions_with_high_threshold)

recall_with_high_threshold = recall_score(y_true=test_data['sentiment'].as_matrix(), y_pred=predictions_with_high_threshold)

                                      

print "Precision (threshold = 0.5): %s" % precision_with_default_threshold
print "Recall (threshold = 0.5)   : %s" % recall_with_default_threshold

print "Precision (threshold = 0.9): %s" % precision_with_high_threshold
print "Recall (threshold = 0.9)   : %s" % recall_with_high_threshold

"""
Explanation: Quiz Question: What happens to the number of positive predicted reviews as the threshold increased from 0.5 to 0.9?
Exploring the associated precision and recall as the threshold varies
By changing the probability threshold, it is possible to influence precision and recall. We can explore this as follows:
End of explanation
"""


threshold_values = np.linspace(0.5, 1, num=100)
print threshold_values

"""
Explanation: Quiz Question (variant 1): Does the precision increase with a higher threshold?
Quiz Question (variant 2): Does the recall increase with a higher threshold?
Precision-recall curve
Now, we will explore various different values of tresholds, compute the precision and recall scores, and then plot the precision-recall curve.
End of explanation
"""


precision_all = []
recall_all = []

probabilities = model.predict_proba(test_matrix)[:,1]
for threshold in threshold_values:
    predictions = apply_threshold(probabilities, threshold)
    
    precision = precision_score(y_true=test_data['sentiment'].as_matrix(), y_pred=predictions)
    recall = recall_score(y_true=test_data['sentiment'].as_matrix(), y_pred=predictions)
    
    precision_all.append(precision)
    recall_all.append(recall)

"""
Explanation: For each of the values of threshold, we compute the precision and recall scores.
End of explanation
"""


import matplotlib.pyplot as plt
%matplotlib inline

def plot_pr_curve(precision, recall, title):
    plt.rcParams['figure.figsize'] = 7, 5
    plt.locator_params(axis = 'x', nbins = 5)
    plt.plot(precision, recall, 'b-', linewidth=4.0, color = '#B0017F')
    plt.title(title)
    plt.xlabel('Precision')
    plt.ylabel('Recall')
    plt.rcParams.update({'font.size': 16})
    
plot_pr_curve(precision_all, recall_all, 'Precision recall curve (all)')

"""
Explanation: Now, let's plot the precision-recall curve to visualize the precision-recall tradeoff as we vary the threshold.
End of explanation
"""


print np.array(threshold_values)[np.array(precision_all) >= 0.965]

"""
Explanation: Quiz Question: Among all the threshold values tried, what is the smallest threshold value that achieves a precision of 96.5% or better? Round your answer to 3 decimal places.
End of explanation
"""


predictions_with_098_threshold = apply_threshold(probabilities, 0.98)
sth = (np.array(test_data['sentiment'].as_matrix()) > 0) * (predictions_with_098_threshold < 0)
print sum(sth)

cmat_098 = confusion_matrix(y_true=test_data['sentiment'].as_matrix(),
                        y_pred=predictions_with_098_threshold,
                        labels=model.classes_)    # use the same order of class as the LR model.
print ' target_label | predicted_label | count '
print '--------------+-----------------+-------'
# Print out the confusion matrix.
# NOTE: Your tool may arrange entries in a different order. Consult appropriate manuals.
for i, target_label in enumerate(model.classes_):
    for j, predicted_label in enumerate(model.classes_):
        print '{0:^13} | {1:^15} | {2:5d}'.format(target_label, predicted_label, cmat_098[i,j])

"""
Explanation: Quiz Question: Using threshold = 0.98, how many false negatives do we get on the test_data? (Hint: You may use the graphlab.evaluation.confusion_matrix function implemented in GraphLab Create.)
End of explanation
"""


baby_reviews =  test_data[test_data['name'].apply(lambda x: 'baby' in str(x).lower())]

"""
Explanation: This is the number of false negatives (i.e the number of reviews to look at when not needed) that we have to deal with using this classifier.
Evaluating specific search terms
So far, we looked at the number of false positives for the entire test set. In this section, let's select reviews using a specific search term and optimize the precision on these reviews only. After all, a manufacturer would be interested in tuning the false positive rate just for their products (the reviews they want to read) rather than that of the entire set of products on Amazon.
Precision-Recall on all baby related items
From the test set, select all the reviews for all products with the word 'baby' in them.
End of explanation
"""


baby_matrix = vectorizer.transform(baby_reviews['review_clean'])
probabilities = model.predict_proba(baby_matrix)[:,1]

"""
Explanation: Now, let's predict the probability of classifying these reviews as positive:
End of explanation
"""


threshold_values = np.linspace(0.5, 1, num=100)

"""
Explanation: Let's plot the precision-recall curve for the baby_reviews dataset.
First, let's consider the following threshold_values ranging from 0.5 to 1:
End of explanation
"""


precision_all = []
recall_all = []

for threshold in threshold_values:
    
    # Make predictions. Use the `apply_threshold` function 
    ## YOUR CODE HERE 
    predictions = apply_threshold(probabilities, threshold)

    # Calculate the precision.
    # YOUR CODE HERE
    precision = precision_score(y_true=baby_reviews['sentiment'].as_matrix(), y_pred=predictions)
    
    # YOUR CODE HERE
    recall = recall_score(y_true=baby_reviews['sentiment'].as_matrix(), y_pred=predictions)
    
    # Append the precision and recall scores.
    precision_all.append(precision)
    recall_all.append(recall)  

plot_pr_curve(precision_all, recall_all, "Precision-Recall (Baby)")

"""
Explanation: Second, as we did above, let's compute precision and recall for each value in threshold_values on the baby_reviews dataset.  Complete the code block below.
End of explanation
"""


print np.array(threshold_values)[np.array(precision_all) >= 0.965]

"""
Explanation: Quiz Question: Among all the threshold values tried, what is the smallest threshold value that achieves a precision of 96.5% or better for the reviews of data in baby_reviews? Round your answer to 3 decimal places.
End of explanation
"""


plot_pr_curve(precision_all, recall_all, "Precision-Recall (Baby)")

"""
Explanation: larger
Quiz Question: Is this threshold value smaller or larger than the threshold used for the entire dataset to achieve the same specified precision of 96.5%?
Finally, let's plot the precision recall curve.
End of explanation
"""


