# Load libraries
import matplotlib.pyplot as plt
import numpy as np
from sklearn.datasets import load_digits
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import validation_curve

"""
Explanation: Title: Plot The Validation Curve
Slug: plot_the_validation_curve
Summary: How to plot the validation curve in scikit-learn for machine learning in Python.  
Date: 2017-09-14 12:00
Category: Machine Learning
Tags: Model Evaluation
Authors: Chris Albon
<a alt="Plot The Validation Curve" href="https://machinelearningflashcards.com">
    <img src="plot_the_validation_curve/Validation_Curve_print.png" class="flashcard center-block">
</a>
Preliminaries
End of explanation
"""


# Load data
digits = load_digits()

# Create feature matrix and target vector
X, y = digits.data, digits.target

"""
Explanation: Load Digits Dataset
End of explanation
"""


# Create range of values for parameter
param_range = np.arange(1, 250, 2)

# Calculate accuracy on training and test set using range of parameter values
train_scores, test_scores = validation_curve(RandomForestClassifier(), 
                                             X, 
                                             y, 
                                             param_name="n_estimators", 
                                             param_range=param_range,
                                             cv=3, 
                                             scoring="accuracy", 
                                             n_jobs=-1)


# Calculate mean and standard deviation for training set scores
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)

# Calculate mean and standard deviation for test set scores
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)

# Plot mean accuracy scores for training and test sets
plt.plot(param_range, train_mean, label="Training score", color="black")
plt.plot(param_range, test_mean, label="Cross-validation score", color="dimgrey")

# Plot accurancy bands for training and test sets
plt.fill_between(param_range, train_mean - train_std, train_mean + train_std, color="gray")
plt.fill_between(param_range, test_mean - test_std, test_mean + test_std, color="gainsboro")

# Create plot
plt.title("Validation Curve With Random Forest")
plt.xlabel("Number Of Trees")
plt.ylabel("Accuracy Score")
plt.tight_layout()
plt.legend(loc="best")
plt.show()

"""
Explanation: Plot Validation Curve
End of explanation
"""


