#  --- SECTION 1 ---
# Import the required libraries
from sklearn import datasets, naive_bayes, svm, neighbors
from sklearn.ensemble import VotingClassifier
from sklearn.metrics import accuracy_score
# Load the dataset
breast_cancer = datasets.load_breast_cancer()
x, y = breast_cancer.data, breast_cancer.target

# Split the train and test samples
test_samples = 100
x_train, y_train = x[:-test_samples], y[:-test_samples]
x_test, y_test = x[-test_samples:], y[-test_samples:]

#  --- SECTION 2 ---
# Instantiate the learners (classifiers)
learner_1 = neighbors.KNeighborsClassifier(n_neighbors=5)
learner_2 = naive_bayes.GaussianNB()
learner_3 = neighbors.KNeighborsClassifier(n_neighbors=50)

#  --- SECTION 3 ---
# Instantiate the voting classifier
voting = VotingClassifier([('5NN', learner_1),
                           ('NB', learner_2),
                           ('50NN', learner_3)],
                            voting='soft')




#  --- SECTION 4 ---
# Fit classifier with the training data
voting.fit(x_train, y_train)
learner_1.fit(x_train, y_train)
learner_2.fit(x_train, y_train)
learner_3.fit(x_train, y_train)

#  --- SECTION 5 ---
# Predict the most probable class
hard_predictions = voting.predict(x_test)

#  --- SECTION 6 ---
# Get the base learner predictions
predictions_1 = learner_1.predict(x_test)
predictions_2 = learner_2.predict(x_test)
predictions_3 = learner_3.predict(x_test)

#  --- SECTION 7 ---
# Accuracies of base learners
print('L1:', accuracy_score(y_test, predictions_1))
print('L2:', accuracy_score(y_test, predictions_2))
print('L3:', accuracy_score(y_test, predictions_3))
# Accuracy of hard voting
print('-'*30)
print('Hard Voting:', accuracy_score(y_test, hard_predictions))

#  --- SECTION 1 ---
# Import the required libraries
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.style.use('seaborn-paper')


#  --- SECTION 2 ---
# Get the wrongly predicted instances
# and the predicted probabilities for the whole test set
errors = y_test-hard_predictions

probabilities_1 = learner_1.predict_proba(x_test)
probabilities_2 = learner_2.predict_proba(x_test)
probabilities_3 = learner_3.predict_proba(x_test)


#  --- SECTION 2 ---
# Store the predicted probability for
# each wrongly predicted instance, for each base learner
# as well as the average predicted probability
#
x=[]
y_1=[]
y_2=[]
y_3=[]
y_avg=[]

for i in range(len(errors)):
    if not errors[i] == 0:
        x.append(i)
        y_1.append(probabilities_1[i][0])
        y_2.append(probabilities_2[i][0])
        y_3.append(probabilities_3[i][0])
        y_avg.append((probabilities_1[i][0]+probabilities_2[i][0]+probabilities_3[i][0])/3)

#  --- SECTION 3 ---
# Plot the predicted probaiblity of each base learner as
# a bar and the average probability as an X
plt.bar(x, y_1, 3,   label='5NN')
plt.bar(x, y_2, 2,   label='NB')
plt.bar(x, y_3, 1,   label='50NN')
plt.scatter(x, y_avg, marker='x', c='k', s=150, label='Average Positive', zorder=10)

y = [0.5 for x in range(len(errors))]
plt.plot(y, c='k', linestyle='--')

plt.title('Positive Probability')
plt.xlabel('Test sample')
plt.ylabel('probability')
plt.legend()







[((241, 270), 'sklearn.datasets.load_breast_cancer', 'datasets.load_breast_cancer', ([], {}), '()\n', (268, 270), False, 'from sklearn import datasets, naive_bayes, svm, neighbors\n'), ((569, 614), 'sklearn.neighbors.KNeighborsClassifier', 'neighbors.KNeighborsClassifier', ([], {'n_neighbors': '(5)'}), '(n_neighbors=5)\n', (599, 614), False, 'from sklearn import datasets, naive_bayes, svm, neighbors\n'), ((628, 652), 'sklearn.naive_bayes.GaussianNB', 'naive_bayes.GaussianNB', ([], {}), '()\n', (650, 652), False, 'from sklearn import datasets, naive_bayes, svm, neighbors\n'), ((666, 712), 'sklearn.neighbors.KNeighborsClassifier', 'neighbors.KNeighborsClassifier', ([], {'n_neighbors': '(50)'}), '(n_neighbors=50)\n', (696, 712), False, 'from sklearn import datasets, naive_bayes, svm, neighbors\n'), ((784, 882), 'sklearn.ensemble.VotingClassifier', 'VotingClassifier', (["[('5NN', learner_1), ('NB', learner_2), ('50NN', learner_3)]"], {'voting': '"""soft"""'}), "([('5NN', learner_1), ('NB', learner_2), ('50NN', learner_3\n    )], voting='soft')\n", (800, 882), False, 'from sklearn.ensemble import VotingClassifier\n'), ((1892, 1922), 'matplotlib.style.use', 'mpl.style.use', (['"""seaborn-paper"""'], {}), "('seaborn-paper')\n", (1905, 1922), True, 'import matplotlib as mpl\n'), ((2880, 2911), 'matplotlib.pyplot.bar', 'plt.bar', (['x', 'y_1', '(3)'], {'label': '"""5NN"""'}), "(x, y_1, 3, label='5NN')\n", (2887, 2911), True, 'import matplotlib.pyplot as plt\n'), ((2915, 2945), 'matplotlib.pyplot.bar', 'plt.bar', (['x', 'y_2', '(2)'], {'label': '"""NB"""'}), "(x, y_2, 2, label='NB')\n", (2922, 2945), True, 'import matplotlib.pyplot as plt\n'), ((2949, 2981), 'matplotlib.pyplot.bar', 'plt.bar', (['x', 'y_3', '(1)'], {'label': '"""50NN"""'}), "(x, y_3, 1, label='50NN')\n", (2956, 2981), True, 'import matplotlib.pyplot as plt\n'), ((2985, 3073), 'matplotlib.pyplot.scatter', 'plt.scatter', (['x', 'y_avg'], {'marker': '"""x"""', 'c': '"""k"""', 's': '(150)', 'label': '"""Average Positive"""', 'zorder': '(10)'}), "(x, y_avg, marker='x', c='k', s=150, label='Average Positive',\n    zorder=10)\n", (2996, 3073), True, 'import matplotlib.pyplot as plt\n'), ((3112, 3146), 'matplotlib.pyplot.plot', 'plt.plot', (['y'], {'c': '"""k"""', 'linestyle': '"""--"""'}), "(y, c='k', linestyle='--')\n", (3120, 3146), True, 'import matplotlib.pyplot as plt\n'), ((3150, 3183), 'matplotlib.pyplot.title', 'plt.title', (['"""Positive Probability"""'], {}), "('Positive Probability')\n", (3159, 3183), True, 'import matplotlib.pyplot as plt\n'), ((3185, 3210), 'matplotlib.pyplot.xlabel', 'plt.xlabel', (['"""Test sample"""'], {}), "('Test sample')\n", (3195, 3210), True, 'import matplotlib.pyplot as plt\n'), ((3212, 3237), 'matplotlib.pyplot.ylabel', 'plt.ylabel', (['"""probability"""'], {}), "('probability')\n", (3222, 3237), True, 'import matplotlib.pyplot as plt\n'), ((3239, 3251), 'matplotlib.pyplot.legend', 'plt.legend', ([], {}), '()\n', (3249, 3251), True, 'import matplotlib.pyplot as plt\n'), ((1523, 1560), 'sklearn.metrics.accuracy_score', 'accuracy_score', (['y_test', 'predictions_1'], {}), '(y_test, predictions_1)\n', (1537, 1560), False, 'from sklearn.metrics import accuracy_score\n'), ((1576, 1613), 'sklearn.metrics.accuracy_score', 'accuracy_score', (['y_test', 'predictions_2'], {}), '(y_test, predictions_2)\n', (1590, 1613), False, 'from sklearn.metrics import accuracy_score\n'), ((1629, 1666), 'sklearn.metrics.accuracy_score', 'accuracy_score', (['y_test', 'predictions_3'], {}), '(y_test, predictions_3)\n', (1643, 1666), False, 'from sklearn.metrics import accuracy_score\n'), ((1733, 1773), 'sklearn.metrics.accuracy_score', 'accuracy_score', (['y_test', 'hard_predictions'], {}), '(y_test, hard_predictions)\n', (1747, 1773), False, 'from sklearn.metrics import accuracy_score\n')]