text: # coding: utf-8
  
  
  from scipy.special import comb
  import math
  import numpy as np
  import matplotlib.pyplot as plt
  from sklearn.base import BaseEstimator
  from sklearn.base import ClassifierMixin
  from sklearn.preprocessing import LabelEncoder
  from sklearn.externals import six
  from sklearn.base import clone
  from sklearn.pipeline import _name_estimators
  import operator
  from sklearn import datasets
  from sklearn.preprocessing import StandardScaler
  from sklearn.model_selection import train_test_split
  from sklearn.linear_model import LogisticRegression
  from sklearn.tree import DecisionTreeClassifier
  from sklearn.neighbors import KNeighborsClassifier 
  from sklearn.pipeline import Pipeline
  from sklearn.model_selection import cross_val_score
  from sklearn.metrics import roc_curve
  from sklearn.metrics import auc
  from itertools import product
  from sklearn.model_selection import GridSearchCV
  import pandas as pd
  from sklearn.ensemble import BaggingClassifier
  from sklearn.metrics import accuracy_score
  from sklearn.ensemble import AdaBoostClassifier
  
  # *Python Machine Learning 2nd Edition* by [Sebastian Raschka](https://sebastianraschka.com), Packt Publishing Ltd. 2017
  # 
  # Code Repository: https://github.com/rasbt/python-machine-learning-book-2nd-edition
  # 
  # Code License: [MIT License](https://github.com/rasbt/python-machine-learning-book-2nd-edition/blob/master/LICENSE.txt)
  
  # # Python Machine Learning - Code Examples
  
  # # Chapter 7 - Combining Different Models for Ensemble Learning
  
  # Note that the optional watermark extension is a small IPython notebook plugin that I developed to make the code reproducible. You can just skip the following line(s).
  
  
  
  
  
  # *The use of `watermark` is optional. You can install this IPython extension via "`pip install watermark`". For more information, please see: https://github.com/rasbt/watermark.*
  
  
  # ### Overview
  
  # - [Learning with ensembles](#Learning-with-ensembles)
  # - [Combining classifiers via majority vote](#Combining-classifiers-via-majority-vote)
  #     - [Implementing a simple majority vote classifier](#Implementing-a-simple-majority-vote-classifier)
  #     - [Using the majority voting principle to make predictions](#Using-the-majority-voting-principle-to-make-predictions)
  #     - [Evaluating and tuning the ensemble classifier](#Evaluating-and-tuning-the-ensemble-classifier)
  # - [Bagging – building an ensemble of classifiers from bootstrap samples](#Bagging----Building-an-ensemble-of-classifiers-from-bootstrap-samples)
  #     - [Bagging in a nutshell](#Bagging-in-a-nutshell)
  #     - [Applying bagging to classify samples in the Wine dataset](#Applying-bagging-to-classify-samples-in-the-Wine-dataset)
  # - [Leveraging weak learners via adaptive boosting](#Leveraging-weak-learners-via-adaptive-boosting)
  #     - [How boosting works](#How-boosting-works)
  #     - [Applying AdaBoost using scikit-learn](#Applying-AdaBoost-using-scikit-learn)
  # - [Summary](#Summary)
  
  
  
  
  
  
  # # Learning with ensembles
  
  
  
  
  def ensemble_error(n_classifier, error):
      k_start = int(math.ceil(n_classifier / 2.))
      probs = [comb(n_classifier, k) * error**k * (1-error)**(n_classifier - k)
               for k in range(k_start, n_classifier + 1)]
      return sum(probs)
  
  
  
  
  ensemble_error(n_classifier=11, error=0.25)
  
  # visualize for a range of base error values
  error_range = np.arange(0.0, 1.01, 0.01)
  ens_errors = [ensemble_error(n_classifier=11, error=error)
                for error in error_range]
  
  
  
  plt.plot(error_range, 
           ens_errors, 
           label='Ensemble error', 
           linewidth=2)
  
  plt.plot(error_range, 
           error_range, 
           linestyle='--',
           label='Base error',
           linewidth=2)
  
  plt.xlabel('Base error')
  plt.ylabel('Base/Ensemble error')
  plt.legend(loc='upper left')
  plt.grid(alpha=0.5)
  #plt.savefig('images/07_03.png', dpi=300)
  plt.show()
  
  
  
  # # Combining classifiers via majority vote
  
  # ## Implementing a simple majority vote classifier 
  
  
  
  
  np.argmax(np.bincount([0, 0, 1], weights=[0.2, 0.2, 0.6]))
  
  
  
  
  ex = np.array([[0.9, 0.1],
                 [0.8, 0.2],
                 [0.4, 0.6]])
  
  p = np.average(ex, 
                 axis=0, 
                 weights=[0.2, 0.2, 0.6])
  p
  
  
  np.argmax(p)
  
  
  
  
  
  
  class MajorityVoteClassifier(BaseEstimator, 
                               ClassifierMixin):
      """ A majority vote ensemble classifier
  
      Parameters
      ----------
      classifiers : array-like, shape = [n_classifiers]
        Different classifiers for the ensemble
  
      vote : str, {'classlabel', 'probability'} (default='label')
        If 'classlabel' the prediction is based on the argmax of
          class labels. Else if 'probability', the argmax of
          the sum of probabilities is used to predict the class label
          (recommended for calibrated classifiers).
  
      weights : array-like, shape = [n_classifiers], optional (default=None)
        If a list of `int` or `float` values are provided, the classifiers
        are weighted by importance; Uses uniform weights if `weights=None`.
  
      """
      def __init__(self, classifiers, vote='classlabel', weights=None):
  
          self.classifiers = classifiers
          self.named_classifiers = {key: value for key, value
                                    in _name_estimators(classifiers)}
          self.vote = vote
          self.weights = weights
  
      def fit(self, X, y):
          """ Fit classifiers.
  
          Parameters
          ----------
          X : {array-like, sparse matrix}, shape = [n_samples, n_features]
              Matrix of training samples.
  
          y : array-like, shape = [n_samples]
              Vector of target class labels.
  
          Returns
          -------
          self : object
  
          """
          if self.vote not in ('probability', 'classlabel'):
              raise ValueError("vote must be 'probability' or 'classlabel'"
                               "; got (vote=%r)"
                               % self.vote)
  
          if self.weights and len(self.weights) != len(self.classifiers):
              raise ValueError('Number of classifiers and weights must be equal'
                               '; got %d weights, %d classifiers'
                               % (len(self.weights), len(self.classifiers)))
  
          # Use LabelEncoder to ensure class labels start with 0, which
          # is important for np.argmax call in self.predict
          self.lablenc_ = LabelEncoder()
          self.lablenc_.fit(y)
          self.classes_ = self.lablenc_.classes_
          self.classifiers_ = []
          for clf in self.classifiers:
              fitted_clf = clone(clf).fit(X, self.lablenc_.transform(y))
              self.classifiers_.append(fitted_clf)
          return self
  
      def predict(self, X):
          """ Predict class labels for X.
  
          Parameters
          ----------
          X : {array-like, sparse matrix}, shape = [n_samples, n_features]
              Matrix of training samples.
  
          Returns
          ----------
          maj_vote : array-like, shape = [n_samples]
              Predicted class labels.
              
          """
          if self.vote == 'probability':
              maj_vote = np.argmax(self.predict_proba(X), axis=1)
          else:  # 'classlabel' vote
  
              #  Collect results from clf.predict calls
              predictions = np.asarray([clf.predict(X)
                                        for clf in self.classifiers_]).T
  
              maj_vote = np.apply_along_axis(
                                        lambda x:
                                        np.argmax(np.bincount(x,
                                                  weights=self.weights)),
                                        axis=1,
                                        arr=predictions)
          maj_vote = self.lablenc_.inverse_transform(maj_vote)
          return maj_vote
  
      def predict_proba(self, X):
          """ Predict class probabilities for X.
  
          Parameters
          ----------
          X : {array-like, sparse matrix}, shape = [n_samples, n_features]
              Training vectors, where n_samples is the number of samples and
              n_features is the number of features.
  
          Returns
          ----------
          avg_proba : array-like, shape = [n_samples, n_classes]
              Weighted average probability for each class per sample.
  
          """
          probas = np.asarray([clf.predict_proba(X)
                               for clf in self.classifiers_])
          avg_proba = np.average(probas, axis=0, weights=self.weights)
          return avg_proba
  
      def get_params(self, deep=True):
          """ Get classifier parameter names for GridSearch"""
          if not deep:
              return super(MajorityVoteClassifier, self).get_params(deep=False)
          else:
              out = self.named_classifiers.copy()
              for name, step in six.iteritems(self.named_classifiers):
                  for key, value in six.iteritems(step.get_params(deep=True)):
                      out['%s__%s' % (name, key)] = value
              return out
  
  
  
  # ## Using the majority voting principle to make predictions
  
  
  iris = datasets.load_iris()
  X, y = iris.data[50:, [1, 2]], iris.target[50:]
  le = LabelEncoder()
  y = le.fit_transform(y)
  
  X_train, X_test, y_train, y_test =       train_test_split(X, y, 
                          test_size=0.5, 
                          random_state=1,
                          stratify=y)
  
  
  # define different classifiers
  
  
  clf1 = LogisticRegression(penalty='l2', 
                            C=0.001,
                            random_state=1)
  
  clf2 = DecisionTreeClassifier(max_depth=1,
                                criterion='entropy',
                                random_state=0)
  
  clf3 = KNeighborsClassifier(n_neighbors=1,
                              p=2,
                              metric='minkowski')
  # note only DT classifier can be considered scale invariant; 
  # LR and kNN classifiers need standardized data set
  pipe1 = Pipeline([['sc', StandardScaler()],
                    ['clf', clf1]])
  pipe3 = Pipeline([['sc', StandardScaler()],
                    ['clf', clf3]])
  
  clf_labels = ['Logistic regression', 'Decision tree', 'KNN']
  
  print('10-fold cross validation:\n')
  for clf, label in zip([pipe1, clf2, pipe3], clf_labels):
      scores = cross_val_score(estimator=clf,
                               X=X_train,
                               y=y_train,
                               cv=10,
                               scoring='roc_auc')
      print("ROC AUC: %0.2f (+/- %0.2f) [%s]"
            % (scores.mean(), scores.std(), label))
  
  
  
  
  # Majority Rule (hard) Voting
  
  mv_clf = MajorityVoteClassifier(classifiers=[pipe1, clf2, pipe3])
  
  # mv_clf object has a fit method. so it qualifies 
  # as an estimator object for cross_val_score sci-kit function
  
  clf_labels += ['Majority voting']
  all_clf = [pipe1, clf2, pipe3, mv_clf]
  
  for clf, label in zip(all_clf, clf_labels):
      scores = cross_val_score(estimator=clf,
                               X=X_train,
                               y=y_train,
                               cv=10,
                               scoring='roc_auc')
      print("ROC AUC: %0.2f (+/- %0.2f) [%s]"
            % (scores.mean(), scores.std(), label))
  
  
  
  # # Evaluating and tuning the ensemble classifier
  
  
  # # fit the ensemble on unseen test data
  
  colors = ['black', 'orange', 'blue', 'green']
  linestyles = [':', '--', '-.', '-']
  for clf, label, clr, ls         in zip(all_clf,
                 clf_labels, colors, linestyles):
  
      # assuming the label of the positive class is 1
      y_pred = clf.fit(X_train,
                       y_train).predict_proba(X_test)[:, 1]
      fpr, tpr, thresholds = roc_curve(y_true=y_test,
                                       y_score=y_pred)
      roc_auc = auc(x=fpr, y=tpr)
      plt.plot(fpr, tpr,
               color=clr,
               linestyle=ls,
               label='%s (auc = %0.2f)' % (label, roc_auc))
  
  plt.legend(loc='lower right')
  plt.plot([0, 1], [0, 1],
           linestyle='--',
           color='gray',
           linewidth=2)
  
  plt.xlim([-0.1, 1.1])
  plt.ylim([-0.1, 1.1])
  plt.grid(alpha=0.5)
  plt.xlabel('False positive rate (FPR)')
  plt.ylabel('True positive rate (TPR)')
  
  
  #plt.savefig('images/07_04', dpi=300)
  plt.show()
  
  
  
  # # plot how the decision region actually looks like
  sc = StandardScaler()
  X_train_std = sc.fit_transform(X_train)
  
  
  all_clf = [pipe1, clf2, pipe3, mv_clf]
  
  x_min = X_train_std[:, 0].min() - 1
  x_max = X_train_std[:, 0].max() + 1
  y_min = X_train_std[:, 1].min() - 1
  y_max = X_train_std[:, 1].max() + 1
  
  xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1),
                       np.arange(y_min, y_max, 0.1))
  
  f, axarr = plt.subplots(nrows=2, ncols=2, 
                          sharex='col', 
                          sharey='row', 
                          figsize=(7, 5))
  
  for idx, clf, tt in zip(product([0, 1], [0, 1]),
                          all_clf, clf_labels):
      clf.fit(X_train_std, y_train)
      
      Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
      Z = Z.reshape(xx.shape)
  
      axarr[idx[0], idx[1]].contourf(xx, yy, Z, alpha=0.3)
      
      axarr[idx[0], idx[1]].scatter(X_train_std[y_train==0, 0], 
                                    X_train_std[y_train==0, 1], 
                                    c='blue', 
                                    marker='^',
                                    s=50)
      
      axarr[idx[0], idx[1]].scatter(X_train_std[y_train==1, 0], 
                                    X_train_std[y_train==1, 1], 
                                    c='green', 
                                    marker='o',
                                    s=50)
      
      axarr[idx[0], idx[1]].set_title(tt)
  
  plt.text(-3.5, -5., 
           s='Sepal width [standardized]', 
           ha='center', va='center', fontsize=12)
  plt.text(-12.5, 4.5, 
           s='Petal length [standardized]', 
           ha='center', va='center', 
           fontsize=12, rotation=90)
  
  #plt.savefig('images/07_05', dpi=300)
  plt.show()
  
  
  
  # # examples of accessinng individual params of the majority vote classifier
  mv_clf.get_params()
  
  
  
  # # demo tune the following indiv params via a grid search
  
  params = {'decisiontreeclassifier__max_depth': [1, 2],
            'pipeline-1__clf__C': [0.001, 0.1, 100.0]}
  
  grid = GridSearchCV(estimator=mv_clf,
                      param_grid=params,
                      cv=10,
                      scoring='roc_auc')
  grid.fit(X_train, y_train)
  
  for r, _ in enumerate(grid.cv_results_['mean_test_score']):
      print("%0.3f +/- %0.2f %r"
            % (grid.cv_results_['mean_test_score'][r], 
               grid.cv_results_['std_test_score'][r] / 2.0, 
               grid.cv_results_['params'][r]))
  
  
  
  
  print('Best parameters: %s' % grid.best_params_)
  print('Accuracy: %.2f' % grid.best_score_)
  
  
  # **Note**  
  # By default, the default setting for `refit` in `GridSearchCV` is `True` (i.e., `GridSeachCV(..., refit=True)`), which means that we can use the fitted `GridSearchCV` estimator to make predictions via the `predict` method, for example:
  # 
  #     grid = GridSearchCV(estimator=mv_clf, 
  #                         param_grid=params, 
  #                         cv=10, 
  #                         scoring='roc_auc')
  #     grid.fit(X_train, y_train)
  #     y_pred = grid.predict(X_test)
  # 
  # In addition, the "best" estimator can directly be accessed via the `best_estimator_` attribute.
  
  # we can set the mv_clf to the best estimator with the best tuned params
  
  grid.best_estimator_.classifiers
  
  
  
  
  mv_clf = grid.best_estimator_
  
  
  
  
  mv_clf.set_params(**grid.best_estimator_.get_params())
  
  
  
  
  mv_clf
  
  
  
  # # Bagging -- Building an ensemble of classifiers from bootstrap samples
  # ## Bagging in a nutshell
  # ## Applying bagging to classify samples in the Wine dataset
  
  df_wine = pd.read_csv('https://archive.ics.uci.edu/ml/'
                        'machine-learning-databases/wine/wine.data',
                        header=None)
  
  df_wine.columns = ['Class label', 'Alcohol', 'Malic acid', 'Ash',
                     'Alcalinity of ash', 'Magnesium', 'Total phenols',
                     'Flavanoids', 'Nonflavanoid phenols', 'Proanthocyanins',
                     'Color intensity', 'Hue', 'OD280/OD315 of diluted wines',
                     'Proline']
  
  # if the Wine dataset is temporarily unavailable from the
  # UCI machine learning repository, un-comment the following line
  # of code to load the dataset from a local path:
  
  # df_wine = pd.read_csv('wine.data', header=None)
  
  # drop 1 class
  df_wine = df_wine[df_wine['Class label'] != 1]
  
  y = df_wine['Class label'].values
  X = df_wine[['Alcohol', 'OD280/OD315 of diluted wines']].values
  
  
  le = LabelEncoder()
  y = le.fit_transform(y)
  
  X_train, X_test, y_train, y_test =            train_test_split(X, y, 
                               test_size=0.2, 
                               random_state=1,
                               stratify=y)
  
  
  tree = DecisionTreeClassifier(criterion='entropy', 
                                max_depth=None,
                                random_state=1)
  
  bag = BaggingClassifier(base_estimator=tree,
                          n_estimators=500, 
                          max_samples=1.0, 
                          max_features=1.0, 
                          bootstrap=True, 
                          bootstrap_features=False, 
                          n_jobs=4, 
                          random_state=1)
  
  ## fitting the DT classifier 
  tree = tree.fit(X_train, y_train)
  y_train_pred = tree.predict(X_train)
  y_test_pred = tree.predict(X_test)
  
  tree_train = accuracy_score(y_train, y_train_pred)
  tree_test = accuracy_score(y_test, y_test_pred)
  print('Decision tree train/test accuracies %.3f/%.3f'
        % (tree_train, tree_test))
  
  ## fitting the bagging classifier
  bag = bag.fit(X_train, y_train)
  y_train_pred = bag.predict(X_train)
  y_test_pred = bag.predict(X_test)
  
  bag_train = accuracy_score(y_train, y_train_pred) 
  bag_test = accuracy_score(y_test, y_test_pred) 
  print('Bagging train/test accuracies %.3f/%.3f'
        % (bag_train, bag_test))
  
  
  
  
  ## compare decision regions
  x_min = X_train[:, 0].min() - 1
  x_max = X_train[:, 0].max() + 1
  y_min = X_train[:, 1].min() - 1
  y_max = X_train[:, 1].max() + 1
  
  xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1),
                       np.arange(y_min, y_max, 0.1))
  
  f, axarr = plt.subplots(nrows=1, ncols=2, 
                          sharex='col', 
                          sharey='row', 
                          figsize=(8, 3))
  
  
  for idx, clf, tt in zip([0, 1],
                          [tree, bag],
                          ['Decision tree', 'Bagging']):
      clf.fit(X_train, y_train)
  
      Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
      Z = Z.reshape(xx.shape)
  
      axarr[idx].contourf(xx, yy, Z, alpha=0.3)
      axarr[idx].scatter(X_train[y_train == 0, 0],
                         X_train[y_train == 0, 1],
                         c='blue', marker='^')
  
      axarr[idx].scatter(X_train[y_train == 1, 0],
                         X_train[y_train == 1, 1],
                         c='green', marker='o')
  
      axarr[idx].set_title(tt)
  
  axarr[0].set_ylabel('Alcohol', fontsize=12)
  plt.text(10.2, -0.5,
           s='OD280/OD315 of diluted wines',
           ha='center', va='center', fontsize=12)
  
  plt.tight_layout()
  #plt.savefig('images/07_08.png', dpi=300, bbox_inches='tight')
  plt.show()
  
  
  
  # # Leveraging weak learners via adaptive boosting
  
  # ## How boosting works
  
  
  
  
  
  # #--------------------------------------------------------------------------
  
  
  
  # ## Applying AdaBoost using scikit-learn
  
  # # setup a weak learner a - a DT stump
  tree = DecisionTreeClassifier(criterion='entropy', 
                                max_depth=1,
                                random_state=1)
  
  # # train on 500 DT forest
  
  ada = AdaBoostClassifier(base_estimator=tree,
                           n_estimators=500, 
                           learning_rate=0.1,
                           random_state=1)
  
  # # compare the weak learner alone and adaboosted classifier
  tree = tree.fit(X_train, y_train)
  y_train_pred = tree.predict(X_train)
  y_test_pred = tree.predict(X_test)
  
  tree_train = accuracy_score(y_train, y_train_pred)
  tree_test = accuracy_score(y_test, y_test_pred)
  print('Decision tree train/test accuracies %.3f/%.3f'
        % (tree_train, tree_test))
  
  ada = ada.fit(X_train, y_train)
  y_train_pred = ada.predict(X_train)
  y_test_pred = ada.predict(X_test)
  
  ada_train = accuracy_score(y_train, y_train_pred) 
  ada_test = accuracy_score(y_test, y_test_pred) 
  print('AdaBoost train/test accuracies %.3f/%.3f'
        % (ada_train, ada_test))
  
  
  
  # # visualize the decision regions
  x_min, x_max = X_train[:, 0].min() - 1, X_train[:, 0].max() + 1
  y_min, y_max = X_train[:, 1].min() - 1, X_train[:, 1].max() + 1
  xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1),
                       np.arange(y_min, y_max, 0.1))
  
  f, axarr = plt.subplots(1, 2, sharex='col', sharey='row', figsize=(12,4))
  
  fsize = 8
  hpos = 'right'
  vpos = 'bottom'
  for idx, clf, tt in zip([0, 1],
                          [tree, ada],
                          ['Decision tree', 'AdaBoost']):
      clf.fit(X_train, y_train)
  
      Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
      Z = Z.reshape(xx.shape)
  
      axarr[idx].contourf(xx, yy, Z, alpha=0.3)
      axarr[idx].scatter(X_train[y_train == 0, 0],
                         X_train[y_train == 0, 1],
                         c='blue', marker='^')
      axarr[idx].scatter(X_train[y_train == 1, 0],
                         X_train[y_train == 1, 1],
                         c='green', marker='o')
      axarr[idx].set_title(tt, fontsize=fsize, ha=hpos)
  
      axarr[idx].set_ylabel('Alcohol', fontsize=fsize, ha=hpos)
      axarr[idx].set_xlabel('OD280/OD315 of diluted wines', fontsize=fsize, ha=hpos)
  
  plt.tight_layout()
  #plt.savefig('images/07_11.png', dpi=300, bbox_inches='tight')
  plt.show()
  
  
  
  # # Summary
  
  # ...
  
  # ---
  # 
  # Readers may ignore the next cell.
  
  
  
  
  