import time, os, re, zipfile
import numpy as np, pandas as pd
%matplotlib inline
import matplotlib.pyplot as plt

"""
Explanation: eXtreme Gradient Boosting library (XGBoost)
<center>An unfocused introduction by Ivan Nazarov</center>
Import the main toolkit.
End of explanation
"""


import sklearn as sk, xgboost as xg
# from sklearn.model_selection import train_test_split
from sklearn.cross_validation import train_test_split

"""
Explanation: Now import some ML stuff
End of explanation
"""


random_state = np.random.RandomState( seed = 0x0BADC0DE )

"""
Explanation: Mind the seed!!
End of explanation
"""


df_train = pd.read_csv( zipfile.ZipFile( 'train.csv.zip' ).open( 'train.csv' ), index_col = 'id' )
X = np.asanyarray( df_train.drop( 'target', axis = 1 ) )
y = sk.preprocessing.LabelEncoder( ).fit_transform( df_train[ 'target' ] )

"""
Explanation: Let's begin this introduction with usage examples.
The demonstration uses the dataset, which was originally used in 
Otto Group Product Classification Challenge. We load the data directly from ZIP archives.
End of explanation
"""


X_train, X_, y_train, y_ = train_test_split( X, y, test_size = 0.25, random_state = random_state )
X_valid, X_test, y_valid, y_test = train_test_split( X_, y_, test_size = 0.5, random_state = random_state )

"""
Explanation: As usual do the train-test split.
End of explanation
"""


clf_ = xg.XGBClassifier( n_estimators = 50,
                         gamma = 1.0,
                         max_depth = 1000,
                         objective = "multi:softmax",
                         nthread = -1,
                         silent = False )

"""
Explanation: scikit-learn interface
Use scikit-learn compatible interface of XGBoost.
End of explanation
"""


clf_.fit( X_train, y_train, eval_set = [ ( X_valid, y_valid ), ], verbose = True )

"""
Explanation: Fit the a gradient boosted tree ensemble.
End of explanation
"""


y_predict = clf_.predict( X_test )
y_score = clf_.predict_proba( X_test )

"""
Explanation: Now let's validate.
End of explanation
"""


pd.DataFrame( sk.metrics.confusion_matrix( y_test, y_predict ), index = clf_.classes_, columns = clf_.classes_ )

"""
Explanation: Let's check out the confusuion matrix
End of explanation
"""


fig = plt.figure( figsize = ( 16, 9 ) )
axis = fig.add_subplot( 111 )
axis.set_title( 'ROC-AUC (ovr) curves for the heldout dataset' )
axis.set_xlabel( "False positive rate" ) ; axis.set_ylabel( "True positive rate" )
axis.set_ylim( -0.01, 1.01 ) ; axis.set_xlim( -0.01, 1.01 )
for cls_ in clf_.classes_ :
    fpr, tpr, _ = sk.metrics.roc_curve( y_test, y_score[:, cls_], pos_label = cls_ )
    axis.plot( fpr, tpr, lw = 2, zorder = cls_, label = "C%d" % ( cls_, ) )
axis.legend( loc = 'lower right', shadow = True, ncol = 3 )

"""
Explanation: Let's plot one-vs-all ROC-AUC curves
End of explanation
"""


train_dmat = xg.DMatrix( data = X_train,
                         label = y_train,
                         feature_names = None,
                         feature_types = None )
test_dmat = xg.DMatrix( data = X_test, label = y_test )

"""
Explanation: alternative interface
Internally XGBoost relies heavily on a custom dataset format DMatrix. It is ...
The interface, which is exposed into python has three capabilities:
- load datasets in libSVM compatible format;
- load SciPy's sparse matrices;
- load Numpy's ndarrays.
Let's load the train dataset using numpy interface :
- data : the matrix of features $X$;
- label : the observation labels $y$ (could be categorical or numeric);
- missing : a vector of values that encode missing observations;
- feature_names : the columns names of $X$;
- feature_types : defines the python types of each column of $X$, in case of heterogeneous data;
- weight : the vector of nonnegative weights of each observation in the dataset.
End of explanation
"""


xgb_params = {
    'bst:max_depth':2,
    'bst:eta':1,
    'silent':1,
    'objective':'multi:softmax',
    'num_class': 9,
    'nthread': 2, 
    'eval_metric' : 'auc'
}

"""
Explanation: DMatrix exports several useful methods:
- num_col() : returns the number of columns;
- num_row() : gets the number of items;
- save_binary( fname ) : saves the DMatrix object into a specified file.
For a more detailed list, it is useful to have a look at the official manual
Having dafined the datasets, it is the right time to initialize the booster. To this end one uses xgboost.Learner class. Among other parameters, its instance is initialized with a dictionary of parameters, which allows for a more flexible booster initialization.
End of explanation
"""


xgbooster_ = xg.train( params = xgb_params,
                       dtrain = train_dmat, 
                       num_boost_round = 10,
                       evals = (),
                       obj = None,
                       feval = None,
                       maximize = False,
                       early_stopping_rounds = None,
                       evals_result = None,
                       verbose_eval = True,
                       learning_rates = None,
                       xgb_model = None )

"""
Explanation: The xgboost.train class initalizes an appropriate booster, and then fits it on the provided train dataset. Besides the booster parameters and the train DMatrix , the class initializer accepts:
- num_boost_round : and interger number of boosting iterations which is the number of trees in the final ensemble;
- evals : a list of DMatrix validation datasets to be evaluated during training;
- obj : a custom objective function;
- feval : a custom evaluation function;
- early_stopping_rounds : Activates early stopping, which checks every early_stopping_rounds round(s) if the validation error has decreased in order to continue training;
- maximize : a flag, which determines if the objective (feval) should be maximized;
- learning_rates : a schedule for learning rates for each boosting round or a function that calculates $\eta$, for the current round;
- xgb_model : an XGB model (booster or file), the training of which is to be continued.
End of explanation
"""


y_predict = xgbooster_.predict( test_dmat )
y_score = xgbooster_.predict( test_dmat, output_margin = True )

"""
Explanation: The method xgboost.booster.update performs one iteration of gradinet boosting:
- dtrain : DMatrix of train dataset;
- iteration : the current iteration number;
- fobj : a custom objective function touse for this update.
The method xboost.booster.boost performs one iteration of boosting on the custom gradient statistics:
- dtrain : the DMatrix dataset to operate on;
- grad, hess : pair of lists of loss gradients and hessians, respectively, evaluated at each datapoint in dtrain.
The method xgboost.booster.predict returns either the learned value, or the index of the target leaf. The parameters are :
- data : a DMatrix object storing the input;
- output_margin : a flag, determining, if raw untransformed margin values should be returned;
- ntree_limit : limit the number of trees used for predicting (defaults to 0, which use all trees);
- pred_leaf : determined wether the output should be a matrix of $(n, K)$ of predicted leaf indices, where $K$ is the number of trees in the ensemble.
The returned result is a numpy ndarray.
End of explanation
"""


pd.DataFrame( sk.metrics.confusion_matrix( y_test, y_predict ), index = clf_.classes_, columns = clf_.classes_ )

"""
Explanation: Besides these methods xgboost.booster exports:
load_model( fname ) and save_model( fname ).
Let's check out the confusuion matrix
End of explanation
"""


fig = plt.figure( figsize = ( 16, 9 ) )
axis = fig.add_subplot( 111 )
axis.set_title( 'ROC-AUC (ovr) curves for the heldout dataset' )
axis.set_xlabel( "False positive rate" ) ; axis.set_ylabel( "True positive rate" )
axis.set_ylim( -0.01, 1.01 ) ; axis.set_xlim( -0.01, 1.01 )
for cls_ in clf_.classes_ :
    fpr, tpr, _ = sk.metrics.roc_curve( y_test, y_score[:, cls_], pos_label = cls_ )
    axis.plot( fpr, tpr, lw = 2, zorder = cls_, label = "C%d" % ( cls_, ) )
axis.legend( loc = 'lower right', shadow = True, ncol = 3 )

"""
Explanation: Let's plot one-vs-all ROC-AUC curves
End of explanation
"""


