%pylab inline

"""
Explanation: Viva la Factory
Factory is way to train several different classifiers on the same dataset and compare the quailty of predictions.
First, enable plotting
End of explanation
"""


!cd toy_datasets; wget -O MiniBooNE_PID.txt -nc MiniBooNE_PID.txt https://archive.ics.uci.edu/ml/machine-learning-databases/00199/MiniBooNE_PID.txt

import numpy, pandas
from rep.utils import train_test_split

import numpy, pandas
from rep.utils import train_test_split
from sklearn.metrics import roc_auc_score

data = pandas.read_csv('toy_datasets/MiniBooNE_PID.txt', sep='\s*', skiprows=[0], header=None, engine='python')
labels = pandas.read_csv('toy_datasets/MiniBooNE_PID.txt', sep=' ', nrows=1, header=None)
labels = [1] * labels[1].values[0] + [0] * labels[2].values[0]
data.columns = ['feature_{}'.format(key) for key in data.columns]

train_data, test_data, train_labels, test_labels = train_test_split(data, labels, train_size=0.5)

"""
Explanation: Loading data
download particle identification Data Set from UCI
End of explanation
"""


train_variables = ["feature_new01: feature_0/feature_1", "feature_2", "feature_26", "feature_12", "feature_24",
                   "feature_25", "feature_16",]
plot_variables = train_variables + ['feature_3']

"""
Explanation: Variables needed for analysis
End of explanation
"""


from rep.metaml import ClassifiersFactory
from rep.estimators import TMVAClassifier, SklearnClassifier, XGBoostClassifier
from sklearn.ensemble import AdaBoostClassifier

"""
Explanation: Factory of different models
This class is OrderedDict, with additional interface, main methods are:


factory.add_classifier(name, classifier) 


factory.fit(X, y, sample_weight=None, ipc_profile=None, features=None) <br />
    train all classifiers in factory<br />
    if features is not None, then all classifiers will be trained on these features <br />
    you can pass the name of ipython cluster via ipc_profile for parallel training


factory.test_on_lds(lds) - test all models on lds(rep.data.storage.LabeledDataStorage)  <br />
    returns report (rep.report.classification.ClassificationReport)
End of explanation
"""


factory = ClassifiersFactory()
# There are different ways to add classifiers to Factory:
factory.add_classifier('tmva', TMVAClassifier(NTrees=50, features=train_variables, Shrinkage=0.05))
factory.add_classifier('ada', AdaBoostClassifier(n_estimators=10))
factory['xgb'] = XGBoostClassifier(features=train_variables)

"""
Explanation: Define classifiers (that will be compared)
Please pay attention that we set very small number of trees, just to make this notebook work fast. 
Don't forget to tune classifier!
End of explanation
"""


from copy import deepcopy
factory_copy = deepcopy(factory)

"""
Explanation: Create a copy of the factory with all classifiers
End of explanation
"""


%time factory.fit(train_data, train_labels, features=train_variables)
pass

factory.predict_proba(train_data)

%time factory_copy.fit(train_data, train_labels)
pass

"""
Explanation: Training
pay attention: <br />
for the first factory we pointed features that will be used in training and all classifiers will use them, <br />
for the second factory all the classifiers will use those features we set in their constuctors
End of explanation
"""


report = factory.test_on(test_data, test_labels)

"""
Explanation: Everybody loves plots!
Visualizing of training result with factory
ClassificationReport class provides the posibility to get classification description to compare different models. <br />Below you can find available functions which can help you to analyze result on arbitrary dataset.
There are different plotting backends supported:
* matplotlib (default, de-facto standard plotting library, mpld3 allows turning this into interactive plots), 
* plotly (proprietary package with interactive plots, information is kept on the server), 
* ROOT (the library used by CERN people),
* bokeh (open-source package with interactive plots) 
Get ClassificationReport object
report has many useful methods
End of explanation
"""


features_importances = report.feature_importance()
features_importances.plot()

"""
Explanation: Plot importances of features
Only the features used in training are compared
End of explanation
"""


features_importances.plot_plotly('importances', figsize=(15, 6))

"""
Explanation: Now  feature_importances is object that can be plotted
not only in matplotlib, but in other libraries too. For instance, with plotly
End of explanation
"""


from rep.report.metrics import RocAuc

learning_curve = report.learning_curve(RocAuc(), metric_label='ROC AUC', steps=1)
learning_curve.plot(new_plot=True)

# plotting the same curve (without recomputing) using different plotting library
learning_curve.plot_plotly(plotly_filename='learning curves', figsize=(18, 8))

"""
Explanation: Plot learning curves to see possible overfitting of trained classifier
Learning curves are powerful and simple tool to analyze the behaviour of your model.
End of explanation
"""


correlation_pairs = []
correlation_pairs.append((plot_variables[0], plot_variables[1]))
correlation_pairs.append((plot_variables[0], plot_variables[2]))

report.scatter(correlation_pairs, alpha=0.01).plot()

"""
Explanation: Plot correlation between features
End of explanation
"""


# plot correlations between variables for signal-like and bck-like events
report.features_correlation_matrix(features=plot_variables).plot(new_plot=True, show_legend=False, figsize=(7, 5))

report.features_correlation_matrix_by_class(features=plot_variables).plot(new_plot=True, show_legend=False, figsize=(15, 5))

# plot correlations between variables just for bck-like events
corr = report.features_correlation_matrix_by_class(features=plot_variables[:4], labels_dict={0: 'background'}, grid_columns=1)
corr.plot_plotly(plotly_filename='correlations', show_legend=False, fontsize=8, figsize=(8, 6))

"""
Explanation: Plot data information: features correlation matrix
End of explanation
"""


# use just common features for all classifiers
report.features_pdf().plot()

# use all features in data
report.features_pdf(data.columns).plot_plotly('distributions')

"""
Explanation: Plot distribution for each feature
End of explanation
"""


report.prediction_pdf().plot(new_plot=True, figsize = (9, 4))

report.prediction_pdf(labels_dict={0: 'background'}, size=5).plot_plotly('models pdf')

"""
Explanation: Plot predictions distributions
End of explanation
"""


report.roc().plot(xlim=(0.5, 1))

report.roc(physical_notion=False).plot(ylim=(0.5, 1))

# plot the same distribution using interactive plot
report.roc().plot_plotly(plotly_filename='ROC')

"""
Explanation: ROC curves (receiver operating characteristic)
Plot roc curve for train, test data (it's the same as BackgroundRejection vs Signal Efficiency plot)
End of explanation
"""


efficiencies = report.efficiencies(['feature_3'], ignored_sideband=0.01)
efficiencies_with_errors = report.efficiencies(['feature_3'], errors=True, bins=15, ignored_sideband=0.01)

efficiencies.plot(figsize=(18, 25), fontsize=12, show_legend=False)
efficiencies_with_errors.plot(figsize=(18, 25), fontsize=12, show_legend=False)

efficiencies.plot_plotly("efficiencies", show_legend=False, figsize=(18, 20))
efficiencies_with_errors.plot_plotly("efficiencies error", show_legend=False, figsize=(18, 20))

"""
Explanation: Plot 'flatness' of classifier prediction
(this is dependence of efficiency on variables of dataset)
End of explanation
"""


# define metric functions of interest
def AMS(s, b): 
    b_reg = 0.01
    radicand = 2 *( (s+b+b_reg) * numpy.log (1.0 + s/(b+b_reg)) - s)
    return numpy.sqrt(radicand)

metrics = report.metrics_vs_cut(AMS, metric_label='AMS')
metrics.plot(new_plot=True, figsize=(15, 6))

from rep.report.metrics import significance
metrics = report.metrics_vs_cut(significance, metric_label='significance')
metrics.plot(new_plot=True, figsize=(15, 6))

"""
Explanation: Quality on different metrics
look how you can estimate the quality with your custom binary metrics and look for optimal threshold
End of explanation
"""


metrics.plot_plotly('metrics')

"""
Explanation: The same using plotly
End of explanation
"""


