execfile('preprocessing.py')

# keep only good rows
chosen_indices = range(0, 53) + range(68, 109) + range(113, 169) + range(183, 234) + range(248, 294)
train_chosen = train.iloc[:, chosen_indices]
print train_chosen.shape, train.shape

"""
Explanation: Load data
Extract labels
Remove labels and quote number
    Convert date to datetime
End of explanation
"""


X, y = convert_data(train, cat2vectors = False, normalize_numeric = False)
print X.shape, y.shape

"""
Explanation: convert date.
extract year, month, day, weekday from date and drop date
End of explanation
"""


Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, test_size=0.1, random_state=888)
#Xvalidation, Xtest, yvalidation, ytest = train_test_split(Xtest, ytest, test_size = 0.7, random_state = 8)
print Xtrain.shape, Xvalidation.shape, Xtest.shape

print xgb.cv?

def check_xboost(Xtrain, ytrain, Xvalidation, yvalidation, Xtest, ytest, num_boost_root=30):
    dtrain = xgb.DMatrix(Xtrain, ytrain)
    clf = xgb.XGBClassifier(learning_rate=0.02, max_depth = 10, 
                            subsample =0.85, colsample_bytree = 0.66)
    booster_params = clf.get_params()
    booster_params['eval_metric'] = "auc"
    booster_params['eval_set'] = ((Xvalidation, yvalidation), (Xtrain, ytrain))
    v = xgb.cv(booster_params, dtrain, num_boost_round = num_boost_round,
              early_stopping_rounds = 150
              )
    print v

def check_classifier(Xtrain, ytrain, Xvalidation, yvalidation, Xtest, ytest, clf):
    print Xtrain.shape, ytrain.shape, Xvalidation.shape, yvalidation.shape
    clf.fit(Xtrain, ytrain,
           #eval_metric = "auc", 
            #early_stopping_rounds = 150,
            #eval_set=((Xtrain, ytrain), (Xvalidation, yvalidation), ),
            #verbose = False
           )
    print clf.score(Xtrain, ytrain)
    print clf.score(Xtest, ytest)

clf.fit?

%%timeit -n 1 -r 1
dc = lambda: xgb.XGBClassifier(n_estimators=100,learning_rate=0.02, max_depth = 10, subsample =0.85, 
                               colsample_bytree = 0.66)
clf = dc()
check_classifier(Xtrain, ytrain, Xvalidation, yvalidation, Xtest, ytest,clf )
#check_xboost(Xtrain, ytrain, Xvalidation, yvalidation, Xtest, ytest)

clf = dc()
clfbag = BaggingClassifier(clf, n_estimators=5)
check_classifier(Xtrain, ytrain,  Xvalidation, yvalidation, Xtest, ytest,clfbag )

clf = dc()
clf_isotonic = CalibratedClassifierCV(clf, cv=5, method='isotonic')
check_classifier(Xtrain, ytrain,  Xvalidation, yvalidation, Xtest, ytest,clf_isotonic )

from sklearn.linear_model import LogisticRegression
clf = LogisticRegression()
clf_isotonic = CalibratedClassifierCV(clf, cv=5, method='isotonic')
check_classifier(Xtrain, ytrain, Xtest, ytest,clf_isotonic )


clf = RandomForestClassifier(n_estimators=111, n_jobs=-1)
check_classifier(Xtrain, ytrain, Xtest, ytest,clf )

clf = RandomForestClassifier(n_estimators=111, n_jobs=-1)
clfbag = BaggingClassifier(clf, n_estimators=5)
check_classifier(Xtrain, ytrain, Xtest, ytest,clfbag )

clf = RandomForestClassifier(n_estimators=111, n_jobs=-1)
clf_isotonic = CalibratedClassifierCV(clf, cv=5, method='isotonic')
check_classifier(Xtrain, ytrain, Xtest, ytest,clf_isotonic )


clf = RandomForestClassifier(n_estimators=111, n_jobs=-1)
check_classifier(Xtrain, ytrain, Xtest, ytest,clf )

clf = RandomForestClassifier(n_estimators=111, n_jobs=-1)
clfbag = BaggingClassifier(clf, n_estimators=5)
check_classifier(Xtrain, ytrain, Xtest, ytest,clfbag )

clf = RandomForestClassifier(n_estimators=111, n_jobs=-1)
clf_isotonic = CalibratedClassifierCV(clf, cv=5, method='isotonic')
check_classifier(Xtrain, ytrain, Xtest, ytest,clf_isotonic )

from sklearn.naive_bayes import GaussianNB

clf = GaussianNB()
check_classifier(Xtrain, ytrain, Xtest, ytest,clf )

clf = GaussianNB()
clfbag = BaggingClassifier(clf, n_estimators=5)
check_classifier(Xtrain, ytrain, Xtest, ytest,clfbag )

clf = GaussianNB()
clf_isotonic = CalibratedClassifierCV(clf, cv=5, method='isotonic')
check_classifier(Xtrain, ytrain, Xtest, ytest,clf_isotonic )

from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from itertools import product
from sklearn.ensemble import VotingClassifier
dc = lambda: KNeighborsClassifier(7)

clf = dc()
check_classifier(Xtrain, ytrain, Xtest, ytest,clf )

clf = dc()
clfbag = BaggingClassifier(clf, n_estimators=5)
check_classifier(Xtrain, ytrain, Xtest, ytest,clfbag )

clf = dc()
clf_isotonic = CalibratedClassifierCV(clf, cv=5, method='isotonic')
check_classifier(Xtrain, ytrain, Xtest, ytest,clf_isotonic )

from sklearn.ensemble import VotingClassifier
from sklearn.naive_bayes import GaussianNB
clf1 = LogisticRegression(random_state=1)
clf1_isotonic = CalibratedClassifierCV(clf1, cv=5, method='isotonic')

clf2 = RandomForestClassifier(n_estimators = 111)
clf2_isotonic = CalibratedClassifierCV(clf2, cv=5, method='isotonic')

clf3 = GaussianNB()
clf3_isotonic = CalibratedClassifierCV(clf3, cv=5, method='isotonic')

eclf = VotingClassifier(estimators=[('lr', clf1_isotonic), 
                                    ('rf', clf2_isotonic), 
                                    ('gnb', clf3_isotonic)], voting='soft')
check_classifier(Xtrain, ytrain, Xtest, ytest,eclf )

import sklearn
sklearn.__file__

"""
Explanation: label encoder vs one hot encoder
median calculation
data normalization (without median)
End of explanation
"""


