"""
# <h1><center> Predictive Analysis - Porto Seguro’s Safe Driver Prediction | Kaggle
"""
"""
# <h1><center> I. Importation & Missing Value Check
"""
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.offline as py     # 绘图的函数


import plotly.graph_objs as go              # 可用于绘制不同图型，如 go.bar()
import plotly.express as px                 # 可用于绘制不同图型，如 px.bar()
from plotly.subplots import make_subplots   # 创建子图
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
init_notebook_mode(connected=True)    #THIS LINE IS MOST IMPORTANT AS THIS WILL DISPLAY PLOT ON 
#NOTEBOOK WHILE KERNEL IS RUNNING
train = pd.read_csv('../input/porto-seguro-safe-driver-prediction/train.csv')
test = pd.read_csv('../input/porto-seguro-safe-driver-prediction/test.csv')
train.head()
# Function to calculate missing values by column# Funct 
def missing_values_table(df):
        # Total missing values
        mis_val = df.isnull().sum()

        # Percentage of missing values
        mis_val_percent = 100 * df.isnull().sum() / len(df)

        # Make a table with the results
        mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)

        # Rename the columns
        mis_val_table_ren_columns = mis_val_table.rename(
        columns = {0 : 'Missing Values', 1 : '% of Total Values'})

        # Sort the table by percentage of missing descending
        mis_val_table_ren_columns = mis_val_table_ren_columns[
            mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(
        '% of Total Values', ascending=False).round(1)

        # Print some summary information
        print ("Your selected dataframe has " + str(df.shape[1]) + " columns.\n"      
            "There are " + str(mis_val_table_ren_columns.shape[0]) +
            " columns that have missing values.")

        # Return the dataframe with missing information
        return mis_val_table_ren_columns
train_copy = train
train_copy = train_copy.replace(-1, np.NaN)
# Missing values statistics
missing_values = missing_values_table(train_copy)  # train_copy 是一个 dataframe
missing_values.head(20)
"""
可以看出 `ps_car_03_cat` 和 `ps_car_05_cat` 所占缺失值比例很高
"""
"""
# <center>II. Data Cleaning & Visualisation
"""
# train_counts = train.target.value_counts()
# train_counts = pd.DataFrame(train_counts)

# fig = px.bar(train_counts,x=train_counts.index,y='target',barmode='group',color='target')
# fig.update_traces(textposition='outside')
# fig.update_layout(template='seaborn',title='target (counts)')
# fig.show()
"""
### 目标变量检测

对于分类变量我们需要进行目标变量检测，如果数据存在严重的不平衡，预测得出的结论往往也是有偏的，即分类结果会偏向于较多观测的类。

"""
train_counts = train.target.value_counts()
train_counts = pd.DataFrame(train_counts)

fig = px.bar(train_counts,x=train_counts.index,y='target',barmode='group',color='target',text='target') # text 可以标上数值
fig.update_traces(textposition='outside')
fig.update_layout(yaxis_title='counts',xaxis_title='target',template='seaborn',title='target (counts)')
fig.show()
"""
### 二进制数据检测 `bin`
"""
bin_col = [col for col in train.columns if '_bin' in col]
zero_list = []
one_list = []
for col in bin_col:
    zero_list.append((train[col]==0).sum())
    one_list.append((train[col]==1).sum())
trace1 = go.Bar(
    x=bin_col,
    y=zero_list ,
    name='Zero count'
)
trace2 = go.Bar(
    x=bin_col,
    y=one_list,
    name='One count'
)

data = [trace1, trace2]
layout = go.Layout(
    barmode='stack',
    title='Count of 1 and 0 in binary variables'
)

fig = go.Figure(data=data, layout=layout)
fig.show()

"""
我们能看到，对于`10_bin` `11_bin` `12_bin` `13_bin` 基本上全是目标值都为 0，那么我们可以初步判断这几个特征可能对我们的目标值预测起不了作用，等一下我们也可以再进一步的去验证。
"""
"""
## 数据不平衡

可以发现标签之前存在不平衡的状态，如果数据存在严重的不平衡，预测得出的结论往往也是有偏的，即分类结果会偏向于较多观测的类。

比如我们使用准确率来进行模型的评估，即使我们全部预测成 `target == 0`，那么也有很高的准确率: `573518/(573518+21694)=0.96`。

所以对于分类不平衡的数据，我们可以进行如下操作：

### 欠采样：

随机欠采样（下采样）的目标是通过随机地消除占多数的类的样本来平衡类分布；直到多数类和少数类的实例实现平衡，目标才算达成。

* `随机欠采样（下采样）`的目标是通过随机地消除占多数的类的样本来平衡类分布；直到多数类和少数类的实例实现平衡，目标才算达成。

* `随机下采样的优点：`
    
    它可以提升运行时间；并且当训练数据集很大时，可以通过减少样本数量来解决存储问题。

* `随机下采样的缺点：`
    
    它会丢弃对构建规则分类器很重要的有价值的潜在信息。

    被随机欠采样选取的样本可能具有偏差。它不能准确代表大多数。从而在实际的测试数据集上得到不精确的结果。果。


### 过采样


* `随机过采样` 通过随机复制少数类来增加其中的实例数量，从而可增加样本中少数类的代表性。

* `随机过采样的优点：`

    与欠采样不同，这种方法不会带来信息损失。

    表现优于欠采样。

* `随机过采样的缺点：`
    
    由于复制少数类事件，它加大了过拟合的可能性。
    

本 notebook 将采用 SMOTE 来进行过采样
"""
from sklearn.model_selection import train_test_split

import lightgbm as lgb
from xgboost import XGBClassifier

from sklearn.metrics import confusion_matrix
import itertools

from sklearn.metrics import accuracy_score
from sklearn.metrics import recall_score
"""
### 不进行采样，直接进行模型训练和预测查看模型质量

* 通过对比准确度和召回率
    
* 通过绘制混淆矩阵
"""
"""
### 1、对比准确度和召回率
"""
train_target = train['target']
# train_feature = train.drop(columns='target')
train_feature = train.drop(columns = ['target','id'])


x_train,x_test,y_train,y_test = train_test_split(train_feature,train_target,test_size= 0.2,random_state=10)
# model = XGBClassifier(n_estimators=1000)
model = XGBClassifier()


model.fit(x_train,y_train)
y_pred = model.predict(x_test)

acc = accuracy_score(y_test,y_pred)
recall = recall_score(y_test,y_pred)

print('Accuracy: {:.3f}'.format(acc* 100.0))
print('recall: {:.3f}'.format(recall* 100.0))
"""
可以看到，即使你的准确率非常高，但是召回率却非常低    
"""
"""
### 2、绘制混淆矩阵

混淆矩阵用法示例，用于评估数据集上分类器输出的质量。 对角线元素表示预测标签等于真实标签的点数，而非对角线元素则是分类器未正确标记的元素。 混淆矩阵的对角线值越高，表示对数越多越好。混淆矩阵用法示例，用于评估数据集上分类器输出的质量。 对角线元素表示预测标签等于真实标签的点数，而非对角线元素则是分类器未正确标记的元素。 混淆矩阵的对角线值越高，表示对数越多越好。
"""
def plot_confusion_matrix(cm, classes,
                                normalize=False,
                                title='Confusion matrix',
                                cmap=plt.cm.Blues):
        """
            此函数打印并绘制混淆矩阵。
            可以通过设置“ normalize = True”来应用归一化。
        """
        plt.imshow(cm, interpolation='nearest', cmap=cmap)
        plt.title(title)
        plt.colorbar()
        tick_marks = np.arange(len(classes))
        plt.xticks(tick_marks, classes, rotation=45)
        plt.yticks(tick_marks, classes)

        if normalize:
            cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
            print("Normalized confusion matrix")
        else:
            print('Confusion matrix, without normalization')

        print(cm)

        thresh = cm.max() / 2.
        for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
            plt.text(j, i, cm[i, j],
                        horizontalalignment="center",
                        color="white" if cm[i, j] > thresh else "black")

        plt.tight_layout()
        plt.ylabel('True label')
        plt.xlabel('Predicted label')
classes = ['target_0','target_1'] # 顺序别搞错
np.set_printoptions(precision=2)

cm = confusion_matrix(y_test,y_pred)
plt.figure()
plot_confusion_matrix(cm,classes)
plt.show()

# plt.figure()
# plot_confusion_matrix(cm,classes,normalize=True)
# plt.show()
"""
### 使用 SMOTE 进行过采样
"""
from imblearn.over_sampling import SMOTE
from sklearn.preprocessing import StandardScaler
"""
### 查看2D数据的分布
"""
# 定义绘图函数
def plot_2d_space(X, y, label='Classes'):   
            colors = ['#1F77B4', '#FF7F0E']
            markers = ['o', 's']
            for l, c, m in zip(np.unique(y), colors, markers):
                plt.scatter(
                    X[y==l, 0],
                    X[y==l, 1],
                    c=c, label=l, marker=m
                )
            plt.title(label)
            plt.legend(loc='upper right')
            plt.show()
            
print("label0: ",len(x_train[y_train==0]))
print("label1: ",len(x_train[y_train==1]))

ss = StandardScaler()
X = ss.fit_transform(x_train)

# `2、`如果数据存在多维特征可使用PCA来降维，使其能在2D图中展示

from sklearn.decomposition import PCA

pca = PCA(n_components=2)
X = pca.fit_transform(X)

plot_2d_space(X, y_train, 'Imbalanced dataset (2 PCA components)')
oversampler=SMOTE(random_state=0)
# 开始人工合成数据
os_features,os_labels=oversampler.fit_sample(x_train,y_train)

# 查看生成结果
print("label1: ",len(os_labels[os_labels==1]))
print("label0: ",len(os_labels[os_labels==0]))
oversampler=SMOTE(random_state=0)
# 开始人工合成数据
os_features_test,os_labels_test=oversampler.fit_sample(x_test,y_test)

# 查看生成结果
print("label1: ",len(os_labels_test[os_labels_test==1]))
print("label0: ",len(os_labels_test[os_labels_test==0]))
ss = StandardScaler()
X = ss.fit_transform(os_features)

pca = PCA(n_components=2)
X = pca.fit_transform(X)

plot_2d_space(X, os_labels, 'Imbalanced dataset (2 PCA components)')
"""
### 特征相关性检测
"""
"""
1、特征和目标特征之间的相关性检测
"""
new_os_features = os_features.copy()
new_os_features['target'] = os_labels

# Find correlations with the target and sort
corrs = new_os_features.corr()['target'].sort_values(ascending=False)
correlations = pd.DataFrame(corrs)

# Display correlations
print('Most Positive Correlations:\n')
correlations.head()
print('Most Negative Correlations:\n')
correlations.tail()
correlations.loc[correlations.index.isin(['ps_ind_10_bin','ps_ind_11_bin','ps_ind_12_bin','ps_ind_13_bin'])]
np.abs(corrs).sort_values(ascending=False).tail(15)
"""
我们可以看出 在 `bin` 类型的特征中 `ps_ind_10_bin`,`ps_ind_11_bin`,`ps_ind_12_bin`,`ps_ind_13_bin` 确实是影响力最低的特征
"""
"""
2、特征与特征之间进行相关性检测
"""
corrs = os_features.corr()
"""
3、删除共线特征
"""
# 设置阈值
threshold = 0.8

# 创建一个空字典以容纳相关变量
above_threshold_vars = {}

# 对于每一列，记录index行中的那个值高于阈值的变量
for col in corrs:
    above_threshold_vars[col] = list(corrs.index[corrs[col] > threshold])
# above_threshold_vars
# 跟踪要删除的列和已检查的列
cols_to_remove = []
cols_seen = []
cols_to_remove_pair = []

# 遍历列和相关列
for key, value in above_threshold_vars.items():
    # 跟踪已检查的列
    cols_seen.append(key)
    for x in value:
        if x == key:
            next
        else:
            # 如果存在高相关的特征，只保留一个
            if x not in cols_seen:                  # 如果该特征在之前的 key 数据中没有出现过。
                cols_to_remove.append(x)            # 存在高度相关，将高度相关的特征放入 cols_to_remove 中。
                cols_to_remove_pair.append(key)     # cols_to_remove 和 cols_to_remove_pair 得到的结果一致。

cols_to_remove = list(set(cols_to_remove))
print('Name of columns to remove: ', cols_to_remove)
print('Number of columns to remove: ', len(cols_to_remove))
train_corrs_removed = os_features.drop(columns = cols_to_remove)
test_corrs_removed = os_features_test.drop(columns = cols_to_remove)

print('Training Corrs Removed Shape: ', train_corrs_removed.shape)
print('Testing Corrs Removed Shape: ', test_corrs_removed.shape)
train_corrs_removed.to_csv('train_corrs_removed.csv', index = False)
test_corrs_removed.to_csv('test_corrs_removed.csv', index = False)
"""
# <center>III. ML Approach
"""
"""
### XGBoost训练模型

使用 gini 系数评估 + 交叉验证
"""
# `1、定义基尼系数：`

def gini(y, pred):
    g = np.asarray(np.c_[y, pred, np.arange(len(y)) ], dtype=np.float)
    g = g[np.lexsort((g[:,2], -1*g[:,1]))]
    gs = g[:,0].cumsum().sum() / g[:,0].sum()
    gs -= (len(y) + 1) / 2.
    return gs / len(y)

# `2、定义 xgb gini 系数：`

# 返回一个 normalized 后的 gini 分数
def gini_xgb(pred, y):
    y = y.get_label()
    return 'gini', gini(y, pred) / gini(y, y)
"""
### 过采样前数据进行训练
"""
from sklearn.model_selection import StratifiedKFold
import xgboost as xgb
import lightgbm as lgb
params = {'eta': 0.02, 'max_depth': 4, 'subsample': 0.9, 'colsample_bytree': 0.9, 
          'objective': 'binary:logistic', 'eval_metric': 'auc', 'silent': True}

nrounds=200  
kfold = 2
skf = StratifiedKFold(n_splits=kfold, random_state=0)

for i, (train_index, test_index) in enumerate(skf.split(train_feature, train_target)):
    print(' xgb kfold: {}  of  {} : '.format(i+1, kfold))
    X_train, X_valid = train_feature.loc[train_index], train_feature.loc[test_index]
    y_train, y_valid = train_target.loc[train_index], train_target.loc[test_index]
    d_train = xgb.DMatrix(X_train, y_train) 
    d_valid = xgb.DMatrix(X_valid, y_valid) 
    watchlist = [(d_train, 'train'), (d_valid, 'valid')]
    xgb_model = xgb.train(params, d_train, nrounds, watchlist, early_stopping_rounds=100, 
                          feval=gini_xgb, maximize=True, verbose_eval=100)
    
# xgb_y_pred = xgb_model.predict(xgb.DMatrix(test[features].values), 
#                         ntree_limit=xgb_model.best_ntree_limit+50)
features = train_feature.columns
test_ids = test['id']
xgb_test_predictions1 = xgb_model.predict(xgb.DMatrix(test[features]),ntree_limit=xgb_model.best_ntree_limit+50)

submission = pd.DataFrame({'id': test_ids, 'target': xgb_test_predictions1})
submission.to_csv('before_oversampling_xgb.csv', index = False, float_format='%.5f')
"""
`Kfold = 2` and `nrounds = 200` 提交后：`score=0.254` 

`Kfold = 5` and `nrounds = 2000` 提交后：`score=0.2767` 
"""
"""
### 特征选择

绘制 xgb 模型特征的重要性图
"""
def model_feature_importances(model):
    trace = go.Scatter(
        y = np.array(list(model.get_fscore().values())),
        x = np.array(list(model.get_fscore().keys())),
        mode='markers',
        marker=dict(
            sizemode = 'diameter',
            sizeref = 1,
            size = 13,
            #size= model.feature_importances_,
            #color = np.random.randn(500), #set color equal to a variable
            color =  np.array(list(model.get_fscore().values())),
            colorscale='Portland',
            showscale=True
        ),
        text = np.array(list(model.get_fscore().keys()))
    )
    data = [trace]

    layout= go.Layout(
        autosize= True,
        title= 'xgb Feature Importance',
        hovermode= 'closest',
         xaxis= dict(
             ticklen= 5,
             showgrid=False,
            zeroline=False,
            showline=False
         ),
        yaxis=dict(
            title= 'Feature Importance',
            showgrid=False,
            zeroline=False,
            ticklen= 5,
            gridwidth= 2
        ),
        showlegend= False
    )
    fig = go.Figure(data=data, layout=layout)
    fig.show()

model_feature_importances(xgb_model)
"""
更近一步，我们还能通过绘制柱状图(横)来对特征重要性进行效果展示
"""
xgb_importance = np.array(list(xgb_model.get_fscore().values()))
xgb_features = np.array(list(xgb_model.get_fscore().keys()))

x, y = (list(x) for x in zip(*sorted(zip(xgb_importance,xgb_features), reverse = False)))
trace2 = go.Bar(
    x=x ,
    y=y,
    marker=dict(
        color=x,
        colorscale = 'Viridis',
        reversescale = True
    ),
    name='Random Forest Feature importance',
    orientation='h',
)

layout = dict(
    title='Barplot of Feature importances',
    width = 900, height = 2000,
    yaxis=dict(
        showgrid=False,
        showline=False,
        showticklabels=True,
#         domain=[0, 0.85],
    ))

fig1 = go.Figure(data=[trace2])
fig1['layout'].update(layout)
py.iplot(fig1, filename='plots')
"""
从图中我们能发现个别特征重要性非常低，我们可以进行剔除
"""
new_features = pd.DataFrame(xgb_model.get_fscore(),index=['features_importance']).T\
                        .sort_values(by='features_importance',ascending=False)\
                        .iloc[0:38]
new_features 
train_feature.shape
new_train_feature = train_feature[new_features.index]
new_train_feature.head()
new_train_feature.shape
params = {'eta': 0.02, 'max_depth': 4, 'subsample': 0.9, 'colsample_bytree': 0.9, 
          'objective': 'binary:logistic', 'eval_metric': 'auc', 'silent': True}

nrounds=200  
kfold = 2
skf = StratifiedKFold(n_splits=kfold, random_state=0)

for i, (train_index, test_index) in enumerate(skf.split(new_train_feature, train_target)):
    print(' xgb kfold: {}  of  {} : '.format(i+1, kfold))
    X_train, X_valid = new_train_feature.loc[train_index], new_train_feature.loc[test_index]
    y_train, y_valid = train_target.loc[train_index], train_target.loc[test_index]
    d_train = xgb.DMatrix(X_train, y_train) 
    d_valid = xgb.DMatrix(X_valid, y_valid) 
    watchlist = [(d_train, 'train'), (d_valid, 'valid')]
    xgb_model2 = xgb.train(params, d_train, nrounds, watchlist, early_stopping_rounds=100, 
                          feval=gini_xgb, maximize=True, verbose_eval=100)
xgb_test_predictions2 = xgb_model2.predict(xgb.DMatrix(test[new_features.index]),ntree_limit=xgb_model2.best_ntree_limit+50)

submission = pd.DataFrame({'id': test_ids, 'target': xgb_test_predictions2})
submission.to_csv('before_oversampling_after_feature_choose_xgb.csv', index = False, float_format='%.5f')
"""
### xgb + lgb 堆叠训练预测

对选出来的特征进行训练
"""
def gini_lgb(preds, dtrain):
    y = list(dtrain.get_label())
    score = gini(y, preds) / gini(y, y)
    return 'gini', score, True
# https://www.kaggle.com/rshally/porto-xgb-lgb-kfold-lb-0-282

# xgb
# params = {'eta': 0.02, 'max_depth': 4, 'subsample': 0.9, 'colsample_bytree': 0.9, 
#         'objective': 'binary:logistic', 'eval_metric': 'auc', 'silent': True}

# submission=test['id'].to_frame()
# submission['target']=0

# nrounds=200  # need to change to 2000
# kfold = 2  # need to change to 5
# skf = StratifiedKFold(n_splits=kfold, random_state=0)
# for i, (train_index, test_index) in enumerate(skf.split(new_train_feature, train_target)):
#     print(' xgb kfold: {}  of  {} : '.format(i+1, kfold))
#     X_train, X_valid = new_train_feature.loc[train_index], new_train_feature.loc[test_index]
#     y_train, y_valid = train_target.loc[train_index], train_target.loc[test_index]
#     d_train = xgb.DMatrix(X_train, y_train) 
#     d_valid = xgb.DMatrix(X_valid, y_valid) 
#     watchlist = [(d_train, 'train'), (d_valid, 'valid')]
#     xgb_model3 = xgb.train(params, d_train, nrounds, watchlist, early_stopping_rounds=100, 
#                         feval=gini_xgb, maximize=True, verbose_eval=100)

#     # 结尾 除以 (2*kfold)，是因为要将 xgb 和 lgb 去平均然后将结果相加合并
#     submission['target'] += xgb_model3.predict(xgb.DMatrix(test[new_features.index]), 
#                         ntree_limit=xgb_model3.best_ntree_limit+50) / (2*kfold)
    
# submission.head(2)

# # lgb
# params = {'metric': 'auc', 'learning_rate' : 0.01, 'max_depth':10, 'max_bin':10,  'objective': 'binary', 
#         'feature_fraction': 0.8,'bagging_fraction':0.9,'bagging_freq':10,  'min_data': 500}

# skf = StratifiedKFold(n_splits=kfold, random_state=1)
# for i, (train_index, test_index) in enumerate(skf.split(new_train_feature, os_labels)):
#     print(' lgb kfold: {}  of  {} : '.format(i+1, kfold))
#     X_train, X_eval = new_train_feature.loc[train_index], new_train_feature.loc[test_index]
#     y_train, y_eval = train_target.loc[train_index], train_target.loc[test_index]
#     lgb_model = lgb.train(params, lgb.Dataset(X_train, label=y_train), nrounds, 
#                 lgb.Dataset(X_eval, label=y_eval), verbose_eval=100, 
#                 feval=gini_lgb, early_stopping_rounds=100)

#     # 结尾 除以 (2*kfold)，是因为要将 xgb 和 lgb 去平均然后将结果相加合并
#     submission['target'] += lgb_model.predict(test[new_features.index], 
#                         num_iteration=lgb_model.best_iteration) / (2*kfold)

# submission.to_csv('before_oversampling_lgb+xgb.csv', index=False, float_format='%.5f') 

# submission.head(2)
# submission.to_csv('before_oversampling_after_feature_choose_xgb.csv', index = False, float_format='%.5f')
"""
### 过采样后数据进行训练

oversampling good or bad?

对采样后的数据进行训练，发现预测出来的结果分数很低
"""
# params = {'eta': 0.02, 'max_depth': 4, 'subsample': 0.9, 'colsample_bytree': 0.9, 
#           'objective': 'binary:logistic', 'eval_metric': 'auc', 'silent': True}

# nrounds=200  
# kfold = 2  
# skf = StratifiedKFold(n_splits=kfold, random_state=0)

# for i, (train_index, test_index) in enumerate(skf.split(os_features, os_labels)):
#     print(' xgb kfold: {}  of  {} : '.format(i+1, kfold))
#     X_train, X_valid = os_features.loc[train_index], os_features.loc[test_index]
#     y_train, y_valid = os_labels.loc[train_index], os_labels.loc[test_index]
#     d_train = xgb.DMatrix(X_train, y_train) 
#     d_valid = xgb.DMatrix(X_valid, y_valid) 
#     watchlist = [(d_train, 'train'), (d_valid, 'valid')]
#     xgb_model2 = xgb.train(params, d_train, nrounds, watchlist, early_stopping_rounds=100, 
#                           feval=gini_xgb, maximize=True, verbose_eval=100)

# xgb_test_predictions = xgb_model2.predict(xgb.DMatrix(test[features]), 
#                         ntree_limit=xgb_model2.best_ntree_limit+50)
# submission = pd.DataFrame({'id': test_ids, 'target': xgb_test_predictions})
# submission.to_csv('after_oversampling_xgb.csv', index = False, float_format='%.5f')
"""
# <center> IV. Extend...
"""
"""
### RandomForestClassifier 训练数据

接下来我们将：

* 1、使用 RandomForestClassifier 来进行训练数据

* 2、绘制特征的重要性图

* 3、并且进一步进行特征选择
"""
# from sklearn.ensemble import RandomForestClassifier
# os_features.drop(['id'],axis=1,inplace=True)
# os_features_test.drop(['id'],axis=1,inplace=True)


# rdf_clf = RandomForestClassifier(n_estimators=150, max_depth=8, min_samples_leaf=4, max_features=0.2, n_jobs=-1, random_state=0)
# rdf_clf.fit(os_features, os_labels)
# features = os_features.columns.values
# print("----- Training Done -----")
# RandomForestClassifier feature importances Scatter plot 

# def rdf_feature_importances(model):
#     trace = go.Scatter(
#         y = model.feature_importances_,
#         x = features,
#         mode='markers',
#         marker=dict(
#             sizemode = 'diameter',
#             sizeref = 1,
#             size = 13,
#             #size= model.feature_importances_,
#             #color = np.random.randn(500), #set color equal to a variable
#             color = model.feature_importances_,
#             colorscale='Portland',
#             showscale=True
#         ),
#         text = features
#     )
#     data = [trace]

#     layout= go.Layout(
#         autosize= True,
#         title= 'Random Forest Feature Importance',
#         hovermode= 'closest',
#          xaxis= dict(
#              ticklen= 5,
#              showgrid=False,
#             zeroline=False,
#             showline=False
#          ),
#         yaxis=dict(
#             title= 'Feature Importance',
#             showgrid=False,
#             zeroline=False,
#             ticklen= 5,
#             gridwidth= 2
#         ),
#         showlegend= False
#     )
#     fig = go.Figure(data=data, layout=layout)
#     fig.show()

# rdf_feature_importances(model=rdf_clf)
"""
更近一步，我们还能通过绘制柱状图(横)来对特征重要性进行效果展示
"""
# x, y = (list(x) for x in zip(*sorted(zip(rdf_clf.feature_importances_, features), 
#                                                             reverse = False)))
# trace2 = go.Bar(
#     x=x ,
#     y=y,
#     marker=dict(
#         color=x,
#         colorscale = 'Viridis',
#         reversescale = True
#     ),
#     name='Random Forest Feature importance',
#     orientation='h',
# )

# layout = dict(
#     title='Barplot of Feature importances',
#      width = 900, height = 2000,
#     yaxis=dict(
#         showgrid=False,
#         showline=False,
#         showticklabels=True,
# #         domain=[0, 0.85],
#     ))

# fig1 = go.Figure(data=[trace2])
# fig1['layout'].update(layout)
# py.iplot(fig1, filename='plots')
# y_pred = rdf_clf.predict(os_features_test)

# acc = accuracy_score(os_labels_test,y_pred)
# recall = recall_score(os_labels_test,y_pred)

# print('Accuracy: {:.3f}'.format(acc* 100.0))
# print('recall: {:.3f}'.format(recall* 100.0))
# model = XGBClassifier()


# model.fit(os_features,os_labels)
# y_pred = model.predict(os_features_test)

# acc = accuracy_score(os_labels_test,y_pred)
# recall = recall_score(os_labels_test,y_pred)

# print('Accuracy: {:.3f}'.format(acc* 100.0))
# print('recall: {:.3f}'.format(recall* 100.0))
"""
我们可以看出相比于之前的不平衡数据，过采样后的数据，召回率得到了明显的上升
"""
"""
### 对数据进行预测

使用 交叉验证 放法进行模型训练
"""
"""
随机森林预测结果：
"""
# test_predictions = rdf_clf.predict(new_test)

# submission = pd.DataFrame({'id': test_ids, 'target': test_predictions})
# submission.to_csv('RandomForestClassifier_predict_1.csv', index = False)
# submission.head()
"""
### 特征选择
"""