Credit scoring using scorecardpy with XGBoost I used XGBoost for scoring creditworthiness. At first I thought I could use predict_proba for scoring but then I saw that there was a module scorecardpy based on WOE to claculate code scoring. I tried to use it with my XGBoost like in an exemple but my ROC AUC fell to 0.5 and I don't see what I am doing wrong. Thanks for your help.

data = pd.read_csv('data.csv')


train_index = data['date'] &lt; '2018-04-01'
test_index = data['date'] &gt;= '2018-04-01'

data_final = data.drop('date', axis=1)

df_train = data_final[train_index]
df_test = data_final[test_index]

data_final_vars = data_final.columns.values.tolist()
y=['label']
X=[i for i in data_final_vars if i not in y]


# woe binning ------
bins = sc.woebin(data_final, y="label")
sc.woebin_plot(bins)

# binning adjustment
# # adjust breaks interactively
# breaks_adj = sc.woebin_adj(dt_s, "creditability", bins) 
# # or specify breaks manually
breaks_adj = {
    'age': [26, 35, 40, 50, 60]
}
bins_adj = sc.woebin(data_final, y="label", breaks_list=breaks_adj)

# converting train and test into woe values
train_woe = sc.woebin_ply(df_train, bins_adj)
test_woe = sc.woebin_ply(df_test, bins_adj)


ytrain = train_woe.loc[:,'label']
xtrain = train_woe.loc[:, train_woe.columns != 'label']
ytest = test_woe.loc[:,'label']
xtest = test_woe.loc[:, test_woe.columns != 'label']

print("shape of xtrain: {}".format(xtrain.shape))
print("shape of xtrain: {}".format(xtest.shape))

from xgboost import XGBClassifier

XGB = XGBClassifier(n_estimators=100, n_jobs=6, verbose=1)
# List the default parameters.
print(XGB.get_xgb_params())

# Train and evaluate 
XGB.fit(xtrain, ytrain, eval_metric=['rmse'], eval_set=[((xtrain, ytrain)),(xtest, ytest)])


# # Classifier

from sklearn.metrics import roc_auc_score

probs = XGB.predict_proba(xtest)
roc = roc_auc_score(y_true=ytest, y_score=probs[:, 1])
print("RF roc score: {}".format(roc))


from sklearn.metrics import roc_curve
fpr, tpr, thresholds = roc_curve(ytest, probs[:,1])
plt.figure()
plt.plot(fpr, tpr, label='XGBoost Classifier (area = %0.2f)' % roc)
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic')
plt.legend(loc="lower right")
plt.savefig('XGB_ROC')


from sklearn import model_selection
from sklearn.model_selection import cross_val_score
kfold = model_selection.KFold(n_splits=10, random_state=7)
modelCV = XGB
scoring = 'accuracy'
results = model_selection.cross_val_score(modelCV, xtrain, ytrain, cv=kfold, scoring=scoring)
print("10-fold cross validation average accuracy: {}".format(results.mean()))


# score ------
card = sc.scorecard(bins_adj, XGB, xtrain.columns)
# credit score
train_score = sc.scorecard_ply(df_train, card, print_step=0)
test_score = sc.scorecard_ply(df_test, card, print_step=0)

# psi
sc.perf_psi(
  score = {'train':train_score, 'test':test_score},
  label = {'train':y_train, 'test':y_test}
)
It happened to me as well, although I used a logistic regression model not XGBoost.

The problem is not about which model to choose, but rather there is something wrong with woebin_ply function. I didn't read the source code but the woe value I'm getting doesn't match the value for the corresponding bin/input value (You can double check your results as well).

After manually matching input value with bin with corresponding woe value, my scorecard model performs at similar level with my benchmarking models.

Hope this help!