# Basic Libraries
import numpy as np
import pandas as pd

# Visualizations Libraries
import matplotlib.pyplot as plt
import seaborn as sns

import plotly.express as px
from plotly.offline import iplot
from plotly.subplots import make_subplots
import plotly.offline as py
import plotly.figure_factory as ff


# Data Pre-processing Libraries
from sklearn.preprocessing import StandardScaler,MinMaxScaler,OneHotEncoder,PowerTransformer,OrdinalEncoder
from sklearn.model_selection import train_test_split,cross_validate, cross_val_score,GridSearchCV
from sklearn.pipeline import Pipeline,make_pipeline

from sklearn.compose import make_column_transformer,make_column_selector
from sklearn.feature_selection import SelectKBest,f_classif,mutual_info_classif,chi2,SelectFromModel

# Modelling Libraries
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC,LinearSVC,SVR
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier,AdaBoostClassifier,GradientBoostingClassifier
from xgboost import XGBClassifier
from catboost import CatBoostClassifier
from lightgbm import LGBMClassifier

# Evaluation & CV Libraries
from sklearn.metrics import precision_score,accuracy_score,mean_squared_error,r2_score,confusion_matrix
from sklearn.metrics import classification_report, plot_confusion_matrix,roc_auc_score,f1_score,recall_score

import optuna

import warnings
warnings.filterwarnings("ignore")
plt.rcParams["figure.figsize"] = (10,6)
pd.set_option("max_columns",100)
pd.set_option("max_rows",900)
pd.set_option("max_colwidth",200)
data = pd.read_csv("../input/heart-failure-prediction/heart.csv")
df=data.copy()
df.head()
df.describe()
df.skew()
def missing(df):
    missing_number = df.isnull().sum().sort_values(ascending=False)
    missing_percent = (df.isnull().sum()/df.isnull().count()).sort_values(ascending=False)
    missing_values = pd.concat([missing_number, missing_percent], axis=1, keys=['Missing_Number', 'Missing_Percent'])
    return missing_values

missing(df)
df.duplicated().sum()
numerical= df.drop(['HeartDisease'], axis=1).select_dtypes('number').columns

categorical = df.select_dtypes('object').columns
df["HeartDisease"].value_counts()
matrix = np.triu(df.corr())
fig,ax = plt.subplots(figsize=(12,6),dpi=100)
sns.heatmap(df.corr(),annot=True,vmax=1,vmin=-1,center=0,ax=ax,mask=matrix,fmt=".2f");
plt.figure(figsize=(15,5))
sns.countplot(x ='Age', data = df)
plt.title('Age Distribution')
plt.ylabel('Age')
plt.show();
plt.figure(figsize=(15,5))
plt.subplot(221)
sns.histplot(x='Age', data=df, kde =True)
plt.title('Age Distribution')
plt.xlabel('Age')
plt.ylabel('Count')

plt.subplot(222)
sns.histplot(x ='Cholesterol', data=df, color='red', kde = True)
plt.title('Cholesterol Distribution')
plt.xlabel('Cholesterol')
plt.ylabel('Count')

plt.figure(figsize=(15,5))
plt.subplot(223)
sns.histplot(x='MaxHR', data=df, kde =True)
plt.title('MaxHR Distribution')
plt.xlabel('MaxHR')
plt.ylabel('Count')

plt.subplot(224)
sns.histplot(x ='RestingBP', data=df, color='red', kde = True)
plt.title('RestingBP Distribution')
plt.xlabel('RestingBP')
plt.ylabel('Count');
plt.figure(figsize=(8,8))

explode = [0,0.1]
plt.pie(df['Sex'].value_counts(), explode=explode,autopct='%1.1f%%', shadow=True,startangle=140)
plt.legend(labels=['Male','Female'])
plt.title('Male and Female Distribution')
plt.axis('off');
fig,ax = plt.subplots(figsize=(6,6),dpi =100)
sns.countplot(x='Sex', data=df,ax=ax)
plt.title('Sex Distribution')
plt.xlabel('Sex')
plt.ylabel('Count')
for p in ax.patches:
    ax.annotate(str(p.get_height()), (p.get_x() +0.4, p.get_height() + 10));
px.histogram(df, x=df.ExerciseAngina, color="HeartDisease",facet_col="Sex")
px.histogram(df, x="RestingECG", color="HeartDisease",facet_col="Sex")
px.histogram(df, x="Sex", color="HeartDisease",width=800)
px.box(df,x=df.MaxHR,facet_col="HeartDisease",animation_frame=df.Age)
plt.figure(figsize=(8,8))

explode = [0,0.1]
plt.pie(df['HeartDisease'].value_counts(), explode=explode,autopct='%1.1f%%', shadow=True,startangle=140)
plt.legend(labels=['1','0'])
plt.title('HeartDisease Distribution')
plt.axis('off');
"""
## **Compare models with default values.**
"""
ohe = OneHotEncoder(sparse=False,handle_unknown="ignore")
scaled = StandardScaler()

X= df.drop('HeartDisease', axis=1)
y= df['HeartDisease']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)


ct =make_column_transformer((ohe,categorical),
                            (scaled,numerical),remainder='passthrough')

pipe = make_pipeline(ct,LogisticRegression(random_state=42,class_weight="balanced"))
pipe.fit(X_train,y_train)
y_pred = pipe.predict(X_test)
y_pred_train = pipe.predict(X_train)

log_f1 = f1_score(y_test, y_pred)
log_recall = recall_score(y_test, y_pred)
log_auc = roc_auc_score(y_test, y_pred)
print("-------------------------")
print("LOG_MODEL")
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))


pipe = make_pipeline(ct,KNeighborsClassifier())
pipe.fit(X_train,y_train)
y_pred = pipe.predict(X_test)
y_pred_train = pipe.predict(X_train)

knn_f1 = f1_score(y_test, y_pred)
knn_recall = recall_score(y_test, y_pred)
knn_auc = roc_auc_score(y_test, y_pred)
print("-------------------------")
print("KNN_MODEL")
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))

pipe = make_pipeline(ct,SVC(random_state=42))
pipe.fit(X_train,y_train)
y_pred = pipe.predict(X_test)
y_pred_train = pipe.predict(X_train)

svc_f1 = f1_score(y_test, y_pred)
svc_recall = recall_score(y_test, y_pred)
svc_auc = roc_auc_score(y_test, y_pred)
print("-------------------------")
print("SVM_MODEL")
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))

pipe = make_pipeline(ct,DecisionTreeClassifier(random_state=42))
pipe.fit(X_train,y_train)
y_pred = pipe.predict(X_test)
y_pred_train = pipe.predict(X_train)

dt_f1 = f1_score(y_test, y_pred)
dt_recall = recall_score(y_test, y_pred)
dt_auc = roc_auc_score(y_test, y_pred)
print("-------------------------")
print("DT_MODEL")
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))

pipe = make_pipeline(ct,RandomForestClassifier(random_state=42))
pipe.fit(X_train,y_train)
y_pred = pipe.predict(X_test)
y_pred_train = pipe.predict(X_train)


rf_f1 = f1_score(y_test, y_pred)
rf_recall = recall_score(y_test, y_pred)
rf_auc = roc_auc_score(y_test, y_pred)
print("-------------------------")
print("RF_MODEL")
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))

pipe = make_pipeline(ct,AdaBoostClassifier(n_estimators=50, random_state=42))
pipe.fit(X_train,y_train)
y_pred = pipe.predict(X_test)
y_pred_train = pipe.predict(X_train)


ada_f1 = f1_score(y_test, y_pred)
ada_recall = recall_score(y_test, y_pred)
ada_auc = roc_auc_score(y_test, y_pred)
print("-------------------------")
print("ADABOOST_MODEL")
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test,y_pred))


pipe = make_pipeline(ct,GradientBoostingClassifier(random_state=42))
pipe.fit(X_train,y_train)
y_pred = pipe.predict(X_test)
y_pred_train = pipe.predict(X_train)


gb_f1 = f1_score(y_test, y_pred)
gb_recall = recall_score(y_test, y_pred)
gb_auc = roc_auc_score(y_test, y_pred)
print("-------------------------")
print("GRADİENT BOSTİNG_MODEL")
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test,y_pred))

pipe = make_pipeline(ct,XGBClassifier(random_state=42))
pipe.fit(X_train,y_train)
y_pred = pipe.predict(X_test)
y_pred_train = pipe.predict(X_train)


xgb_f1 = f1_score(y_test, y_pred)
xgb_recall = recall_score(y_test, y_pred)
xgb_auc = roc_auc_score(y_test, y_pred)
print("-------------------------")
print("XGB_MODEL")
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test,y_pred))

pipe = make_pipeline(ct,CatBoostClassifier(random_state=42,verbose=0))
pipe.fit(X_train,y_train)
y_pred = pipe.predict(X_test)
y_pred_train = pipe.predict(X_train)
cat_f1 = f1_score(y_test, y_pred)
cat_recall = recall_score(y_test, y_pred)
cat_auc = roc_auc_score(y_test, y_pred)
print("-------------------------")
print("CATBOOST_MODEL")
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test,y_pred))


compare = pd.DataFrame({"Model": ["Logistic Regression", "KNN", "SVM", "Decision Tree", "Random Forest", "AdaBoost",
                                 "GradientBoost", "XGBoo","CatBoost"],
                        "F1": [log_f1, knn_f1, svc_f1, dt_f1, rf_f1, ada_f1, gb_f1, xgb_f1,cat_f1 ],
                        "Recall": [log_recall, knn_recall, svc_recall, dt_recall, rf_recall, ada_recall, gb_recall, xgb_recall,cat_recall],
                        "ROC_AUC": [log_auc, knn_auc, svc_auc, dt_auc, rf_auc, ada_auc, gb_auc, xgb_auc,cat_auc]})

def labels(ax):
    for p in ax.patches:
        width = p.get_width()                        # get bar length
        ax.text(width,                               # set the text at 1 unit right of the bar
                p.get_y() + p.get_height() / 2,      # get Y coordinate + X coordinate / 2
                '{:1.3f}'.format(width),             # set variable to display, 2 decimals
                ha = 'left',                         # horizontal alignment
                va = 'center')                       # vertical alignment
    
plt.figure(figsize=(14,10))
plt.subplot(311)
compare = compare.sort_values(by="F1", ascending=False)
ax=sns.barplot(x="F1", y="Model", data=compare, palette="Blues_d")
labels(ax)

plt.subplot(312)
compare = compare.sort_values(by="Recall", ascending=False)
ax=sns.barplot(x="Recall", y="Model", data=compare, palette="Blues_d")
labels(ax)

plt.subplot(313)
compare = compare.sort_values(by="ROC_AUC", ascending=False)
ax=sns.barplot(x="ROC_AUC", y="Model", data=compare, palette="Blues_d")
labels(ax)
plt.show()
"""
## **SVC Model**
"""
def objective(trial):
    
    X= df.drop('HeartDisease', axis=1)
    y= df['HeartDisease']
    
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)
    
    params = {"C":trial.suggest_int("C",1,5),
             "degree":trial.suggest_int("degree",2,6),
             "kernel": trial.suggest_categorical("kernel",("rbf","linear")),
             "coef0":trial.suggest_float("coef0",0,1)}
    ct =  make_column_transformer((StandardScaler(),make_column_selector(dtype_exclude=object)),
         (OneHotEncoder(sparse=False,handle_unknown="ignore"),make_column_selector(dtype_include=object)),remainder="passthrough")
    
    model =SVC(random_state=42)
    pipe = make_pipeline(ct,model)
    pipe.fit(X_train,y_train)
    preds = pipe.predict(X_test)
    pred_labels = np.rint(preds)
    accuracy = accuracy_score(y_test, pred_labels)
    return accuracy

if __name__ == "__main__":
    study = optuna.create_study(direction="maximize")
    study.optimize(objective, n_trials=50, timeout=600)

    trial = study.best_trial

    print("  Params: ")
    for key, value in trial.params.items():
        print("    {}: {}".format(key, value))
svc_model = SVC(C= 2,degree= 2,kernel = "rbf",coef0=  0.4371031882388341,random_state=42,verbose =0 )

ct = make_column_transformer((ohe,categorical),
                            (scaled,numerical),remainder='passthrough')

pipe = make_pipeline(ct,svc_model)
pipe.fit(X_train,y_train)
y_pred = pipe.predict(X_test)
y_pred_train = pipe.predict(X_train)

print("-------Test Scores-------")
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test,y_pred))

print("-------Train Scores-------")
print(confusion_matrix(y_train, y_pred_train))
print(classification_report(y_train, y_pred_train))
"""

## **RandomForest Model**
"""
ohe =  OneHotEncoder(sparse=False,handle_unknown="ignore")
model = RandomForestClassifier(random_state=42, class_weight="balanced")
scaled = StandardScaler()
X= df.drop('HeartDisease', axis=1)
y= df['HeartDisease']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

ct =make_column_transformer((ohe,categorical),
                            (scaled,numerical),remainder='passthrough')

pipe = make_pipeline(ct,model)
pipe.fit(X_train,y_train)
y_pred = pipe.predict(X_test)
y_pred_train = pipe.predict(X_train)

print("-------Test Scores-------")
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test,y_pred))

print("-------Train Scores-------")
print(confusion_matrix(y_train, y_pred_train))
print(classification_report(y_train, y_pred_train))
def objective(trial):
    X= df.drop('HeartDisease', axis=1)
    y= df['HeartDisease']
    
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)
    
    ct =make_column_transformer((ohe,categorical),
                                (scaled,numerical),remainder='passthrough')

    
    param = {"n_estimators":trial.suggest_int("n_estimators",100,500),
        "max_depth": trial.suggest_float("max_depth",1,10),
        "max_samples": trial.suggest_float("max_samples",0.01,1)}

    rf_model = RandomForestClassifier(random_state=42,**param,n_jobs=-1,verbose=False)
    pipe = make_pipeline(ct,rf_model)
    pipe.fit(X_train,y_train)

    preds = pipe.predict(X_test)
    pred_labels = np.rint(preds)
    accuracy = accuracy_score(y_test, pred_labels)
    return accuracy


if __name__ == "__main__":
    study = optuna.create_study(direction="maximize")
    study.optimize(objective, n_trials=50, timeout=600)

    trial = study.best_trial

    print("  Params: ")
    for key, value in trial.params.items():
        print("    {}: {}".format(key, value))
rf_model = RandomForestClassifier(n_estimators = 105,max_depth = 8.415478369250112,
                                  max_samples = 0.8643345901350294,random_state=42)

ct = make_column_transformer((ohe,categorical),
                            (scaled,numerical),remainder='passthrough')

pipe = make_pipeline(ct,rf_model)
pipe.fit(X_train,y_train)
y_pred = pipe.predict(X_test)
y_pred_train = pipe.predict(X_train)

print("-------Test Scores-------")
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test,y_pred))

print("-------Train Scores-------")
print(confusion_matrix(y_train, y_pred_train))
print(classification_report(y_train, y_pred_train))
"""
## **CatBoostClassifier Model**
"""
def objective(trial):
    X= df.drop('HeartDisease', axis=1)
    y= df['HeartDisease']
    categorical_features_indices = np.where(X.dtypes != np.float)[0]
    
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

    param = {
        "objective": trial.suggest_categorical("objective", ["Logloss", "CrossEntropy"]),
        "colsample_bylevel": trial.suggest_float("colsample_bylevel", 0.01, 0.1),
        "depth": trial.suggest_int("depth", 1, 12)}
    
    cat_model = CatBoostClassifier(**param)
    cat_model.fit(X_train, y_train, eval_set=[(X_test, y_test)], cat_features=categorical_features_indices,verbose=0, early_stopping_rounds=100)
    preds = cat_model.predict(X_test)
    pred_labels = np.rint(preds)
    accuracy = accuracy_score(y_test, pred_labels)
    return accuracy

if __name__ == "__main__":
    study = optuna.create_study(direction="maximize")
    study.optimize(objective, n_trials=75)

    print("Best trial:")
    trial = study.best_trial

    print("  Value: {}".format(trial.value))

    print("  Params: ")
    for key, value in trial.params.items():
        print("    {}: {}".format(key, value))
X= df.drop('HeartDisease', axis=1)
y= df['HeartDisease']
categorical_features_indices = np.where(X.dtypes != np.float)[0]

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

model = CatBoostClassifier(verbose=False,random_state=42,
                          objective= 'Logloss',
                          colsample_bylevel= 0.08369028629134112,
                          depth= 9)

model.fit(X_train, y_train,cat_features=categorical_features_indices,eval_set=(X_test, y_test))
y_pred = model.predict(X_test)

print("-------Test Scores-------")
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test,y_pred))

print("-------Train Scores-------")
print(confusion_matrix(y_train, y_pred_train))
print(classification_report(y_train, y_pred_train))