import numpy as np
import os
import pandas as pd 
import statsmodels.api as sm
import statsmodels.formula.api as smf
import seaborn as sns
from sklearn.preprocessing import scale 
from sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score
from sklearn.metrics import confusion_matrix, accuracy_score, classification_report
from sklearn.metrics import roc_auc_score,roc_curve
import statsmodels.formula.api as smf
import matplotlib.pyplot as plt
from sklearn.neighbors import KNeighborsClassifier
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn import tree
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier
from catboost import CatBoostClassifier

from warnings import filterwarnings
filterwarnings('ignore')
"""
# Preparing Data
"""
test = pd.read_csv("test.csv") #importing test data set 
df_test = test.copy()
train = pd.read_csv("train.csv") #importing test data set 
df_train = train.copy()

df_test.head()
df_train.head()
df_train.columns
df_train.isnull().values.any()
df_test.info()
df_train.info()
df_train.describe().T
 ## handling the null values
df_train.isnull().sum() # I'll delete the variable cabin because of it's over 78 percent, 
df_train["Cabin"]
df_train = df_train.drop(["Cabin","Name","Ticket"], axis=1) # I'll delete the variable cabin because of it's over 78 percent, 
df_test = df_test.drop(["Cabin","Name", "Ticket",], axis =1) # same processes on the our test dataset
# we can understand that the name, and ticket, variables can't effect our independent variable which is survived
df_test.isnull().sum()
df_train.isnull().sum()
"""
 Handling the missing values
- Fill the missing ‘Embarked’ values by the most frequent value
- For now the only age variable has missing value we are going to handle them by knnimput method(nearest neighbor imputation.)
"""
# Filling missing Embarked values with most common value
df_train['Embarked'] = df_train['Embarked'].fillna(df_train['Embarked'].mode()[0])
df_train.isnull().sum()
# Filling missing Fare value with mean value
df_test['Fare'] = df_test['Fare'].fillna(df_test['Fare'].mean())
df_test.isnull().sum()
"""
- 'Pclass' is a categorical feature so we convert its values to strings
"""
#Pclass' is a categorical feature so we convert its values to strings
df_train['Pclass'] = train['Pclass'].apply(str)
df_test['Pclass'] = test['Pclass'].apply(str)
from ycimpute.imputer import knnimput
n_dftrain = df_train.select_dtypes(include = ['float64', 'int64']) #because the imputer method don't except the string value 
n_dftest = df_train.select_dtypes(include = ['float64', 'int64'])
var_namestrain = list(n_dftrain)
var_namestest = list(n_dftest)
var_namestrain
var_namestest
#we have changed the our dataframe to numpy array because our knninmput want it 
n_dftrain = np.array(n_dftrain) 
n_dftest = np.array(n_dftest) 
n_dftrain = knnimput.KNN(k = 4).complete(n_dftrain)
n_dftest = knnimput.KNN(k = 4).complete(n_dftest)
type(n_dftrain) # we have change this to pandas DataFrame
n_dftrain = pd.DataFrame(n_dftrain, columns = var_namestrain)
n_dftest = pd.DataFrame(n_dftest, columns = var_namestest)
n_dftrain.head()
n_dftest.head()
#We replaced our age variable with the one with null values filled in by ML model
df_train["Age"] = n_dftrain["Age"] 
df_test["Age"] = n_dftest["Age"]
df_train.isnull().sum()
df_test.isnull().sum() # we handle the missing value issue by using KNN imputation
# Let’s perform a basic one hot encoding of categorical features
for col in df_train.dtypes[df_train.dtypes == 'object'].index:
   for_dummy = df_train.pop(col)
   df_train = pd.concat([df_train, pd.get_dummies(for_dummy, prefix=col)], axis=1)
for col in df_test.dtypes[df_test.dtypes == 'object'].index:
   for_dummy = df_test.pop(col)
   df_test = pd.concat([df_test, pd.get_dummies(for_dummy, prefix=col)], axis=1)
df_train.head()
df_test.head()
# we convert the some  float values of Age to integer
df_train['Age'] = df_train['Age'].apply(int)
df_test['Age'] = df_test['Age'].apply(int)
df_train.head()
df_test.head()
# we have finished the our data preparation now we can start creating our model 
df_train["Survived"].value_counts().plot.barh(); # we can see that distribution of variable Survived by bar plot
df_train.describe().T
"""
# Logistic Regression
"""
"""
### Model
"""
y = df_train["Survived"] # this is our dependent variable
X = df_train.drop(["Survived"], axis=1) # these are our independent variables
X_train, X_test, y_train, y_test = train_test_split(X, y, 
                                                    test_size = 0.30, 
                                                    random_state = 42)
# we splitted our dataset to test and train
from sklearn.linear_model import LogisticRegression
# we created our model then have fitted it to our train data
loj = LogisticRegression(solver = "liblinear") 
loj_model = loj.fit(X_train,y_train)
loj_model
loj_model.intercept_ # interception to y axis
loj_model.coef_ #coefficent of our independent variables
"""
## Model tuning

It does not have an external model hyperparameter. Therefore, there is no situation that we can tune, there is a situation we can verify.
"""
y_pred = loj_model.predict(X_train) # now our model is predicting the y
accuracy_score(y_train ,y_pred) # this our train accuracy score it is bad predictor of our test score
accuracy_score(y_test ,loj_model.predict(X_test)) #  accuracy score
cross_val_score(loj_model, X_test, y_test, cv=10)
cross_val_score(loj_model, X_test, y_test, cv=10).mean() # verified accuracy score
"""
### I did these operations to upload my model's predicts to kaggle
"""
y_pred2 = loj_model.predict(df_test) # we predicted our test data set
y_pred2[:5]
df = pd.DataFrame(y_pred2, columns = ['Survived']) # we change te numpy array's type to pandas dataframe
test1 = pd.concat([df_test["PassengerId"], df], axis=1) # concatenating the passengerıd and
test1.head() 
test1.to_csv('test1.csv',index=False) # to csv file in your current workig directory
#now yuo can upload your predicts yo kaggle competition
"""
# Naive Bayes 

It does not have an external model hyperparameter. Therefore, there is no situation that we can tune, there is a situation we can verify.
"""
y = df_train["Survived"] # this is our dependent variable
X = df_train.drop(["Survived"], axis=1) # these are our independent variables
X_train, X_test, y_train, y_test = train_test_split(X, y, 
                                                    test_size = 0.30, 
                                                    random_state = 42)
# we splitted our dataset to test and train
from sklearn.naive_bayes import GaussianNB
# we created our model then have fitted it to our train data
nb = GaussianNB()
nb_model = nb.fit(X_train, y_train) 
nb_model
nb_model.predict(X_test)[0:10] # we can see predicted values like this
nb_model.predict_proba(X_test)[0:10] # these are probablity of predicted values
y_pred = nb_model.predict(X_test) #  now our model is predicting the y values
accuracy_score(y_test ,y_pred) # test doğruluk oranı
cross_val_score(nb_model, X_test, y_test, cv=10).mean() # verified accuracy score
"""
# K Nearest Neigbors Algoritm
"""
"""
### Model & Predict
"""
y = df_train["Survived"] # this is our dependent variable
X = df_train.drop(["Survived"], axis=1) # these are our independent variables
X_train, X_test, y_train, y_test = train_test_split(X, y, 
                                                    test_size = 0.30, 
                                                    random_state = 42)
# we splitted our dataset to test and train
knn = KNeighborsClassifier()
knn_model = knn.fit(X_train, y_train)
knn_model
y_pred = knn_model.predict(X_test) #  now our model is predicting the y values
accuracy_score(y_test, y_pred) # primitive score
"""
### Model Tuning
"""
knn_params = {"n_neighbors": np.arange(1,50)}
knn = KNeighborsClassifier()
knn_cv = GridSearchCV(knn, knn_params, cv=10) # we implement algoritm 
knn_cv.fit(X_train, y_train)
print("Best Score:" + str(knn_cv.best_score_)) 
print("Best Parameters: " + str(knn_cv.best_params_))
knn = KNeighborsClassifier(25) #  We created our model with best parameters 
knn_tuned = knn.fit(X_train, y_train)    
knn_tuned.score(X_test, y_test)
"""
# SVC ( Support Vector Classification)
"""
y = df_train["Survived"] # this is our dependent variable
X = df_train.drop(["Survived"], axis=1) # these are our independent variables
X_train, X_test, y_train, y_test = train_test_split(X, y, 
                                                    test_size = 0.30, 
                                                    random_state = 42)
# we splitted our dataset to test and train
svm_model = SVC(kernel = 'linear').fit(X_train, y_train)
y_pred = svm_model.predict(X_test)
accuracy_score(y_test , y_pred)
"""
## Model Tuning
"""
svc_params = {"C": np.arange(1,10)} # 0 value gives us an exception 

svc = SVC(kernel = "linear")

svc_cv_model = GridSearchCV(svc,svc_params, 
                            cv = 10, 
                            n_jobs = -1, 
                            verbose = 2 )
svc_cv_model.fit(X_train, y_train)
print("Best Params: " + str(svc_cv_model.best_params_))
svc_tuned = SVC(kernel = "linear", C = 3).fit(X_train, y_train) # we created new model with best params that we have found
y_pred = svc_tuned.predict(X_test)
accuracy_score(y_test, y_pred)
"""
# RBF(Radial Basis Function) SVC (Non Linear)
"""
y = df_train["Survived"] # this is our dependent variable
X = df_train.drop(["Survived"], axis=1) # these are our independent variables
X_train, X_test, y_train, y_test = train_test_split(X, y, 
                                                    test_size = 0.30, 
                                                    random_state = 42)
# we splitted our dataset to test and train
svc_model = SVC(kernel = "rbf").fit(X_train, y_train) # we using kernel = rbf (radial basis function)
y_pred = svc_model.predict(X_test)
accuracy_score(y_test, y_pred) # primitive score
svc_params = {"C": [0.0001, 0.001, 0.1, 1, 5, 10 ,50 ,100],
             "gamma": [0.0001, 0.001, 0.1, 1, 5, 10 ,50 ,100]}
svc = SVC()
svc_cv_model = GridSearchCV(svc, svc_params, 
                         cv = 10, 
                         n_jobs = -1,
                         verbose = 2)

svc_cv_model.fit(X_train, y_train)
print("Best Params: " + str(svc_cv_model.best_params_))
svc_tuned = SVC(C = 100, gamma = 0.0001).fit(X_train, y_train)
y_pred = svc_tuned.predict(X_test)
accuracy_score(y_test, y_pred)
"""
# Artificial neural networks
"""
y = df_train["Survived"] # this is our dependent variable
X = df_train.drop(["Survived"], axis=1) # these are our independent variables
"""
- Artificial neural networks are affected by the scales of variables and the situations between them.So we have to standarized them 
"""
# import our scaler object
from sklearn.preprocessing import StandardScaler 
scaler = StandardScaler() # OOP sampling
# we transform our data
scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)
X_train_scaled[0:2]
X_test_scaled[0:2]
from sklearn.neural_network import MLPClassifier
mlpc = MLPClassifier().fit(X_train_scaled, y_train)
mlpc.coefs_
y_pred = mlpc.predict(X_test_scaled)
accuracy_score(y_test, y_pred) # primitive score
"""
## Model Tuning
"""
mlpc_params = {"alpha": [0.1, 0.01, 0.02, 0.005, 0.0001,0.00001],
              "hidden_layer_sizes": [(10,10,10),
                                     (100,100,100),
                                     (100,100),
                                     (3,5), 
                                     (5, 3)],
              "solver" : ["lbfgs","adam","sgd"],
              "activation": ["relu","logistic"]}

from sklearn.model_selection import RandomizedSearchCV
# we use RandomizedSearchCV because its faster than GridSearchcv
mlpc = MLPClassifier()
mlpc_cv_model = RandomizedSearchCV(mlpc, mlpc_params, 
                         cv = 10, 
                         n_jobs = -1,
                         verbose = 2)

mlpc_cv_model.fit(X_train_scaled, y_train) 
print("Best Params: " + str(mlpc_cv_model.best_params_))
mlpc_tuned = MLPClassifier(activation = "logistic", 
                           alpha = 0.1, 
                           hidden_layer_sizes = (100, 100),
                          solver = "adam")
mlpc_tuned.fit(X_train_scaled, y_train)
y_pred = mlpc_tuned.predict(X_test_scaled)
accuracy_score(y_test, y_pred) # our validated score
"""
# Classification and Regression Trees (CART)
"""
"""
## Model & Predict
"""
y = df_train["Survived"] # this is our dependent variable
X = df_train.drop(["Survived"], axis=1) # these are our independent variables
X_train, X_test, y_train, y_test = train_test_split(X, y, 
                                                    test_size = 0.30, 
                                                    random_state = 42)
# we splitted our dataset to test and train
from sklearn.tree import DecisionTreeClassifier
cart = DecisionTreeClassifier()
cart_model = cart.fit(X_train, y_train)
from skompiler import skompile
print(skompile(cart_model.predict).to("python/code")) # this is our decision tree it has turned into the ython code
y_pred = cart_model.predict(X_test)
accuracy_score(y_test, y_pred) # this is primitive score
"""
## Model Tuning
"""
cart_grid = {"max_depth": range(1,10),
            "min_samples_split" : list(range(2,50)) }
cart = tree.DecisionTreeClassifier()
cart_cv = GridSearchCV(cart, cart_grid, cv = 10, n_jobs = -1, verbose = 2)
cart_cv_model = cart_cv.fit(X_train, y_train)
print("Best Params: " + str(cart_cv_model.best_params_))
cart = tree.DecisionTreeClassifier(max_depth = 5, min_samples_split = 19) # we have created our model again with best params
cart_tuned = cart.fit(X_train, y_train)
y_pred = cart_tuned.predict(X_test)
accuracy_score(y_test, y_pred) # this is our accuracy score
"""
# Random Forest Classifier
"""
y = df_train["Survived"] # this is our dependent variable
X = df_train.drop(["Survived"], axis=1) # these are our independent variables
X_train, X_test, y_train, y_test = train_test_split(X, y, 
                                                    test_size = 0.30, 
                                                    random_state = 42)
# we splitted our dataset to test and train
from sklearn.ensemble import RandomForestClassifier
rf_model = RandomForestClassifier().fit(X_train, y_train)
y_pred = rf_model.predict(X_test)
accuracy_score(y_test, y_pred) # primitive score
"""
## Model Tuning
"""
rf_params = {"max_depth": [2,5,8,10],
            "max_features": [2,5,8],
            "n_estimators": [10,500,1000],
            "min_samples_split": [2,5,10]}
rf_model = RandomForestClassifier()

rf_cv_model = RandomizedSearchCV(rf_model, 
                           rf_params, 
                           cv = 10, 
                           n_jobs = -1, 
                           verbose = 2) 
rf_cv_model.fit(X_train, y_train)
print("Best Params: " + str(rf_cv_model.best_params_))
#final model with best params that we have found
rf_tuned = RandomForestClassifier(max_depth = 10,
                                  max_features = 5, 
                                  min_samples_split = 10,
                                  n_estimators = 500)

rf_tuned.fit(X_train, y_train)
y_pred = rf_tuned.predict(X_test)
accuracy_score(y_test, y_pred)
"""
- You can see the variable significance levels with this code fragments
"""
Importance = pd.DataFrame({"Importance": rf_tuned.feature_importances_*100},
                         index = X_train.columns)
Importance.sort_values(by = "Importance", 
                       axis = 0, 
                       ascending = True).plot(kind ="barh", color = "r")

plt.xlabel("Variable Significance Levels")
"""
# Gradient Boosting Machines
"""
y = df_train["Survived"] # this is our dependent variable
X = df_train.drop(["Survived"], axis=1) # these are our independent variables
X_train, X_test, y_train, y_test = train_test_split(X, y, 
                                                    test_size = 0.30, 
                                                    random_state = 42)
# we splitted our dataset to test and train
from sklearn.ensemble import GradientBoostingClassifier
gbm_model = GradientBoostingClassifier().fit(X_train, y_train)
y_pred = gbm_model.predict(X_test)
accuracy_score(y_test, y_pred) # primitive accuracy score
"""
## Model Tuning
"""
gbm_params = {"learning_rate" : [0.001, 0.01, 0.1, 0.05],
             "n_estimators": [100,500,100],
             "max_depth": [3,5,10],
             "min_samples_split": [2,5,10]}
gbm = GradientBoostingClassifier()

gbm_cv_model = RandomizedSearchCV(gbm, gbm_params, cv = 10, n_jobs = -1, verbose = 2)
gbm_cv_model.fit(X_train, y_train)
print("Best Params: " + str(gbm_cv_model.best_params_))
gbm = GradientBoostingClassifier(learning_rate = 0.05, 
                                 max_depth = 3,
                                min_samples_split = 2,
                                n_estimators = 100)
gbm_tuned =  gbm.fit(X_train,y_train)
y_pred = gbm_tuned.predict(X_test)
accuracy_score(y_test, y_pred) # our tuned accuracy score
"""
# Extreme Gradient Boosting (XGBoost)
"""
y = df_train["Survived"] # this is our dependent variable
X = df_train.drop(["Survived"], axis=1) # these are our independent variables
X_train, X_test, y_train, y_test = train_test_split(X, y, 
                                                    test_size = 0.30, 
                                                    random_state = 42)
# we splitted our dataset to test and train
from xgboost import XGBClassifier
xgb_model = XGBClassifier().fit(X_train, y_train)
y_pred = xgb_model.predict(X_test)
accuracy_score(y_test, y_pred)
"""
## Model Tuning
"""
xgb_params = {
        'n_estimators': [100, 500, 1000, 2000],
        'subsample': [0.6, 0.8, 1.0],
        'max_depth': [3, 4, 5,6],
        'learning_rate': [0.1,0.01,0.02,0.05],
        "min_samples_split": [2,5,10]}
from sklearn.model_selection import RandomizedSearchCV
xgb = XGBClassifier()

xgb_cv_model = RandomizedSearchCV(xgb, xgb_params, cv = 10, n_jobs = -1, verbose = 2)
xgb_cv_model.fit(X_train, y_train)
xgb_cv_model.best_params_
xgb = XGBClassifier(learning_rate = 0.05, 
                    max_depth = 5,
                    min_samples_split = 2,
                    n_estimators = 100,
                    subsample = 0.8)
xgb_tuned =  xgb.fit(X_train,y_train)
y_pred = xgb_tuned.predict(X_test)
accuracy_score(y_test, y_pred)
"""
- I did these operations to upload my model's predicts to kaggle
"""
y_pred2 = xgb_tuned.predict(df_test) # we predicted our test data set
df = pd.DataFrame(y_pred2, columns = ['Survived']) # we change te numpy array's type to pandas dataframe
test3 = pd.concat([df_test["PassengerId"], df], axis=1) # concatenating the passengerıd and
test3 = pd.concat([df_test["PassengerId"], df], axis=1) # concatenating the passengerıd and
test3.head() 
test1.to_csv('test3.csv',index=False) # to csv file in your current workig directory
#now yuo can upload your predicts yo kaggle competition
"""
# Light GBM
"""
y = df_train["Survived"] # this is our dependent variable
X = df_train.drop(["Survived"], axis=1) # these are our independent variables
X_train, X_test, y_train, y_test = train_test_split(X, y, 
                                                    test_size = 0.30, 
                                                    random_state = 42)
# we splitted our dataset to test and train
from lightgbm import LGBMClassifier
lgbm_model = LGBMClassifier().fit(X_train, y_train)
y_pred = lgbm_model.predict(X_test)
accuracy_score(y_test, y_pred) # primitive accuracy score
"""
## Model Tuning
"""
lgbm_params = {
        'n_estimators': [100, 500, 1000, 2000],
        'subsample': [0.6, 0.8, 1.0],
        'max_depth': [3, 4, 5,6],
        'learning_rate': [0.1,0.01,0.02,0.05],
        "min_child_samples": [5,10,20]}
lgbm = LGBMClassifier()

lgbm_cv_model = RandomizedSearchCV(lgbm, lgbm_params, 
                             cv = 10, 
                             n_jobs = -1, 
                             verbose = 2)

lgbm_cv_model.fit(X_train, y_train) 
lgbm_cv_model.best_params_
lgbm = LGBMClassifier(learning_rate = 0.01, 
                       max_depth = 6,
                       subsample = 0.6,
                       n_estimators = 100,
                       min_child_samples = 10)
lgbm_tuned = lgbm.fit(X_train,y_train)
y_pred = lgbm_tuned.predict(X_test)
accuracy_score(y_test, y_pred) #our accuracy score
"""
# Category Boosting (CatBoost)
"""
y = df_train["Survived"] # this is our dependent variable
X = df_train.drop(["Survived"], axis=1) # these are our independent variables
from catboost import CatBoostClassifier
cat_model = CatBoostClassifier().fit(X_train, y_train)
y_pred = cat_model.predict(X_test)
accuracy_score(y_test, y_pred) # primitive accuracy
"""
## Model Tuning
"""
catb_params = {
    'iterations': [200,500],
    'learning_rate': [0.01,0.05, 0.1],
    'depth': [3,5,8] }
catb = CatBoostClassifier()
catb_cv_model = GridSearchCV(catb, catb_params, cv=5, n_jobs = -1, verbose = 2)
catb_cv_model.fit(X_train, y_train)
catb_cv_model.best_params_
catb_cv_model.best_params_
catb = CatBoostClassifier(iterations = 200, 
                          learning_rate = 0.01, 
                          depth = 8)

catb_tuned = catb.fit(X_train, y_train)
y_pred = catb_tuned.predict(X_test)
y_pred = catb_tuned.predict(X_test)
accuracy_score(y_test, y_pred)
"""
# Comparison of All Models
"""
modeller = [
    knn_tuned,
    loj_model,
    svc_tuned,
    nb_model,
    mlpc_tuned,
    cart_tuned,
    rf_tuned,
    gbm_tuned,
    catb_tuned,
    lgbm_tuned,
    xgb_tuned
    
]


for model in modeller:
    isimler = model.__class__.__name__
    y_pred = model.predict(X_test)
    dogruluk = accuracy_score(y_test, y_pred)
    print("-"*28)
    print(isimler + ":" )
    print("Accuracy: {:.4%}".format(dogruluk))