# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Building Predictive Models - Part 1

import pandas as pd
import os
import numpy as np

# ## Import Data

# set the path of the processed data
processed_data_path = os.path.join(os.path.pardir, 'data','processed')
train_file_path = os.path.join(processed_data_path, 'train.csv')
test_file_path = os.path.join(processed_data_path, 'test.csv')

train_df = pd.read_csv(train_file_path, index_col='PassengerId')
test_df = pd.read_csv(test_file_path, index_col='PassengerId')

train_df.info()
# in train data frame, we have 891 rows and 33 features; out of these 33 features, 'Survived' is the output label,
# while, the rest of 32 features will be used to build the model;

test_df.info()
# in test data frame, we have 418 rows and 32 features; we need to predict the survival for these passengers;

# ## Data Preparation

# create input variable X and output variable y; 
# for X we extract all columns from 'Age' onwards, so excluding the 'Survived' column; Also, convert a dataframe to a matrix
# (use .to_numpy() function instead of .as_matrix()), and each element of a matrix to a data type -> float;
# for an output variable we create y array and for that we use 'Survived' column; using NumPy array function .ravel() we create
# a flattened one-dimensional array;
X = train_df.loc[:, 'Age':].to_numpy().astype('float')
y = train_df['Survived'].ravel()

# +
# use .shape method to see the shape of the variables X (891 rows and 32 columns) and y (891 rows);
# it a common rule to use uppercase label for matrix array or multi-dimensional array and lowercase for one-dimensional
# array called a vector;

print (X.shape, y.shape)
# -

# train test split -> split an array X into two parts: 
# 1. X train data -> for training the model
# 2. X test data -> for evaluating our trained and predicted model performance
# firts, import a scikit-learn package for the function 'train_test_split', inside the function define arrays X, y and test size
# to be 20% (of actual training data) which will be used for model evalueation, while the rest of 80% of training data will be 
# used for model training; random_state parameter set to be zero means every time the line is executed, we get the same output;
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split (X, y, test_size=0.2, random_state=0)
print (X_train.shape, y_train.shape)
print (X_test.shape, y_test.shape)

# average survival in train and test
# we have around 39% of positive outcomes, which is similar in both, train and test dataset; ideally, you want positive cases
# to be evenly distributed in the train and test data.
# secondly, only 39% of data has positive cases, while the rest of 61% are negative classes. So, we have some kind of imbalance
# between the positive and negative class. In same cases, this can be a problem for evalueating the model properly.
print ('mean survival in train : {0:.3f}'.format(np.mean(y_train)))
print ('mean survival in test : {0:.3f}'.format(np.mean(y_test)))

# #### Check Scikit-Learn Version

import sklearn

sklearn.__version__

# +
# we need to use DummyClassifier or .dummy function inside Scikit learn library, but it is available on versions 0.19 onwards;
# # !conda update -y scikit-learn
# -

# ## Baseline Model

# import function DummyClassifier in order to build baseline classification model
from sklearn.dummy import DummyClassifier

# create model object as the most frequent, in our case it's 0 or not survived;
model_dummy = DummyClassifier (strategy='most_frequent', random_state=0)

# train model by using .fit function on the model object;
# X_train is the INPUT data;
# y_train is the OUTPUT data;
# we use X_
model_dummy.fit(X_train, y_train)

# use .score method to evaluate the model performance on the test data;
# we pass the test data to evaluate the performance; model will first predic the output on X_test,
# then, it will compare the predicted output with the actual output y_test;
# also, for classification model, the default score represents the model accuracy;
# this is our baseline accuracy of 61%;
# also, we can calculate exsplicitely the model accuracy with the function of accuracy score, confusion matrix...
print ('score for baseline model:{0:.2f}'.format(model_dummy.score(X_test, y_test)))

# performance metrics
from sklearn.metrics import accuracy_score, confusion_matrix, precision_score, recall_score
# y_test is the actual output;
# model_dummy.predict(X_test) is the predicted output;
# these are the same for all metrics below;

# accuracy score
print ('accuracy for baseline model:{0:.2f}'.format(accuracy_score(y_test, model_dummy.predict(X_test))))

# confusion matrix
print ('confusion matrix for baseline model: \n {0}'.format(confusion_matrix(y_test, model_dummy.predict(X_test))))

# precision and recall scores
print ('precision for baseline model: {0:.2f}'.format(precision_score(y_test, model_dummy.predict(X_test))))
print ('recall for baseline model: {0:.2f}'.format(recall_score(y_test, model_dummy.predict(X_test))))

# ## First Kaggle Submission

# preparing the data, converting to the matrix with .to_numpy()
test_X = test_df.to_numpy().astype('float')

# get predictions by using the .predict method and save it as 'predictions' variable
predictions = model_dummy.predict(test_X)

# we need to attach the predictions to the 'PassengerId'
# we create pandas dataframe and extract the 'PassengerId' from the .index value of the test data 
# and create the column of 'Survived' where we use predictions;
df_submission = pd.DataFrame({'PassengerId': test_df.index, 'Survived': predictions})

df_submission.head()

# save the file in the file path and according to the file name
submission_data_path = os.path.join(os.path.pardir, 'data', 'external')
submission_file_path = os.path.join(submission_data_path, '01_dummy.csv')

# create .csv file and set the index to False (no added indexed column)
df_submission.to_csv(submission_file_path, index=False)


# put all of the steps of creating the submission file into one single function 'get_submission_file'
def get_submission_file (model, filename):
    # converting to the matrix
    test_X = test_df.to_numpy().astype('float')
    # make predictions
    predictions = model.predict(test_X)
    # submission dataframe
    df_submission = pd.DataFrame({'PassengerId': test_df.index, 'Survived': predictions})
    # submission file
    submission_data_path = os.path.join(os.path.pardir, 'data', 'external')
    submission_file_path = os.path.join(submission_data_path, filename)
    # write to the file
    df_submission.to_csv(submission_file_path, index=False)    


# get submission file
get_submission_file(model_dummy, '01_dummy.csv')

# ## Logistic Regression Model

# import library sci-kit learn and the function for LogisticRegression
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier

# create model
clf = RandomForestClassifier()
clf = LogisticRegression(solver='liblinear')
# create model object step here:
model_lr_1 = LogisticRegression(random_state=0)

# train model with .fit function on the model object:
model_lr_1.fit(X_train, y_train)

# evaluate model -> as the model get trained, we evaluate the model performance by using the .score function and 
# by passing the test data to it;
# our predicted model is better than the baseline model;
print ('score for logistic regression - version 1 : {0:.2f}'.format(model_lr_1.score(X_test, y_test)))

# performance metrics 
# accuracy
print ('accuracy for logistic regression - version 1: {0:.2f}'.format(accuracy_score(y_test, model_lr_1.predict(X_test))))
# confusion matrix
print ('confusion matrix for logistic regression - version 1: \n {0}'.format (confusion_matrix(y_test, model_lr_1.predict(X_test))))
# precision
print ('precision for logistic regression - version 1: {0:.2f}'.format(precision_score(y_test, model_lr_1.predict(X_test))))
# recall
print('recall for logistic regression - version 1: {0:.2f}'.format(recall_score(y_test, model_lr_1.predict(X_test))))

# extract the model coefficients (or model weights or parameters)
model_lr_1.coef_

# ## Second Kaggle Submission

# get submission file
get_submission_file(model_lr_1, '02_lr.csv')

# # Building Predictive Models - Part 2

# ### Hyperparameter Optimization

# create the basic logistic regression model
# added part of solver specification to be liblinear (fixed the error)
model_lr = LogisticRegression(random_state=0, solver='liblinear')

# import GridSearchCV function which we will use for hyperparameters optimization
from sklearn.model_selection import GridSearchCV

# create parameters dictionary, where for each hyperparameters in the logistic 
# regression fuction, we have listed out the values that we want to try out during
# the optimization process;
parameters = {'C':[1.0, 10.0, 50.0, 100.0, 1000.0], 'penalty':['l1', 'l2']}
# create grid search object 'clf' using the .GridSearchCV function, specifying the
# base model and setting the grid parameters using the param_grid attribute, with cv value
# set to 3, which means it will perform the 3-fold cross-validation;
clf = GridSearchCV (model_lr, param_grid=parameters, cv=3)

# pass the train data to train different models with 
# different hyperparameter combinations;
clf.fit(X_train, y_train)

# use the best_param property to get the best settings;
clf.best_params_

print ('best score : {0:.2f}'.format(clf.best_score_))

# evaluate model
print ('score for logistic regression - version 2 : {0:.2f}'.format(clf.score(X_test, y_test)))

# ### Making Third Submission

# get submission file
get_submission_file(clf, '03_lr.csv')

# ## Feature Normalization and Standardization

# import scikit learn functions used for normalization -> MinMaxScaler and
# standardization -> StandardScaler
from sklearn.preprocessing import MinMaxScaler, StandardScaler

# ### Feature Normalization

# feature normalization
# firstly, create the scaler object itself:
scaler = MinMaxScaler ()
# secondly, pass the data to the .fit_transform function -> combination of two steps:
# 1. use the X_train to fit the scaler
# 2. transform the X_train to create the scaled output, which we save here in the variable X_train_scaled
X_train_scaled = scaler.fit_transform(X_train)
# Now, we have our scaled output. It should have been scaled to 0 to 1 level, which is the default for MinMaxScaler function.

# extract one column and printing its minimum and maximum value to check the impact of scaler;
# so, we have zero and 1 as expected;
X_train_scaled[:,0].min(), X_train_scaled[:,0].max()

# normalize test data
# Now, also we scale our test data, because if you will train your model on normalized data,
# then the model will expect that you pass the test data also in the normalized form.
X_test_scaled = scaler.transform(X_test)

# ### Feature Standardization

# feature standardization
# for standardization, we can use StandardScaler() method, steps are the same as for normalization;
# this time, we are using StandardScaler() function which will standardize all of the features with
# 0 mean and unit variance
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# ### Create model after standardization

# base model using the gridsearch technique;
model_lr = LogisticRegression(solver='liblinear')
parameters = {'C':[1.0, 10.0, 50.0, 100.0, 1000.0], 'penalty':['l1', 'l2']}
clf = GridSearchCV (model_lr, param_grid=parameters, cv=3)
clf.fit(X_train_scaled, y_train)

clf.best_score_

# evaluate model
print('score for logistic regression - version 2 : {0:.2f}'.format(clf.score(X_test_scaled, y_test)))

# #### Conclusion is that for logistic regression, feature standardization may not add too much value, due to some technical reasons for such behaviour.

# ## Model Persistence

# import pickle library
import pickle

# create the file paths
# put all the models in the model folder in accordance with the cookie-cutter data science template;
# also, use .pkl extention for the pickle files;
model_file_path = os.path.join(os.path.pardir, 'models', 'lr_model.pk1')
scaler_file_path = os.path.join(os.path.pardir, 'models', 'lr_scaler.pk1')

# open the files to write
# open both files in the write mode, or 'wb', so we can write in the binary mode;
model_file_pickle = open(model_file_path, 'wb')
scaler_file_pickle = open(scaler_file_path, 'wb')

# persist the model and scaler
# use .dump function to accomplish this task of writing the model and the scaler objects;
pickle.dump (clf, model_file_pickle)
pickle.dump (scaler, scaler_file_pickle)

# close the file
model_file_pickle.close()
scaler_file_pickle.close()

# #### load the persisted file

# open files in read mode
model_file_pickle = open(model_file_path, 'rb')
scaler_file_pickle = open(scaler_file_path, 'rb')
# load the pickle files with .load functions
clf_loaded = pickle.load(model_file_pickle)
scaler_loaded = pickle.load(scaler_file_pickle)
# close files
model_file_pickle.close()
scaler_file_pickle.close()

clf_loaded

scaler_loaded

# transform the test data using loaded scaler object
X_test_scaled = scaler_loaded.transform(X_test)
# calculate the score using loaded model object
print ('score for persisted logistic regression : {0:.2f}'.format(clf_loaded.score(X_test_scaled, y_test)))

StarcoderdataJupyterScriptsDedupFiltered