# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

"""
Purpose: 
    Predict the birth weight with different variables of the parents.
    The steps are the following: 
    1) Importing and analysing the data
    2) Missing values detection, Flagging and Imputation
    3) Feature Engineering 
    4) Correlation Analysis
    5) Linear model with the whole dataset
    6) OLS/lasso/ARD Trials
    7) Comparing the score of different regression models
    
"""


# ## Importing and Analysing the data

# +
import pandas as pd #data science essentials
import matplotlib.pyplot as plt #essential graphical output
import seaborn as sns  #enhanced graphical output
import statsmodels.formula.api as smf #mathematical essentials
from sklearn.model_selection import train_test_split #training dataframe tools
import numpy as np #regression modeling
from scipy.stats import iqr # IQR for Outlier Detection
from sklearn.linear_model import LinearRegression # Linear Regression

#declaring variable of the file 
file = './birthweight_low.xlsx'
#storing the excel that is going to be the dataframe into a variable
birthweight = pd.read_excel (io = file, 
                            sheet_name = 0,
                            header = 0)

# setting pandas print options
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

#removing the columns that won't help us to predict the birthweight 
birthweight = birthweight.drop(columns = ['fmaps', 'omaps'], axis = 0)


#printing the information of each column
print(birthweight.info())
#printing the sum of the missing values


#printing the 15 first rows
birthweight.head(n=15)


# -

#Analysing the composision of each column
birthweight.describe(include = 'number').round(decimals = 2)

# ## Missing values detection, Flagging and Imputation

#printing the quantity of missing value per feature
print('*' * 50)
print(f"""Sum of missing values:
{birthweight.isnull().sum()}""")
print('*' * 50)

#for every col in birthweight df 
for col in birthweight: 
    #analizing if each column has missing value
    if birthweight[col].isnull().sum() > 0:
        #creating a m column
        birthweight['m_' + col] =  birthweight[col].isnull().astype(int)

# +
# Dropping NA values
bw_dropped = birthweight.dropna()
titles = ['Mother Education', 'Father Education', 'Prenatal Visits']
cols = ['meduc', 'feduc', 'npvis']
i = 0
for col in cols:
    
    # calculating the mean 
    mean = bw_dropped[col].mean()
    # calculating the median 
    median = bw_dropped[col].median()
    # calculating the histplot 
    sns.histplot(bw_dropped[col],
                 color='grey')

    # Mean Line
    plt.axvline(mean,
                color='r',
                linestyle='--')
    # Median Line
    plt.axvline(median,
                color='b',
                linestyle='-')
    #legend
    plt.legend({'Mean':mean,
                'Median':median})
    plt.title (titles[i])
    i += 1
    #showing the plot
    plt.show()



# +
# Filling all missing values with median
#mother education
fill_value = bw_dropped['meduc'].median()
birthweight['meduc'].fillna (value = fill_value,
                             inplace = True)

#prenatal visits
fill_value = bw_dropped['npvis'].median()
birthweight['npvis'].fillna(value = fill_value,
                             inplace = True)
#father education
fill_value = bw_dropped['feduc'].median()
birthweight['feduc'].fillna(value = fill_value,
                             inplace = True)
birthweight.isnull().sum()

# +
#declaring the columns that are not going to be plotted
noplot = ['m_meduc', 'm_npvis', 'm_feduc']
#making a loop of the df columns
for col in birthweight:
    #evaluating that the column is not in notplot list
    if col not in noplot:
        #declaring the size of the plot
        fig, ax = plt.subplots (figsize = (12, 6))
        #performing the histogram plot
        sns.histplot( data = birthweight,
                          x      = col,
                          kde    = True,
                         color = 'grey'
                          )
        #tittle of the plot
        plt.title(label   = "Distribution of Newborn per " + col)
        #X Lable of the plot
        plt.xlabel(xlabel =  col.capitalize()) 
        #Y Lable of the plot
        plt.ylabel(ylabel = "Count")
        plt.show() #showing the plot
        #showing the skewness of all the columns
        print(f"""skewness: {birthweight[col].skew().round(decimals = 2)}""") 
        print('*' * 120)




# -

# Checking different quantiles for outlier detection
bw_quantiles = birthweight.loc[:, :].quantile([0.05,
                                      0.20,
                                      0.40,
                                      0.60,
                                      0.80,
                                      0.95,
                                      1.00])
#displaying df quantiles 
bw_quantiles


# ## Feature Engineering

# +
#making a list of the variables that is not going to be displayed
data =['bwght', 'log_bwght']
log = ['npvis'] 
for col in birthweight:
    if  col not in (data):
        fig, ax = plt.subplots(figsize =(16, 8))
    # developing a boxplot
        sns_boxplot = sns.boxplot(x     = col,
                    y     = 'bwght',
                    data  = birthweight)
        
        plt.axhline(y = 3500, color = "purple", linestyle = '--')
        # titles and labels
        plt.title(label   = 'Relationship between ' + col.capitalize() + ' and Birth Weight')
        plt.xlabel(xlabel = col.capitalize())
        plt.ylabel(ylabel = 'Birth Weight')
        plt.setp(sns_boxplot.get_xticklabels(), rotation=90)
        plt.show()
        

for col in log:
    if birthweight[col].min() == 0 and birthweight[col].value_counts().count() == 2:
        break
    elif birthweight[col].min() == 0:
        birthweight['log_' + col] = np.log(birthweight[col] + 0.99)
    elif birthweight[col].min() > 0:
        birthweight['log_' + col] = np.log(birthweight[col])
    else:
        print('there is an error')
        
birthweight['log_' + 'bwght'] = np.log(birthweight['bwght'])   


# +
birthweight['cigsadddrink'] = birthweight['drink'] +  birthweight['cigs']
# making a iter row function to create distinct help columns
for row, col in birthweight.iterrows():
    #creating a range of mother age
    if birthweight.loc[row, 'mage'] < 31:
        birthweight.loc[row, 'h2_mage'] = 'first'

    elif birthweight.loc[row, 'mage'] < 41:
        birthweight.loc[row, 'h2_mage'] = 'second'
    elif birthweight.loc[row, 'mage'] < 52:
        birthweight.loc[row, 'h2_mage'] = 'third'
    elif birthweight.loc[row, 'mage'] > 51:
        birthweight.loc[row, 'h2_mage'] = 'fourth'
    else:
         print("error1")
    #creating a range of cigarrettes
    if birthweight.loc[row, 'cigs'] <= 1:
        birthweight.loc[row, 'h2_cig'] = 'first'
    elif birthweight.loc[row, 'cigs'] < 21:
        birthweight.loc[row, 'h2_cig'] = 'second'
    elif birthweight.loc[row, 'cigs'] >= 21:
        birthweight.loc[row, 'h2_cig'] = 'third'
    else:
        print("error2")
    #creating a range of drinks  
    if birthweight.loc[row, 'drink'] < 9:
        birthweight.loc[row, 'h2_drink'] = 'first'
    elif birthweight.loc[row, 'drink'] < 11:
        birthweight.loc[row, 'h2_drink'] = 'second'
    elif birthweight.loc[row, 'drink'] > 10:
        birthweight.loc[row, 'h2_drink'] = 'third'
    else:
        print("error4")
     #creating a range of father education    
    if birthweight.loc[row, 'feduc'] < 9:
        birthweight.loc[row, 'h2_feduc'] = 'first'
    elif birthweight.loc[row, 'feduc'] < 12:
        birthweight.loc[row, 'h2_feduc'] = 'second'
    elif birthweight.loc[row, 'feduc'] > 11:
        birthweight.loc[row, 'h2_feduc'] = 'third'
    else:
        print("error5")
    #creating a range of mother education     
    if birthweight.loc[row, 'meduc'] < 15:
        birthweight.loc[row, 'h2_meduc'] = 0
    elif birthweight.loc[row, 'meduc'] >= 15:
        birthweight.loc[row, 'h2_meduc'] = 1
    else:
        print("error5")
    #creating a range of cigarrettes plus drinks     
    if birthweight.loc[row, 'cigsadddrink'] < 18:
        birthweight.loc[row, 'h2_cigsadddrink'] = 'first'
    elif birthweight.loc[row, 'cigsadddrink'] > 17:
        birthweight.loc[row, 'h2_cigsadddrink'] = 'second'
    else:
        print("error6")    
#obtaining the dummy variables     
temp_birthweight = pd.get_dummies(columns = ['h2_cig', 'h2_mage','h2_drink', 'h2_feduc','h2_cigsadddrink'], data = birthweight ) 
birthweight = temp_birthweight

#different trials 

#birthweight['mageplowweight'] =  birthweight['mage']  * birthweight['low_weight']
#birthweight['cigsplowweight'] =  birthweight['cigs']  * birthweight['low_weight']
#birthweight['drinksplowweight'] =  birthweight['drink']  * birthweight['low_weight']
#birthweight['drinksaddcigplowweight'] = (birthweight['drink'] +  birthweight['cigs'] )* birthweight['low_weight']
#birthweight['log_magesqr'] =  birthweight['log_mage']  * birthweight['log_mage']  
#birthweight['magesqr'] = birthweight['mage'] *  birthweight['mage']
#birthweight['fagesqr'] = birthweight['fage'] *  birthweight['fage']
#birthweight['magesqraddfagesqr'] = birthweight['fagesqr'] + birthweight['magesqr'] 
#birthweight['monpresqr'] = birthweight['monpre'] *  birthweight['monpre']
#birthweight['monpre:npvis'] = birthweight['monpre'] *  birthweight['npvis']
#birthweight['drink:mage'] = birthweight['drink'] *  birthweight['mage']
#birthweight['mblck:mage'] = birthweight['mage'] *  birthweight['mblck']
#birthweight['cigs:mage'] = birthweight['mage'] *  birthweight['cigs']
#birthweight['drinksaddcigs']= birthweight['drink'] *  birthweight['cigs']
#birthweight['cigsadddrinkspermage'] = (birthweight['drink'] +  birthweight['cigs']) * birthweight['mage']
#birthweight['cigssqr'] = birthweight['cigs'] * birthweight['cigs']
#birthweight['drinksqr'] = birthweight['drink'] * birthweight['drink']
#birthweight['mageaddfage'] = birthweight['mage'] * birthweight['fage']
#birthweight['cig:mwhte'] = birthweight['mwhte'] * birthweight['cigs']
#birthweight['npvissqr'] = birthweight['npvis'] * birthweight['npvis']
#birthweight['meducadddfedc'] = birthweight['feduc'] + birthweight['meduc']
#conditions  = [ (birthweight['mblck'] > 0) & (birthweight['fblck'] > 0), birthweight['mblck'] < 1  ]
#choices     = [ 1, 0 ]
#birthweight["parentsblack"] = np.select(conditions, choices, default=0)

# -

# ## Correlation among the independent features versus birthweight

# +
# creating a (Pearson) correlation matrix
df_corr = birthweight.corr(method = 'pearson').round(2)


# printing (Pearson) correlations with SalePrice
print(df_corr.loc['bwght'].sort_values(ascending = False))

# +
# making a copy of birthweight
birthweight_explanatory = birthweight.copy()
# dropping bwght and log_bwght from the explanatory variable set
birthweight_explanatory = birthweight_explanatory.drop(['bwght',
                                                'log_bwght'], axis = 1)

# formatting each explanatory variable for statsmodels
for val in birthweight_explanatory:
    print(f"{val} +")
# -

# ## First Linear model with whole dataset

# #### I'am removing the columns h2_mage_first and h_meduc because are generating a high p_value

# +
# building a full model

# blueprinting a model type
lm_full = smf.ols(formula = """bwght ~                                            
                                        h2_cig_first +
                                        drink +
                                        h2_cig_third +
                                        h2_mage_fourth +
                                        h2_drink_first +
                                        h2_feduc_third +
                                        h2_cigsadddrink_first 
                                        """,
                               data = birthweight)


# telling Python to run the data through the blueprint
results_full = lm_full.fit()
# printing the results
results_full.summary()

# -

# ## OLS/Lasso/ARD Trials

# +
#defining the logarithmic target
log_birthweight_target = birthweight.loc[:,'log_bwght']   
#defining the normal target
normal_birthweight_target = birthweight.loc[:,'bwght']
#defining the whole dataset removing certaing columns 
birthweight_data = birthweight.drop(['bwght',
                                    'log_bwght'],
                               axis = 1) 

#making the split value of the whole dataset
x_train_FULL, x_test_FULL, y_train_FULL, y_test_FULL = train_test_split(
            birthweight_data,     # x-variables
            normal_birthweight_target,   # y-variable
            test_size = 0.25,
            random_state = 219)



# +
import sklearn.linear_model  # linear models
# INSTANTIATING a model object
lasso_model = sklearn.linear_model.Lasso(alpha = 1.0, normalize = True) # default magitud

# FITTING to the training data
lasso_fit = lasso_model.fit(x_train_FULL, y_train_FULL)


# PREDICTING on new data
lasso_pred = lasso_fit.predict(x_test_FULL)


# SCORING the results
print('Lasso Training Score :', lasso_model.score(x_train_FULL, y_train_FULL).round(4))
print('Lasso Testing Score  :', lasso_model.score(x_test_FULL, y_test_FULL).round(4))

# saving scoring data for future use
lasso_train_score = lasso_model.score(x_train_FULL, y_train_FULL).round(4) # using R-square
lasso_test_score  = lasso_model.score(x_test_FULL, y_test_FULL).round(4)   # using R-square
lasso_test_gap = abs(lasso_train_score - lasso_test_score).round(4)

# displaying and saving the gap between training and testing
print('Lasso Train-Test Gap :', abs(lasso_test_gap).round(4))


# +
# zipping each feature name to its coefficient
lasso_model_values = zip(birthweight_data.columns, lasso_fit.coef_.round(decimals = 2))

# setting up a placeholder list to store model features
lasso_model_lst = [('intercept', lasso_fit.intercept_.round(decimals = 2))]

# printing out each feature-coefficient pair one by one
for val in lasso_model_values:
    lasso_model_lst.append(val)
    
# checking the results
for pair in lasso_model_lst:
    print(pair)
# -

#declaring the features for future work
coef = ['h2_cig_first',
'drink',
'h2_cig_third',
'h2_mage_first',
'h2_mage_fourth',
'h2_drink_first',
'h2_feduc_third',
'h2_meduc',
'h2_cigsadddrink_first']

# ### Lasso Model

# +
#declaring the OLS dataframe with the previous features declared
ols_data = birthweight_data.loc[:, coef]

# OLS p-value x-dataset (normal Y)
x_train_OLS, x_test_OLS, y_train_OLS, y_test_OLS = train_test_split(
            ols_data,         # x-variables
            normal_birthweight_target,   # y-variable
            test_size = 0.25,
            random_state = 219)

# INSTANTIATING a model object
lasso_model = sklearn.linear_model.Lasso(alpha = 1.0, normalize = True) # default magitud

# FITTING to the training data
lasso_fit = lasso_model.fit(x_train_OLS, y_train_OLS)

# PREDICTING on new data
lasso_pred = lasso_fit.predict(x_test_OLS)

# SCORING the results
print('Lasso Training Score :', lasso_model.score(x_train_OLS, y_train_OLS).round(4))
print('Lasso Testing Score  :', lasso_model.score(x_test_OLS, y_test_OLS).round(4))


# saving scoring data for future use
lasso_train_score = lasso_model.score(x_train_OLS, y_train_OLS).round(4) # using R-square
lasso_test_score  = lasso_model.score(x_test_OLS, y_test_OLS).round(4)   # using R-square
lasso_test_gap = abs(lasso_train_score - lasso_test_score).round(4)

# displaying and saving the gap between training and testing
print('Lasso Train-Test Gap :', abs(lasso_test_gap).round(4))


# +


# zipping each feature name to its coefficient
lasso_model_values = zip(ols_data.columns, lasso_fit.coef_.round(decimals = 2))


# setting up a placeholder list to store model features
lasso_model_lst = [('intercept', lasso_fit.intercept_.round(decimals = 2))]


# printing out each feature-coefficient pair one by one
for val in lasso_model_values:
    lasso_model_lst.append(val)
    

# checking the results
for pair in lasso_model_lst:
    print(pair)
# -

# ### OLS Model

# +
# INSTANTIATING a model object
lr = LinearRegression()


# FITTING to the training data
lr_fit = lr.fit(x_train_OLS, y_train_OLS)


# PREDICTING on new data
lr_pred = lr_fit.predict(x_test_OLS)


# SCORING the results
print('OLS Training Score :', lr.score(x_train_OLS, y_train_OLS).round(4))  # using R-square
print('OLS Testing Score  :',  lr.score(x_test_OLS, y_test_OLS).round(4)) # using R-square

lr_train_score =  lr.score(x_train_OLS, y_train_OLS)
lr_test_score = lr.score(x_test_OLS, y_test_OLS)

# displaying and saving the gap between training and testing
print('OLS Train-Test Gap :', abs(lr_train_score - lr_test_score).round(4))
lr_test_gap = abs(lr_train_score - lr_test_score).round(4)

# +
# zipping each feature name to its coefficient
lr_model_values = zip(ols_data[coef].columns,
                      lr_fit.coef_.round(decimals = 2))


# setting up a placeholder list to store model features
lr_model_lst = [('intercept', lr_fit.intercept_.round(decimals = 2))]


# printing out each feature-coefficient pair one by one
for val in lr_model_values:
    lr_model_lst.append(val)
    

# checking the results
for pair in lr_model_lst:
    print(pair)
# -

# ### ARD Model

# +
# INSTANTIATING a model object
ard_model = sklearn.linear_model.ARDRegression()


# FITTING the training data
ard_fit = ard_model.fit(x_train_OLS, y_train_OLS)


# PREDICTING on new data
ard_pred = ard_fit.predict(x_test_OLS)
#ard_pred = ard_fit.fit(x_test_FULL)
# saving scoring data for future use
ard_train_score =  ard_model.score(x_train_OLS, y_train_OLS)
ard_test_score  = ard_model.score(x_test_OLS, y_test_OLS)

print('Training Score:',  ard_train_score)
print('Testing Score :',  ard_test_score)

# displaying and saving the gap between training and testing
print('ARD Train-Test Gap :', abs(ard_train_score - ard_test_score).round(4))
ard_test_gap = abs(ard_train_score - ard_test_score).round(4)

# +
# zipping each feature name to its coefficient
ard_model_values = zip(ols_data.columns, ard_fit.coef_.round(decimals = 5))


# setting up a placeholder list to store model features
ard_model_lst = [('intercept', ard_fit.intercept_.round(decimals = 2))]


# printing out each feature-coefficient pair one by one
for val in ard_model_values:
    ard_model_lst.append(val)
    

# checking the results
for pair in ard_model_lst:
    print(pair)
# -

# ### KNN Regression

coef = ['h2_cig_first',
'drink',
'h2_cig_third',
'h2_mage_first',
'h2_mage_fourth',
'h2_drink_first',
'h2_feduc_third',
'h2_meduc',
'h2_cigsadddrink_first']

# +
from sklearn.neighbors import KNeighborsRegressor # KNN for Regression
from sklearn.preprocessing import StandardScaler # standard scaler
# INSTANTIATING a StandardScaler() object
#declaring the OLS dataframe with the previous features declared
birthweight_data = birthweight.drop(['bwght',
                                    'log_bwght'], axis =1)
normal_birthweight_target = birthweight.loc[:,'bwght']
ols_data = birthweight_data.loc[:, coef]



scaler = StandardScaler()

# FITTING the scaler with ols_data
scaler.fit(ols_data)

#y variable is already standarized
# TRANSFORMING our data after fit
x_scaled = scaler.transform(ols_data)

# converting scaled data into a DataFrame
x_scaled_df = pd.DataFrame(x_scaled)

# checking the results
x_scaled_df.describe().round(2)



# +
x_train, x_test, y_train, y_test = train_test_split(
            x_scaled_df,
            normal_birthweight_target,
            test_size = 0.25,
            random_state = 219)

# INSTANTIATING a KNN model object
knn_reg = KNeighborsRegressor(algorithm = 'auto',
                              n_neighbors = 10)


# FITTING to the training data
knn_fit = knn_reg.fit(x_train, y_train)


# PREDICTING on new data
knn_reg_pred = knn_fit.predict(x_test)


# SCORING the results
print('KNN Training Score:', knn_reg.score(x_train, y_train).round(4))
print('KNN Testing Score :',  knn_reg.score(x_test, y_test).round(4))


# saving scoring data for future use
knn_reg_score_train = knn_reg.score(x_train, y_train).round(4)
knn_reg_score_test  = knn_reg.score(x_test, y_test).round(4)


# displaying and saving the gap between training and testing
print('KNN Train-Test Gap:', abs(knn_reg_score_train - knn_reg_score_test).round(4))
knn_reg_test_gap = abs(knn_reg_score_train - knn_reg_score_test).round(4)

# +
# creating lists for training set accuracy and test set accuracy
training_accuracy = []
test_accuracy     = []


# building a visualization of 1 to 50 neighbors
neighbors_settings = range(1, 51)


for n_neighbors in neighbors_settings:
    # Building the model
    clf = KNeighborsRegressor(n_neighbors = n_neighbors)
    clf.fit(x_train, y_train)
    
    # Recording the training set accuracy
    training_accuracy.append(clf.score(x_train, y_train))
    
    # Recording the generalization accuracy
    test_accuracy.append(clf.score(x_test, y_test))


# plotting the visualization
fig, ax = plt.subplots(figsize=(12,8))
plt.plot(neighbors_settings, training_accuracy, label = "training accuracy")
plt.plot(neighbors_settings, test_accuracy, label = "test accuracy")
plt.ylabel("Accuracy")
plt.xlabel("n_neighbors")
plt.legend()
plt.show()
# -

# finding the optimal number of neighbors
opt_neighbors = test_accuracy.index(max(test_accuracy)) + 1
print(f"""The optimal number of neighbors is {opt_neighbors}""")

# ## Comparing the score of different regression models

# +
# comparing results

print(f"""
Model                          Train Score      Test Score      GAP Score
-----                          -----------      ----------      ----------
OLS                            {round(lr_train_score,3)}            {round(lr_test_score,3)}          {round(lr_test_gap,3)}
Lasso  (Final Model)           {round(lasso_train_score,3)}            {round(lasso_test_score,3)}          {round(lasso_test_gap,3)}
ARD                            {round(ard_train_score,3)}            {round(ard_test_score,3)}           {round(ard_test_gap,3)}
KNN Regression                 {round(knn_reg_score_train,3)}             {round(knn_reg_score_test,3)}           {round(knn_reg_test_gap,3)}
""")


# creating a dictionary for model results
model_performance = {
    
    'Model Type'    : ['OLS', 'Lasso', 'ARD'],
           
    'Training' : [lr_train_score, lasso_train_score,
                                   ard_train_score],       
    'Testing'  : [lr_test_score, lasso_test_score,
                                   ard_test_score],               
    'Train-Test Gap' : [lr_test_gap, lasso_test_gap,
                                        ard_test_gap],                
    'Model Size' : [len(lr_model_lst), len(lasso_model_lst),
                                    len(ard_model_lst)],                
    'Model' : [lr_model_lst, lasso_model_lst, ard_model_lst]}
# -

print("""
Features:

h2_cig_first = Number of cigarettes less or equal than 1
drink = quantity of drinks
h2_cig_third = Number of cigarettes more than 21
h2_mage_first = Mother age less than 31 years old
h2_mage_fourth = Mother age more than 51 years old
h2_drink_first = Number of drinks less than 9 cigarettes
h2_feduc_third = years of father education more than 11 
h2_meduc = Years of mother education more than 15
h2_cigsadddrink_first = Cigarettes + drinks less than 18""")



StarcoderdataJupyterScriptsDedupFiltered