# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import pandas as pd
import numpy as np
import tensorflow as tf
from datetime import datetime
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder
from sklearn.model_selection import train_test_split
from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import RandomizedSearchCV
from sklearn.metrics import matthews_corrcoef
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from sklearn.metrics import r2_score
import scipy.stats as stats
from tensorflow import keras
from keras.callbacks import ModelCheckpoint
from keras.models import load_model
from datetime import datetime
from category_encoders import OrdinalEncoder, TargetEncoder
from catboost import CatBoostClassifier, CatBoostRegressor
from mlxtend.feature_selection import SequentialFeatureSelector as SFS
from mlxtend.plotting import plot_sequential_feature_selection as plot_sfs

import warnings
warnings.filterwarnings("ignore")


# +
#Functions

def evaluate(model, X_test, y_test):
    predictions = model.predict(X_test)
    errors = abs(predictions - y_test)
    mape = 100 * np.mean(errors / y_test)
    accuracy = 100 - mape
    roc = roc_auc_score(y_test, predictions)
    print('Model Performance')
    print('Average Error: {:0.4f} degrees'.format(np.mean(errors)))
    print('Accuracy = {:0.2f}%'.format(accuracy))
    print(f'AUC = {roc}')
    return accuracy


# -

train = pd.read_csv('../Data/training_set_features.csv', index_col='respondent_id')
test = pd.read_csv('../Data/test_set_features.csv', index_col ='respondent_id')
labels = pd.read_csv('../Data/training_set_labels.csv', index_col='respondent_id')

train.loc[(train['age_group'] == '65+ Years') & (train['employment_status'].isnull()), 'employment_status'] = 'Not in Labor Force'

# +
num_cols = list(train.select_dtypes('number').columns)

cat_cols = [
    'race',
    'sex',
    'marital_status',
    'rent_or_own',
    'hhs_geo_region',
    'census_msa',
    'employment_industry',
    'employment_occupation'
]

ord_cols = [
    'age_group',
    'education',
    'income_poverty',
    'employment_status'
]




#Impute Train
for col in num_cols:
    train[col] = train[col].fillna(value=-1)
    test[col] = test[col].fillna(value=-1)

for col in (cat_cols + ord_cols):
    train[col] = train[col].fillna(value='None')
    test[col] = test[col].fillna(value='None')
test_labels = labels.copy()    

# +
train['age_group'] = train['age_group'].map({
    '18 - 34 Years': 1,
    '35 - 44 Years': 2,
    '45 - 54 Years': 3,
    '55 - 64 Years': 4,
    '65+ Years': 5
})

train['education'] = train['education'].map({
    '< 12 Years': 1,
    '12 Years': 2,
    'Some College': 3,
    'College Graduate': 4,
    'None': -1
})

train['income_poverty'] = train['income_poverty'].map({
    'None': -1,
    'Below Poverty': 1,
    '<= $75,000, Above Poverty': 2,
    '> $75,000': 3
})

train['employment_status'] = train['employment_status'].map({
    'None': -1,
    'Unemployed': 1,
    'Employed': 2,
    'Not in Labor Force': 3
})




test['education'] = test['education'].map({
    '< 12 Years': 1,
    '12 Years': 2,
    'Some College': 3,
    'College Graduate': 4,
    'None': -1
})

test['income_poverty'] = test['income_poverty'].map({
    'None': -1,
    'Below Poverty': 1,
    '<= $75,000, Above Poverty': 2,
    '> $75,000': 3
})

test['employment_status'] = test['employment_status'].map({
    'None': -1,
    'Unemployed': 1,
    'Employed': 2,
    'Not in Labor Force': 3
})
# -

for x in train[ord_cols].columns:
    print(x, train[x].unique())

all_cols = train.columns
best_cols = [2, 3, 6, 7, 10, 12, 13, 14, 16, 21, 22, 25, 26, 27, 28, 29, 30, 31, 32]

best_cols_names = [all_cols[x] for x in best_cols]

# +
used_cols = list(set(num_cols + best_cols_names))
train = train[used_cols]
test = test[used_cols]

h1n1_labels = labels[['h1n1_vaccine']]
seas_labels = labels[['seasonal_vaccine']]
# -

# # Transformation

# +
#H1N1 Balancing
yeses = h1n1_labels[h1n1_labels['h1n1_vaccine'] == 1]
len_of_yes = len(yeses)
nos = h1n1_labels[h1n1_labels['h1n1_vaccine'] == 0].sample(len_of_yes, random_state=42)

indices = np.concatenate((yeses.index.values, nos.index.values))
h1n1_labels_balanced = h1n1_labels.iloc[indices, :]
h1n1_train_balanced = train.iloc[indices, :]

# enc = OneHotEncoder(categories='auto')
# h1n1_labels_balanced_arr = np.array(h1n1_labels_balanced['h1n1_vaccine']).reshape(-1,1)
# h1n1_labels_trans = enc.fit_transform(h1n1_labels_balanced_arr).toarray()
# h1n1_test_trans = enc.transform(np.array(h1n1_labels['h1n1_vaccine']).reshape(-1,1)).toarray()

# +
#Seasonal Balancing
yeses = seas_labels[seas_labels['seasonal_vaccine'] == 1]
len_of_yes = len(yeses)
nos = seas_labels[seas_labels['seasonal_vaccine'] == 0].sample(len_of_yes, random_state=42)

indices = np.concatenate((yeses.index.values, nos.index.values))
seas_labels_balanced = seas_labels.iloc[indices, :]
seas_train_balanced = train.iloc[indices, :]

# enc = OneHotEncoder(categories='auto')
# seas_labels_balanced_arr = np.array(seas_labels_balanced['seasonal_vaccine']).reshape(-1,1)
# seas_labels_trans = enc.fit_transform(seas_labels_balanced_arr).toarray()
# seas_test_trans = enc.transform(np.array(seas_labels['seasonal_vaccine']).reshape(-1,1)).toarray()
# -

cat_cols = train.select_dtypes('object').columns

# +
ct_h1n1 = ColumnTransformer([('scaler', StandardScaler(), num_cols),
                       ('cat', TargetEncoder(cols=cat_cols, smoothing=100, min_samples_leaf=10), cat_cols)]
                       , remainder='passthrough')

ct_seas = ColumnTransformer([('scaler', StandardScaler(), num_cols),
                       ('cat', TargetEncoder(cols=cat_cols, smoothing=100, min_samples_leaf=10), cat_cols)]
                       , remainder='passthrough')
# -

h1n1_train_trans = ct_h1n1.fit_transform(h1n1_train_balanced, h1n1_labels_balanced)
seas_train_trans = ct_seas.fit_transform(seas_train_balanced, seas_labels_balanced)
test_h1n1 = ct_h1n1.transform(test)
test_seas = ct_seas.transform(test)

# +
true_features = train.sample(1500, random_state=42)
true_indices = true_features.index.values
true_labels = labels.iloc[true_indices,:]

true_features = ct_seas.transform(true_features)
true_labels_rf = true_labels.copy()
true_labels = np.asarray(true_labels)
# -

true_labels

# # H1N1

# ## Random Forest

# ## Train Test Split

X = h1n1_train_trans
y = h1n1_labels_balanced['h1n1_vaccine'].map({'Yes': 1, 'No': 0}).to_numpy()

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42, shuffle=True)

#H1N1 Model
# model_h1n1 = RandomForestRegressor(n_estimators=1200,
#                                min_samples_split=2,
#                                min_samples_leaf=3,
#                                max_features='sqrt',
#                                max_depth=20,
#                                bootstrap=True)
model_h1n1 = CatBoostRegressor(n_estimators=150)
model_h1n1.fit(X_train, y_train)

#Base Model
base_model = RandomForestClassifier(n_estimators=10, random_state=42)
base_model.fit(X_train, y_train)

evaluate(base_model, X_test, y_test)



evaluate(model_h1n1, X_test, y_test)

y_predicted_h1n1 = model_h1n1.predict(X_test)

# ## Neural Network

# ### Train Test Split

X = h1n1_train_trans
y = h1n1_labels_balanced

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42, shuffle=True)

# +
h1n1_mc = ModelCheckpoint('..Models/h1n1_best_model.h5', monitor='val_auc', mode='max', verbose=0, save_best_only=True)

model_h1n1 = keras.Sequential([
    keras.layers.Dense(200, activation='relu', input_dim=31),
    keras.layers.Dense(100, activation='relu'),
    keras.layers.Dense(200, activation='relu'),
    keras.layers.Dense(800, activation='relu'),
    keras.layers.Dense(500, activation='relu'),
    keras.layers.Dense(250, activation='relu'),
    keras.layers.Dense(1, activation='sigmoid')
])
# -

model_h1n1.compile(optimizer='adam',
              loss=tf.keras.losses.BinaryCrossentropy(from_logits=False),
              metrics=[tf.keras.metrics.AUC(from_logits=False), 'accuracy'])

# +
EarlyStopping = tf.keras.callbacks.EarlyStopping(
    monitor='val_loss', min_delta=0, patience=5, verbose=0,
    mode='auto', baseline=None, restore_best_weights=True
)

history = model_h1n1.fit(
    X_train,
    y_train,
    batch_size=50,
    epochs=150,
    validation_data=(X_test, y_test),
    callbacks=[h1n1_mc, EarlyStopping],
    shuffle=True,
    verbose=1
)

# +
#model_h1n1 = load_model('..Models/h1n1_best_model.h5')

y_predicted_h1n1 = model_h1n1.predict(X_test)
roc_auc_score(y_test, y_predicted_h1n1)
# -

# # Seasonal

# ## Random Forest Model

# ### Train Test Split

X = seas_train_trans
y = seas_labels_balanced

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42, shuffle=True)

#Seasonal Model
model_seas = RandomForestRegressor(n_estimators=800,
                                    min_samples_split=2,
                                    min_samples_leaf=4,
                                    max_features='sqrt',
                                    max_depth=20,bootstrap=False)
#model_seas = CatBoostRegressor(n_estimators=150)
model_seas.fit(X_train, y_train)

#Base Model
base_model = RandomForestRegressor(n_estimators=10, random_state=42)
base_model.fit(X_train, y_train)

evaluate(base_model, X_test, y_test)

evaluate(model_seas, X_test, y_test)

y_predicted_seas = model_seas.predict(X_test)

y_predicted = np.vstack((y_predicted_h1n1, y_predicted_seas)).T

roc_auc_score(y_test, y_predicted_seas)

# ## Neural Network

# ### Train Test Split

X = seas_train_trans
y = seas_labels_balanced

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42, shuffle=True)

# +
from tensorflow import keras
from keras.callbacks import ModelCheckpoint

seas_mc = ModelCheckpoint('..Models/seas_best_model.h5', monitor='val_auc_2', mode='max', verbose=1, save_best_only=True)


model_seas = keras.Sequential([
    keras.layers.Dense(100, activation='relu', input_dim=31),
    keras.layers.LeakyReLU(500),
    keras.layers.LeakyReLU(600),
    keras.layers.LeakyReLU(820),
    keras.layers.Dense(200, activation='relu'),
    keras.layers.Dense(1, activation='sigmoid')
])
# -

model_seas.compile(optimizer='adam',
              loss=tf.keras.losses.BinaryCrossentropy(from_logits=False),
              metrics=[tf.keras.metrics.AUC(from_logits=False)])

history = model_seas.fit(
    X_train,
    y_train,
    batch_size=50,
    epochs=150,
    validation_data=(X_test, y_test),
    callbacks=[seas_mc, EarlyStopping]
)

# +
#model_seas = load_model('..Models/seas_best_model.h5')

y_predicted_seas = model_seas.predict(X_test)
roc_auc_score(y_test, y_predicted_seas)

# +
#Full Data Test - Random Forest Network
X_test = true_features
y_test = true_labels_rf.to_numpy()

y_predicted_h1n1 = model_h1n1.predict(X_test).reshape(-1,1)
y_predicted_seas = model_seas.predict(X_test).reshape(-1,1)

y_predicted = np.concatenate((y_predicted_h1n1, y_predicted_seas), axis=1)
roc_auc_score(y_test, y_predicted)
# -





y_predicted_h1n1

pd.DataFrame(y_predicted)

# +
#Full Data Test - Neural Network
X_test = true_features
y_test = true_labels

y_predicted_h1n1 = model_h1n1.predict(X_test)
y_predicted_seas = model_seas.predict(X_test)

y_predicted = np.concatenate((y_predicted_h1n1, y_predicted_seas), axis=1)
roc_auc_score(y_test, y_predicted)
# -



# # Submission

test = pd.read_csv('../Data/test_set_features.csv')
full_test = test.copy()

# +
num_cols = list(test.select_dtypes('number').columns)

cat_cols = [
    'race',
    'sex',
    'marital_status',
    'rent_or_own',
    'hhs_geo_region',
    'census_msa',
    'employment_industry',
    'employment_occupation'
]

ord_cols = [
    'age_group',
    'education',
    'income_poverty',
    'employment_status'
]


#Impute Test
for col in num_cols:
    test[col] = test[col].fillna(value=-1)


for col in (cat_cols+ord_cols):
    test[col] = test[col].fillna(value='None')

    
test['age_group'] = test['age_group'].map({
    '18 - 34 Years': 1,
    '35 - 44 Years': 2,
    '45 - 54 Years': 3,
    '55 - 64 Years': 4,
    '65+ Years': 5
})
    
test['education'] = test['education'].map({
    '< 12 Years': 1,
    '12 Years': 2,
    'Some College': 3,
    'College Graduate': 4,
    'None': -1
})

test['income_poverty'] = test['income_poverty'].map({
    'None': -1,
    'Below Poverty': 1,
    '<= $75,000, Above Poverty': 2,
    '> $75,000': 3
})

test['employment_status'] = test['employment_status'].map({
    'None': -1,
    'Unemployed': 1,
    'Employed': 2,
    'Not in Labor Force': 3
})
# -

test = test[used_cols]

test_h1n1 = ct_h1n1.transform(test)
test_seas = ct_seas.transform(test)



# +
y_h1n1 = model_h1n1.predict(test_h1n1)
y_seas = model_seas.predict(test_seas)

y_comb = np.concatenate((y_h1n1, y_seas), axis=1)
# -

y_comb

# +
results = pd.DataFrame(y_comb, columns=['h1n1_vaccine', 'seasonal_vaccine'])

submission = pd.concat([full_test, results], axis=1)
submission = submission[['respondent_id', 'h1n1_vaccine', 'seasonal_vaccine']]

# +
today = datetime.today().date()

submission.to_csv(f'../Submissions/Neural Network Submission {today}.csv', index=False)
# -

submission











StarcoderdataJupyterScriptsDedupFiltered