train["data"] = "train"
test["data"] = "test"

combined_data = pd.concat([train, test])
encoded = pd.get_dummies(combined_data[["X0", "X1", "X2", "X3", "X4", "X5", "X6", "X8"]])
drop_cat = combined_data.drop(["X0", "X1", "X2", "X3", "X4", "X5", "X6", "X8"], axis=1)
combined_data_clean = drop_cat.join(encoded)

train_data = combined_data_clean[combined_data_clean.data == "train"].copy()
test_data = combined_data_clean[combined_data_clean.data == "test"].copy()

train_data.drop("data", axis=1, inplace=True)
test_data.drop(["data", "y"], axis=1, inplace=True)

train_data.columns

test_data.columns

y_train = train_data["y"].astype(np.float32)
x_train =  train_data.drop("y", axis=1).astype(np.float32)
x_test = test_data.astype(np.float32)

x_train.shape

train_reshaped = np.array([i.reshape((-1, 1)) for i in x_train.values])
train_reshaped = train_reshaped.astype(np.float32)
train_reshaped.shape

"""
Explanation: Combine train and test data for one-hot encoding
End of explanation
"""


n_comp = 128
pca = PCA(n_components=n_comp, random_state=42)
pca2_results_train = pca.fit_transform(x_train)
pca2_results_test = pca.transform(x_test)

train_reshaped = np.array([i.reshape((-1, 1)) for i in pca2_results_train])
train_reshaped = train_reshaped.astype(np.float32)
train_reshaped.shape

"""
Explanation: Adjust for multicollinearity

Idea: treat the features as sequences but using the raw data 0/1 can cause NaN easily / probably due to multicollinearity
Therefore either take VIF or PCA to adjust for it

vif = pd.DataFrame()
vif["VIF Factor"] = [variance_inflation_factor(x_train.values, i) for i in range(x_train.shape[1])]
vif["features"] = x_train.columns
End of explanation
"""


# Idea: Simple model
model = Sequential()
model.add(Bidirectional(SimpleRNN(128, return_sequences=True, activation="relu"), input_shape=(None, 1)))
model.add(Bidirectional(SimpleRNN(64, return_sequences=True, activation="relu")))
model.add(Bidirectional(SimpleRNN(32, return_sequences=False, activation="relu")))
model.add(Dropout(0.5))
model.add(Dense(1, activation="linear"))
model.compile(optimizer="rmsprop", loss="mse")
model.summary()

# Idea: Funnel -> reduce information after each layer / deep model
model = Sequential()
model.add(Bidirectional(SimpleRNN(64, return_sequences=True, activation="relu"), input_shape=(None, 1)))
model.add(Bidirectional(SimpleRNN(64, return_sequences=True, activation="relu")))
model.add(TimeDistributed(Dense(32, activation="relu")))
model.add(Bidirectional(SimpleRNN(32, return_sequences=True, activation="relu")))
model.add(Bidirectional(SimpleRNN(32, return_sequences=True, activation="relu")))
model.add(TimeDistributed(Dense(16, activation="relu")))
model.add(Bidirectional(SimpleRNN(16, return_sequences=False, activation="relu")))
model.add(Dropout(0.5))
model.add(Dense(1, activation="linear"))
model.compile(optimizer="rmsprop", loss="mse")
model.summary()

early_stop = EarlyStopping(monitor="loss", patience=10)
file_path = "weights.{epoch:02d}-{val_loss:.2f}.hdf5"
checkpoint = ModelCheckpoint(file_path)
model_run = model.fit(train_reshaped, y_train, epochs=100 ,validation_split=0.02, callbacks=[early_stop, checkpoint])

y_pred_train = model.predict(train_reshaped)

print("the R2 score is : {}".format(r2_score(y_train, y_pred_train)))

"""
Explanation: Train the model
End of explanation
"""


test_reshaped = np.array([i.reshape((-1, 1)) for i in pca2_results_test])
test_reshaped = test_reshaped.astype(np.float32)
test_reshaped.shape

y_pred_test = model.predict(test_reshaped)

output = pd.DataFrame({"ID": test.index, "y": y_pred_test.reshape(-1)})

output.head()

output.to_csv("submissions_{}.csv".format(datetime.datetime.today()), index=False)

"""
Explanation: Prediction
End of explanation
"""


sub_1 = pd.read_csv("submission_baseLine.csv")

sub_2 = pd.read_csv("submissions_2017-05-31 15:48:40.546392.csv")

sub_3 = output.copy()

mean_pred = (sub_1.y.values + sub_2.y.values + sub_3.y.values) / 3

output_mean = pd.DataFrame({"ID": test.index, "y": mean_pred})

output_mean.to_csv("submissions_mean_{}.csv".format(datetime.datetime.today()), index=False)

sub_1 = pd.read_csv("submission_baseLine.csv")

sub_2 = pd.read_csv("submissions_2017-05-31 15:48:40.546392.csv")

mean_pred = (sub_1.y.values + sub_2.y.values ) / 2

output_mean = pd.DataFrame({"ID": test.index, "y": mean_pred})

output_mean.to_csv("submissions_mean_2_{}.csv".format(datetime.datetime.today()), index=False)

"""
Explanation: Combine submission
End of explanation
"""


