dfnum_t2 = pd.read_csv('transformed_numerical_dataset_imputed.csv', index_col=['Dataset','Id'])

dfnum_t2.head()

dfnum_t2.tail()

"""
Explanation: Load numerical data
Generated in notebook data_exploration_numerical_features.ipynb
End of explanation
"""


target = pd.read_csv('../data/train_target.csv')

scaler = sk.preprocessing.StandardScaler()

def transform_target(target):
    logtarget = np.log1p(target / 1000)
    return scaler.fit_transform(logtarget)


def inverse_transform_target(target_t):
    logtarget = scaler.inverse_transform(target_t)
    return np.expm1(logtarget) * 1000


target_t = transform_target(target)

# Test
assert all(target == inverse_transform_target(target_t))

"""
Explanation: Recreate transformed (standardized) sale price
End of explanation
"""


data = dfnum_t2.loc['train',:].copy()
data['SalePrice'] = target_t


fig, axes = plt.subplots(2,2,figsize=(10,10))
for feature, ax in zip(data.columns[:4], itertools.chain.from_iterable(axes)):
    ax.plot(data[feature], data['SalePrice'], 'o')
    ax.set(xlabel=feature, ylabel='SalePrice')
    


"""
Explanation: Ordinary Least Squares model with key features
We're left with 22 features. The first 4 should all be highly correlated with the price.
End of explanation
"""


regression1 = smapi.ols("SalePrice ~ OverallQual + GrLivArea + GarageCars + GarageArea", data=data).fit()
regression1.summary()

"""
Explanation: Let's build a simple linear regression model based on these 4 features.
End of explanation
"""


data.columns

"""
Explanation: R-squared equals 0.79 so it's pretty good for a first try. Let's see what happens if we include all our numerical features.
End of explanation
"""


data.columns

desc = 'SalePrice ~ ' + ' + '.join(data.drop('SalePrice', axis=1))
desc

"""
Explanation: Statsmodels gets confused with columns that start with a digit, so let's rename that column first
End of explanation
"""


def get_data(X, y):
    df = X.copy()
    df['SalePrice'] = y
    return df

def ols1(X, y):
    data = get_data(X, y)
    return smapi.ols("SalePrice ~ OverallQual + GrLivArea + GarageCars + GarageArea", data=data)

def ols2(X, y):
    data = get_data(X, y)
    return smapi.ols(desc, data=data)

"""
Explanation: As can be seen below, using more numerical values improves R-squared to 0.88 which is pretty good, though there's of course a risk of overfitting.
Cross validation
End of explanation
"""


def rmse(prediction, exact):
    return np.mean((prediction - exact)**2.0)**0.5

def run_experiment(estimator, scoring=rmse):
    Xtrain, Xtest, ytrain, ytest = sk.model_selection.train_test_split(data.drop('SalePrice', axis=1), data['SalePrice'])
    model = estimator(Xtrain, ytrain).fit()
    return scoring(model.predict(Xtest), ytest)

def cross_validate(estimator, cv=5):
    return np.array([run_experiment(estimator) for _ in range(cv)])

for model in [ols1, ols2]:
    errors = cross_validate(model)
    print(errors, errors.mean())

        
        

"""
Explanation: Test the model
Use sklearn.model_selection.train_test_split to run some experiments and validate the models
End of explanation
"""


for model in [ols1, ols2]:
    mse = np.sqrt(-sk.model_selection.cross_val_score(samlib.Regressor(model), data.drop('SalePrice', axis=1), y=data['SalePrice'],  
                                   scoring='neg_mean_squared_error', cv=5))
    print(mse, mse.mean())

"""
Explanation: Use sklearn.model_selection_cross_val_score to validate the models
End of explanation
"""


regression2 = smapi.ols(desc, data=data).fit()
submission_t = regression2.predict(dfnum_t2.loc['test',:])

"""
Explanation: Make a submission
End of explanation
"""


submission = inverse_transform_target(submission_t)
submission

def save(filename, submission):
    df = pd.DataFrame(data={
            "Id": np.arange(len(submission)) + 1461,
            "SalePrice": submission
            })
    df.to_csv(filename, index=False)
    
save('ols_key_numerical_features_only_imputed.csv', submission)

"""
Explanation: Scale the result
End of explanation
"""


