# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import GridSearchCV

# Let's begin with a very simple synthetic dataset.

x = np.linspace(0, 1, 1000)  # between 0 and 1
y = 3 * x**2 - 2 * x + 12 + np.random.randn(1000) * 0.2  # a second-order polynomial in x, with some noise
plt.plot(x, y)

# We're going to fit this with a standard polynomial regression. We don't know what degree the data is, so we'll cross-validate against that parameter and select the best model.

# +
model = Pipeline([
    ("polynomial_features", PolynomialFeatures()),  # let's take linear x and make polynomial features from it
    ("linear_regression", LinearRegression(fit_intercept=True))  # then a linear regression on the polynomial features
])

# Note that the LinearRegression has fit_intercept=True because we always want to fit an intercept, 
# and this is not a parameter we will ever be varying. We talked about this at the pub - in linear regressions,
# there's no need to scale variables ( LinearRegression does offer a "normalize" argument ), but we want
# to look at the actual coefficients that come out of our fits because they have real-world meaning.
# Therefore, we don't normalize, and as such, we must allow the linear regression to fit an intercept.


# Now we decide on what hyperparameters of the model we allow to vary.
# In this case, we'll vary the degree of the polynomial.
parameters = {
    "polynomial_features__degree": [1, 2, 3, 4, 5],
}
# -

# We have a model set out. Now let's fit it and see what the result is like.

x.reshape(-1, 1).shape

# +
gridsearch = GridSearchCV(
    model,  # our pipeline
    parameters,  # the parameters of the pipeline that we want to vary
    n_jobs=-1,  # use all cores on our computer
    scoring="neg_mean_squared_error",  # evaluate the model using mean squared error
    cv=5  # and perform 5-fold cross-validation to score the model
)

gridsearch.fit(x.reshape(-1, 1), y)  # sklearn wants *observations* in the first axis and *features* in the second axis
# -

# So how did we do ?

print("Best MSE score ( lower is better ) : {}".format(gridsearch.best_score_))
print("Best hyperparameters : {}".format(gridsearch.best_params_))

# Looks like we did well, and our model selected the correct polynomial for us : degree 2.
# Let's look at the actual model from within our gridsearch object.

best_model = gridsearch.best_estimator_
print(best_model)

# Great, we've recovered a fully tuned pipeline from our gridsearch. Let's make predictions and compare.

# +
# Predict y_hat from our input features
predictions = best_model.predict(x.reshape(-1, 1))

# Plot to compare
plt.scatter(x, y, alpha=0.3)
plt.plot(x, predictions, lw=5, c="k")
# -

# Great. Onto a more complicated dataset, with multiple features.

# +
# Features : number of rooms, number of baths, and square footage of the house
df = pd.DataFrame({
    "rooms": np.random.randint(1, 6, 1000),  # between one and six rooms
    "baths": np.random.randint(1, 4, 1000),  # between one and four baths
    "sqft": np.random.rand(1000) * 5000 # between 0 and 5000 sqft
})

# GROUND TRUTH : house price ( we don't know this relation )
df["price"] = df.rooms * 400 + \
              df.baths * 200 + \
              df.sqft * 10 + \
              df.sqft ** 2 + \
              df.rooms * df.baths * df.sqft * 0.5 + \
              np.random.randn(1000) * 50000

df.head()
# -

# Let's build the same model and grid search over the polynomial degree but also over whether we only want interaction terms or if we want to allow cross-terms. Our ground truth is a degree-3 polynomial with both cross- and interaction-terms.

# +
model = Pipeline([
    ("polynomial_features", PolynomialFeatures()),
    ("linear_regression", LinearRegression(fit_intercept=True))
])

parameters = {
    "polynomial_features__degree": [1, 2, 3, 4, 5],
    "polynomial_features__interaction_only": [True, False]
}

gridsearch = GridSearchCV(
    model,
    parameters,
    n_jobs=-1,
    scoring="neg_mean_squared_error",
    cv=5
)

gridsearch.fit(df[["rooms", "baths", "sqft"]], df.price)

print("Our best model has the parameters : \n{}".format(gridsearch.best_params_))
# -

# Let's make predictions and compare to our data.

df["yhat"] = gridsearch.best_estimator_.predict(df[["rooms", "baths", "sqft"]])
df.head(10)

# Residuals
plt.scatter(range(1000), df.yhat - df.price)

StarcoderdataJupyterScriptsDedupFiltered