# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ## Setup

import pandas as pd
import numpy as np
import os
np.random.seed(24)

housing = pd.read_csv("datasets/housing/housing.csv")
housing.head()

# #### Data Split by stratification

housing["income_cat"] = np.ceil(housing["median_income"]/1.5)
housing["income_cat"].where(housing["income_cat"]<5, 5.0, inplace=True)
housing["income_cat"].value_counts()

from sklearn.model_selection import StratifiedShuffleSplit
split = StratifiedShuffleSplit(n_splits = 1, test_size=0.2, random_state=42)
for train_index, test_index in split.split(housing, housing["income_cat"]):
    strat_train_set = housing.loc[train_index]
    strat_test_set = housing.loc[test_index]
print("Train set:", strat_train_set.shape)
print("Test set:", strat_test_set.shape)

# # %matplotlib inline
import matplotlib.pyplot as plt
f,ax = plt.subplots(1,2,sharey=True,figsize=(8,6))
# ax[0].hist(strat_train_set["income_cat"])
# ax[0].set_title("training set")
# ax[1].hist(housing["income_cat"])
# ax[1].set_title("full set")
strat_train_set["income_cat"].plot(kind="hist", ax=ax[0], title="training set")
housing["income_cat"].plot(kind="hist", ax=ax[1], title="full dataset")
plt.show()

strat_train_set.drop(["income_cat"], axis=1, inplace=True)

strat_train_set.head(2)

# #### Data Preparation
# * For numerical attributes, fill in missing data, standardization, add additional attribute
# * For categorical attributes, apply one-hot-encoder

housing = strat_train_set.drop("median_house_value", axis=1)
housing_label = strat_train_set["median_house_value"].copy()
print(housing.head(2), "\n")
print(housing_label.head(2))

from sklearn.base import BaseEstimator, TransformerMixin
class CustomBinarizer(BaseEstimator, TransformerMixin):
    def fit(self, X, y=None,**fit_params):
        return self
    def transform(self, X):
        return LabelBinarizer().fit(X).transform(X)


# +
from sklearn.base import BaseEstimator, TransformerMixin

# column index
rooms_ix, bedrooms_ix, population_ix, household_ix = 3, 4, 5, 6
class CombinedAttributesAdder(BaseEstimator, TransformerMixin):
    def __init__(self, add_bedrooms_per_room = True):
        self.add_bedrooms_per_room = True
    def fit(self, X, y=None):
        return self
    def transform(self, X, y=None):
        rooms_per_household = X[:, rooms_ix] / X[:, household_ix]
        population_per_household = X[:, population_ix] / X[:, household_ix]
        if self.add_bedrooms_per_room: 
            bedrooms_per_room = X[:, rooms_ix]/X[:, household_ix]
            return np.c_[X, rooms_per_household, population_per_household, bedrooms_per_room]
        else:
            return np.c_[X, rooms_per_household, population_per_household]


# -

class DataFrameSelector(BaseEstimator, TransformerMixin):
    def __init__(self, attribute_names):
        self.attribute_names = attribute_names
    def fit(self, X, y=None):
        return self # do nothing
    def transform(self, X, y=None):
        return X[self.attribute_names].values # return numpy array


num_attribs = housing.columns.values.tolist()
num_attribs.remove("ocean_proximity")
num_attribs

cat_attribs = ["ocean_proximity"]

# +
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import Imputer
from sklearn.preprocessing import LabelBinarizer
num_pipeline = Pipeline([
        ('selector', DataFrameSelector(num_attribs)), # select numeric columns
        ('imputer', Imputer(strategy="median")), # fill missing values with median
        ('attribs_adder', CombinedAttributesAdder()), # add extra columns
        ('std_scaler', StandardScaler()), # standardize
    ])

cat_pipeline = Pipeline([
        ('selector', DataFrameSelector(cat_attribs)),  # select categorical columns
        ('label_binarizer', CustomBinarizer()), # binarize categorical attributes
    ])

# -

from sklearn.pipeline import FeatureUnion
full_pipeline = FeatureUnion(transformer_list = [
    ("num_pipeline", num_pipeline),
    ("cat_pipeline", cat_pipeline),
])

housing_pred = full_pipeline.fit_transform(housing)

housing_pred

housing_pred.shape

# ## Practice

# ### Question 1

# Try a Support Vector Machine regressor (`sklearn.svm.SVR`), with various hyperparameters such as `kernel="linear"` (with various values for the `C` hyperparameter) or `kernel="rbf"` (with various values for the `C` and `gamma` hyperparameters). Don't worry about what these hyperparameters mean for now. How does the best `SVR` predictor perform?

from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVR
param_grid = [{"kernel": ["linear"], "C": [10., 30., 100., 300., 1000., 3000., 10000., 30000.0]},
              {"kernel": ["rbf"], "C": [1.0, 3.0, 10., 30., 100., 300., 1000.0], 
              "gamma": [0.01, 0.03, 0.1, 0.3, 1.0, 3.0]}]
svm_reg = SVR()
grid_search = GridSearchCV(svm_reg, param_grid, cv=5, scoring="neg_mean_squared_error", verbose=2, n_jobs=4)
grid_search.fit(housing_pred, housing_label)

grid_search.best_estimator_

negative_mse = grid_search.best_score_
print(np.sqrt(-negative_mse))

grid_search.best_params_

# ### Question 2 

# Try replacing `GridSearchCV` with `RandomizedSearchCV`.

from sklearn.model_selection import RandomizedSearchCV
param_hist ={"kernel": ["linear", "rbf"], "C": np.linspace(1.0, 400.0, 10), "gamma": np.linspace(0.1, 3.0,10)}
svm_reg = SVR()
rnd_search = RandomizedSearchCV(estimator=svm_reg, cv=5, param_distributions=param_hist, 
                                n_jobs=4, scoring="neg_mean_squared_error", verbose=4)

rnd_search.fit(housing_pred, housing_label)

rnd_search.best_estimator_

rmse =np.sqrt(-rnd_search.best_score_)
rmse

rnd_search.best_params_

# ### Question 3 

# Try adding a transformer in the preparation pipeline to select only the most important attributes.

from sklearn.model_selection import RandomizedSearchCV
from sklearn.ensemble import RandomForestRegressor
forest_reg = RandomForestRegressor(random_state=42)
param_hist = {"n_estimators": np.arange(10, 500, 20),
             "max_depth": np.arange(5, 50, 5),
             "max_features": [2,4,6,8]}
rnd_search = RandomizedSearchCV(estimator=forest_reg, cv=5, param_distributions=param_hist,
                                scoring="neg_mean_squared_error",
                                n_jobs=4,verbose=4)

rnd_search.fit(housing_pred, housing_label)

cvre = rnd_search.cv_results_
for mean_score, params in zip(cvre["mean_test_score"], cvre["params"]):
    print(np.sqrt(-mean_score), params)

rnd_search.best_estimator_

feature_importances = rnd_search.best_estimator_.feature_importances_
extra_attribs = ["room_per_household", "pop_per_household", "bedroom_per_room"]
encoder = LabelBinarizer()
housing_cat_encoded = encoder.fit_transform(housing["ocean_proximity"])
cat_attribs = encoder.classes_.tolist()
attribs = num_attribs + extra_attribs + cat_attribs
sorted(zip(feature_importances, attribs))

# pick the feature of important
from sklearn.base import BaseEstimator, TransformerMixin
def indices_of_top_k(arr, k):
    return(np.sort(np.argpartition(np.array(arr), -k)[-k:]))
class TopFeatureSelector(BaseEstimator, TransformerMixin):
    def __init__(self, feature_importance, k):
        self.feature_importance = feature_importance
        self.k = k
    def fit(self, X, y=None):
        self.feature_indices_ = indices_of_top_k(self.feature_importance, self.k)
        return self
    def transform(self, X, y=None):
        return(X[:, self.feature_indices_])


indices_of_top_k(feature_importances, 5) # the top five important features

np.array(attribs)[indices_of_top_k(feature_importances, 5)]

preparation_and_feature_selection_pipeline = Pipeline([
    ("preparation", full_pipeline),
    ("feature_selection", TopFeatureSelector(feature_importances, 5))])
housing_prepared_top_k_features = preparation_and_feature_selection_pipeline.fit_transform(housing)
housing_prepared_top_k_features[:3,:]

from sklearn.model_selection import cross_val_score
rnd_reg=RandomForestRegressor(max_features=8, n_estimators=30)
scores = cross_val_score(rnd_reg, X=housing_pred, y=housing_label, 
                           scoring="neg_mean_squared_error", cv=5, 
                           n_jobs=-1,verbose=4)
rmse = [np.sqrt(-i) for i in scores]
print(rmse)

# using n_estimator = 350
rnd_reg = RandomForestRegressor(max_features=8, n_estimators=350)
scores = cross_val_score(estimator=rnd_reg, cv=5, X=housing_pred, y=housing_label, 
                        scoring="neg_mean_squared_error", n_jobs=-1, verbose=4)
rmse = [np.sqrt(-i) for i in scores]
print(rmse)

# ### Question 4

# Try creating a single pipeline that does the full data preparation plus the final prediction.

prepare_prediction_pipeline = Pipeline([
    ("preparation", full_pipeline),
    ("feature_selection", TopFeatureSelector(feature_importances, 6)), 
    ("prediction", rnd_search.best_estimator_)
])

some_data = housing.iloc[:4]
some_data_label = housing_label[:4]
some_data

prepare_prediction_pipeline.fit(some_data, some_data_label)

print("Label", some_data_label.values)
print("Prediction", prepare_prediction_pipeline.predict(some_data))      

# ### Question 5

# Automatically explore some preparation options using GridSearchCV

# +
from sklearn.model_selection import GridSearchCV
param_grid = [
        {'preparation__num_pipeline__imputer__strategy': ['mean', 'median', 'most_frequent'],
         'feature_selection__k': [4, 5, 6]}
]

grid_search_prep = GridSearchCV(preparation_and_feature_selection_pipeline, param_grid, cv=3,
                                scoring='neg_mean_squared_error', verbose=4, n_jobs=-1)
grid_search_prep.fit(housing, housing_label)
# -



StarcoderdataJupyterScriptsDedupFiltered