strat_train_set_copy = strat_train_set.copy()

housing.plot(kind="scatter", x='longitude', y='latitude')

housing.plot(kind="scatter", x='longitude', y='latitude', alpha=0.1)

strat_train_set_copy.plot(kind='scatter', x='longitude', y='latitude', alpha=0.4,
                          s=strat_train_set_copy.population/100,
                          c=strat_train_set_copy.median_house_value,
                          cmap=plt.get_cmap("jet"),
                          label="population", figsize=(15, 15),
                          colorbar=True)
plt.legend()

corr_matrix = strat_train_set_copy.corr()

corr_matrix.median_house_value.sort_values(ascending=False)

from pandas.plotting import scatter_matrix

attributes = ["median_house_value", "median_income", "total_rooms",
"housing_median_age"]
scatter_matrix(housing[attributes], figsize=(12, 8))

strat_train_set_copy.plot.scatter(x="median_income", y="median_house_value", alpha=0.1)

"""
Explanation: Visualizing Data
End of explanation
"""


housing["rooms_per_household"] = housing["total_rooms"] / housing["households"]
housing["bedrooms_per_room"] = housing["total_bedrooms"]/housing["total_rooms"]
housing["population_per_household"]=housing["population"]/housing["households"]

housing.info()

corr_matrix = housing.corr()
corr_matrix['median_house_value'].sort_values(ascending=False)

"""
Explanation: Experimenting with Attribute Combinations
End of explanation
"""


housing = strat_train_set.drop('median_house_value', axis=1)
housing_labels = strat_train_set['median_house_value'].copy()

housing.info()

housing.dropna(subset=['total_bedrooms']).info()

housing.drop('total_bedrooms', axis=1).info()

housing['total_bedrooms'].fillna(housing['total_bedrooms'].median()).describe()

from sklearn.impute import SimpleImputer
imputer = SimpleImputer(strategy='median')
housing_num = housing.drop("ocean_proximity", axis=1)
imputer.fit(housing_num)
imputer.statistics_

imputer.strategy

housing.drop("ocean_proximity", axis=1).median().values

X = imputer.transform(housing_num)
X

housing_tr = pd.DataFrame(X, columns=housing_num.columns)
housing_tr.head()

"""
Explanation: 2.5 Prepare the Data for Machine Learning Algorithms
End of explanation
"""


from sklearn.preprocessing import LabelEncoder

encoder = LabelEncoder()

housing_cat = housing.ocean_proximity

housing_cat.describe()

housing_cat.value_counts()

housing_cat_encoded = encoder.fit_transform(housing_cat)

housing_cat_encoded

type(housing_cat_encoded)

print(encoder.classes_)

"""
Explanation: Handling Text and Categorical Attributes
End of explanation
"""


from sklearn.preprocessing import OneHotEncoder

encoder = OneHotEncoder()

print(housing_cat_encoded.shape)
print(type(housing_cat_encoded))

(housing_cat_encoded.reshape(-1, 1)).shape

housing_cat_1hot = encoder.fit_transform(housing_cat_encoded.reshape(-1, 1))

housing_cat_1hot

type(housing_cat_1hot)

housing_cat_1hot.toarray()

"""
Explanation: One hot encoding
End of explanation
"""


from sklearn.preprocessing import LabelBinarizer

encoder = LabelBinarizer(sparse_output=False)

housing_cat_1hot = encoder.fit_transform(housing_cat)

housing_cat_1hot

type(housing_cat_1hot)

"""
Explanation: Combine
End of explanation
"""


rooms_ix, bedrooms_ix, population_ix, households_ix = 3, 4, 5, 6

housing.head()

housing.iloc[:, 3]

X = housing.values

# This can be achieved by the iloc, with using .values
housing.iloc[:, [rooms_ix, bedrooms_ix, households_ix, population_ix]].head()

rooms_per_household = X[:, rooms_ix] / X[:, households_ix]
population_per_household = X[:, population_ix] / X[:, households_ix]
bedrooms_per_room = X[:, bedrooms_ix] / X[:, rooms_ix]
np.c_[X, rooms_per_household, population_per_household]
np.c_[X, rooms_per_household, population_per_household, bedrooms_per_room]

from sklearn.base import BaseEstimator, TransformerMixin
rooms_ix, bedrooms_ix, population_ix, households_ix = 3, 4, 5, 6

class CombinedAttributesAdder(BaseEstimator, TransformerMixin):
    def __init__(self, add_bedrooms_per_room=False):
        self.add_bedrooms_per_room = add_bedrooms_per_room
    def fit(self, X, y=None):
        return self
    def transform(self, X, y=None):
        rooms_per_household = X[:, rooms_ix] / X[:, households_ix]
        population_per_household = X[:, population_ix] / X[:, households_ix]
        if self.add_bedrooms_per_room:
            bedrooms_per_room = X[:, bedrooms_ix] / X[:, rooms_ix]
            return np.c_[X, rooms_per_household, population_per_household, bedrooms_per_room]
        else:
            return np.c_[X, rooms_per_household, population_per_household]

attr_adder = CombinedAttributesAdder(add_bedrooms_per_room=False)
housing_extra_attribs = attr_adder.transform(X)

print(housing_extra_attribs.shape)
print(housing.shape)

# Convert back to data frame -- My way
new_columns = housing.columns.append(
    pd.Index(['rooms_per_household', 'population_per_household'])
)
new_columns
housing_extra_attribs_df = pd.DataFrame(housing_extra_attribs, columns=new_columns)
housing_extra_attribs_df.head()

"""
Explanation: Custom Transformers
End of explanation
"""


housing.describe()

housing.total_rooms.describe()

from sklearn.preprocessing import MinMaxScaler

scalar = MinMaxScaler()
scalar.fit(housing["total_rooms"].values.reshape(-1, 1))
pd.DataFrame(scalar.transform(housing["total_rooms"].values.reshape(-1, 1)), columns=["total_rooms"])["total_rooms"].describe()

from sklearn.preprocessing import StandardScaler

scalar = StandardScaler()
scalar.fit(housing["total_rooms"].values.reshape(-1, 1))
pd.DataFrame(scalar.transform(housing["total_rooms"].values.reshape(-1, 1)), columns=["total_rooms"])["total_rooms"].describe()

"""
Explanation: 2.5.4 Feature Scaling
End of explanation
"""


from sklearn.pipeline import Pipeline

num_pipeline = Pipeline([
    ('imputer', SimpleImputer(strategy="median")),
    ('attr_adder', CombinedAttributesAdder()),
    ('std_scaler', StandardScaler())
])

# I want to verify the pipelined version
# doest the same thing as the separated steps

num_pipeline_stage1 = Pipeline([
    ('imputer', SimpleImputer(strategy="median")),
])

X_pipeline = num_pipeline_stage1.fit_transform(housing_num)
X = imputer.transform(housing_num)
X_pipeline
np.array_equal(X, X_pipeline)

num_pipeline_stage2 = Pipeline([
    ('imputer', SimpleImputer(strategy="median")),
    ('attr_adder', CombinedAttributesAdder()),
])

Y = attr_adder.fit_transform(X)
Y_pipeline = num_pipeline_stage2.fit_transform(housing_num)
np.array_equal(Y, Y_pipeline)

num_pipeline_stage3 = Pipeline([
    ('imputer', SimpleImputer(strategy="median")),
    ('attr_adder', CombinedAttributesAdder()),
    ('std_scaler', StandardScaler())
])

Z = scalar.fit_transform(Y)
Z.std(), Z.mean()
Z_pipeline = num_pipeline_stage3.fit_transform(housing_num)
np.array_equal(Z, Z_pipeline)

from sklearn.base import BaseEstimator, TransformerMixin

class DataFrameSelector(BaseEstimator, TransformerMixin):
    def __init__(self, attribute_names):
        self.attribute_names = attribute_names
    def fit(self, X, y=None):
        return self
    def transform(self, X):
        return X[self.attribute_names].values

class CustomizedLabelBinarizer(BaseEstimator, TransformerMixin):
    def __init__(self, sparse_output=False):
        self.encode = LabelBinarizer(sparse_output = sparse_output)
    def fit(self, X, y=None):
        return self.encode.fit(X)
    def transform(self, X):
        return self.encode.transform(X)


num_attribs = list(housing_num)
cat_attribs = ["ocean_proximity"]

num_pipeline = Pipeline([
    ('selector', DataFrameSelector(num_attribs)),
    ('imputer', SimpleImputer(strategy="median")),
    ('attr_adder', CombinedAttributesAdder()),
    ('std_scaler', StandardScaler()),
]
)

cat_pipeline = Pipeline([
    ('selector', DataFrameSelector(cat_attribs)),
    ('label_binarizer', CustomizedLabelBinarizer()),
]
)

# LabelBinarizer().fit_transform(DataFrameSelector(cat_attribs).fit_transform(housing))
# num_pipeline.fit_transform(housing)
# cat_pipeline.fit_transform(housing)

from sklearn.pipeline import FeatureUnion

full_pipeline = FeatureUnion(transformer_list=[
    ('num_pipeline', num_pipeline),
    ('cat_pipeline', cat_pipeline),
])

housing_prepared = full_pipeline.fit_transform(housing)
print(housing_prepared.shape)
housing_prepared

"""
Explanation: 2.5.5 Transformation Pipeline
End of explanation
"""


from sklearn.linear_model import LinearRegression

lin_reg = LinearRegression()
lin_reg.fit(housing_prepared, housing_labels)

some_data = housing[:5]
some_data

some_labels = housing_labels[:5]
some_labels

some_data_prepared = full_pipeline.transform(some_data)
some_data_prepared

print(f'Prediction:\t{lin_reg.predict(some_data_prepared)}')
print(f'Lables:\t\t{list(some_labels)}')

from sklearn.metrics import mean_squared_error

housing_prediction = lin_reg.predict(housing_prepared)
lin_mse = mean_squared_error(housing_prediction, housing_labels)
lin_rmse = np.sqrt(lin_mse)
lin_rmse

"""
Explanation: 2.6.1 Training and Evaluating on the Training Set
End of explanation
"""


from sklearn.tree import DecisionTreeRegressor

tree_reg = DecisionTreeRegressor()
tree_reg.fit(housing_prepared, housing_labels)
tree_predictions = tree_reg.predict(housing_prepared)
tree_mse = mean_squared_error(tree_predictions, housing_labels)
tree_rmse = np.sqrt(tree_mse)
tree_rmse

"""
Explanation: Tree model
End of explanation
"""


from sklearn.model_selection import cross_val_score
scores = cross_val_score(tree_reg, housing_prepared, housing_labels, scoring="neg_mean_squared_error", cv=10)

rmse_scores = np.sqrt(-scores)
rmse_scores

def display_scores(scores):
    print(f'Scores: {scores}')
    print(f'Mean: {scores.mean()}')
    print(f'STD: {scores.std()}')

display_scores(rmse_scores)

"""
Explanation: 2.6.2 Better Evaluation Using Cross-Validation
End of explanation
"""


from sklearn.ensemble import RandomForestRegressor

forest_reg = RandomForestRegressor()
forest_reg.fit(housing_prepared, housing_labels)
forest_scores = cross_val_score(forest_reg, housing_prepared, housing_labels, scoring="neg_mean_squared_error", cv=10)
forest_rmse_scores = np.sqrt(-forest_scores)
display_scores(forest_rmse_scores)

forest_prediction = forest_reg.predict(housing_prepared)
forest_rmse = np.sqrt(mean_squared_error(forest_prediction, housing_labels))
forest_rmse

"""
Explanation: Random Forest
End of explanation
"""


from sklearn.base import BaseEstimator, TransformerMixin

class EX3NumSelector(BaseEstimator, TransformerMixin):
    def __init__(self):
        pass
    def fit(self, X, y=None):
        return self
    def transform(self, X):
        X['pop_per_hhold'] = X['population'] / X['households']
        return X[['median_income', 'pop_per_hhold', 'longitude']].values

class EX3CatSelector(BaseEstimator, TransformerMixin):
    def __init__(self):
        pass
    def fit(self, X, y=None):
        return self
    def transform(self, X):
        Y = housing['ocean_proximity']
        Y[Y != 'INLAND'] = 'NON_INLAND'
        return Y.values

num_sel = EX3NumSelector()
num_sel.fit_transform(housing)

cat_sel = EX3CatSelector()
cat_sel.fit_transform(housing)

num_pipeline = Pipeline([
    ('selector', EX3NumSelector()),
    ('imputer', SimpleImputer(strategy="median")),
    ('std_scaler', StandardScaler()),
]
)

cat_pipeline = Pipeline([
    ('selector', EX3CatSelector()),
    ('label_binarizer', CustomizedLabelBinarizer()),
]
)

full_pipeline = FeatureUnion(transformer_list=[
    ('num_pipeline', num_pipeline),
    ('cat_pipeline', cat_pipeline),
])

housing_prepared = full_pipeline.fit_transform(housing)
print(housing_prepared.shape)
housing_prepared

forest_reg.fit(housing_prepared, housing_labels)
forest_scores = cross_val_score(forest_reg, housing_prepared, housing_labels, scoring="neg_mean_squared_error", cv=10)
forest_rmse_scores = np.sqrt(-forest_scores)
display_scores(forest_rmse_scores)

"""
Explanation: Ex03
Try adding a transformer in the preparation pipeline to select only the most
important attributes.
The importance of each feature is show below in 2.7.4
```python



sorted(zip(feature_importances, attributes), reverse=True)
[(0.32649798665134971, 'median_income'),
(0.15334491760305854, 'INLAND'),
(0.11305529021187399, 'pop_per_hhold'),
(0.07793247662544775, 'bedrooms_per_room'),
(0.071415642259275158, 'longitude'),
(0.067613918945568688, 'latitude'),
(0.060436577499703222, 'rooms_per_hhold'),
(0.04442608939578685, 'housing_median_age'),
(0.018240254462909437, 'population'),
(0.01663085833886218, 'total_rooms'),
(0.016607686091288865, 'total_bedrooms'),
(0.016345876147580776, 'households'),
(0.011216644219017424, '<1H OCEAN'),
(0.0034668118081117387, 'NEAR OCEAN'),
(0.0026848388432755429, 'NEAR BAY'),
(8.4130896890070617e-05, 'ISLAND')]
```



Based on the ranking, I will select the following 3:
* median_income
* INLAND
* pop_per_hhold
End of explanation
"""


# from sklearn.model_selection import GridSearchCV

# param_grid = [
#     {'n_estimators': [3, 10, 30], 'max_features': [2,4,6,8]},
#     {'bootstrap': [False], 'n_estimators': [3, 10, 30], 'max_features': [2,4,6,8]}
# ]

# forest_reg = RandomForestRegressor()

# grid_search = GridSearchCV(forest_reg, param_grid, cv=5, scoring="neg_mean_squared_error")

# grid_search.fit(housing_prepared, housing_labels)

# grid_search.best_params_

# grid_search.best_estimator_

# cvres = grid_search.cv_results_
# for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
#     print(np.sqrt(-mean_score), params)

"""
Explanation: Conclusions of Ex03
With only 3 features, we do see the performance degradation. Adding one more feature 'longitude' improves a little
bit, which makes sense.
2.7.1 Grid Search
End of explanation
"""


# feature_importances = grid_search.best_estimator_.feature_importances_
# feature_importances

# extra_attribs = ['rooms_per_hhold', 'pop_per_hhold']

# cat_one_hot_attribs = list(encoder.classes_)
# cat_one_hot_attribs

# attributes = num_attribs + extra_attribs + cat_one_hot_attribs
# attributes, len(attributes)

# sorted(zip(feature_importances, attributes), reverse=True)

"""
Explanation: 2.7.4 Analyze the best models and their errors
End of explanation
"""


# final_model = grid_search.best_estimator_
# X_test = strat_test_set.drop("median_house_value", axis=1)
# y_test = strat_test_set.median_house_value.copy()
# X_test_prepared = full_pipeline.transform(X_test)

# final_predictions = final_model.predict(X_test_prepared)
# final_mse = mean_squared_error(final_predictions, y_test)
# final_rmse = np.sqrt(final_mse)
# final_rmse

"""
Explanation: 2.7.5 Evaluate Your System on the Test Set
End of explanation
"""


