strat_train_set_copy = strat_train_set.copy()

housing.plot(kind="scatter", x='longitude', y='latitude')

housing.plot(kind="scatter", x='longitude', y='latitude', alpha=0.1)

strat_train_set_copy.plot(kind='scatter', x='longitude', y='latitude', alpha=0.4,
                          s=strat_train_set_copy.population/100,
                          c=strat_train_set_copy.median_house_value,
                          cmap=plt.get_cmap("jet"),
                          label="population", figsize=(15, 15),
                          colorbar=True)
plt.legend()

corr_matrix = strat_train_set_copy.corr()

corr_matrix.median_house_value.sort_values(ascending=False)

from pandas.plotting import scatter_matrix

attributes = ["median_house_value", "median_income", "total_rooms",
"housing_median_age"]
scatter_matrix(housing[attributes], figsize=(12, 8))

strat_train_set_copy.plot.scatter(x="median_income", y="median_house_value", alpha=0.1)

"""
Explanation: Visualizing Data
End of explanation
"""


housing["rooms_per_household"] = housing["total_rooms"] / housing["households"]
housing["bedrooms_per_room"] = housing["total_bedrooms"]/housing["total_rooms"]
housing["population_per_household"]=housing["population"]/housing["households"]

housing.info()

corr_matrix = housing.corr()
corr_matrix['median_house_value'].sort_values(ascending=False)

"""
Explanation: Experimenting with Attribute Combinations
End of explanation
"""


housing = strat_train_set.drop('median_house_value', axis=1)
housing_labels = strat_train_set['median_house_value'].copy()

housing.info()

housing.dropna(subset=['total_bedrooms']).info()

housing.drop('total_bedrooms', axis=1).info()

housing['total_bedrooms'].fillna(housing['total_bedrooms'].median()).describe()

from sklearn.impute import SimpleImputer
imputer = SimpleImputer(strategy='median')
housing_num = housing.drop("ocean_proximity", axis=1)
imputer.fit(housing_num)
imputer.statistics_

imputer.strategy

housing.drop("ocean_proximity", axis=1).median().values

X = imputer.transform(housing_num)
X

housing_tr = pd.DataFrame(X, columns=housing_num.columns)
housing_tr.head()

"""
Explanation: 2.5 Prepare the Data for Machine Learning Algorithms
End of explanation
"""


from sklearn.preprocessing import LabelEncoder

encoder = LabelEncoder()

housing_cat = housing.ocean_proximity

housing_cat.describe()

housing_cat.value_counts()

housing_cat_encoded = encoder.fit_transform(housing_cat)

housing_cat_encoded

type(housing_cat_encoded)

print(encoder.classes_)

"""
Explanation: Handling Text and Categorical Attributes
End of explanation
"""


from sklearn.preprocessing import OneHotEncoder

encoder = OneHotEncoder()

print(housing_cat_encoded.shape)
print(type(housing_cat_encoded))

(housing_cat_encoded.reshape(-1, 1)).shape

housing_cat_1hot = encoder.fit_transform(housing_cat_encoded.reshape(-1, 1))

housing_cat_1hot

type(housing_cat_1hot)

housing_cat_1hot.toarray()

"""
Explanation: One hot encoding
End of explanation
"""


from sklearn.preprocessing import LabelBinarizer

encoder = LabelBinarizer(sparse_output=False)

housing_cat_1hot = encoder.fit_transform(housing_cat)

housing_cat_1hot

type(housing_cat_1hot)

"""
Explanation: Combine
End of explanation
"""


rooms_ix, bedrooms_ix, population_ix, households_ix = 3, 4, 5, 6

housing.head()

housing.iloc[:, 3]

X = housing.values

# This can be achieved by the iloc, with using .values
housing.iloc[:, [rooms_ix, bedrooms_ix, households_ix, population_ix]].head()

rooms_per_household = X[:, rooms_ix] / X[:, households_ix]
population_per_household = X[:, population_ix] / X[:, households_ix]
bedrooms_per_room = X[:, bedrooms_ix] / X[:, rooms_ix]
np.c_[X, rooms_per_household, population_per_household]
np.c_[X, rooms_per_household, population_per_household, bedrooms_per_room]

from sklearn.base import BaseEstimator, TransformerMixin
rooms_ix, bedrooms_ix, population_ix, households_ix = 3, 4, 5, 6

class CombinedAttributesAdder(BaseEstimator, TransformerMixin):
    def __init__(self, add_bedrooms_per_room=False):
        self.add_bedrooms_per_room = add_bedrooms_per_room
    def fit(self, X, y=None):
        return self
    def transform(self, X, y=None):
        rooms_per_household = X[:, rooms_ix] / X[:, households_ix]
        population_per_household = X[:, population_ix] / X[:, households_ix]
        if self.add_bedrooms_per_room:
            bedrooms_per_room = X[:, bedrooms_ix] / X[:, rooms_ix]
            return np.c_[X, rooms_per_household, population_per_household, bedrooms_per_room]
        else:
            return np.c_[X, rooms_per_household, population_per_household]

attr_adder = CombinedAttributesAdder(add_bedrooms_per_room=False)
housing_extra_attribs = attr_adder.transform(X)

print(housing_extra_attribs.shape)
print(housing.shape)

# Convert back to data frame -- My way
new_columns = housing.columns.append(
    pd.Index(['rooms_per_household', 'population_per_household'])
)
new_columns
housing_extra_attribs_df = pd.DataFrame(housing_extra_attribs, columns=new_columns)
housing_extra_attribs_df.head()

"""
Explanation: Custom Transformers
End of explanation
"""


housing.describe()

housing.total_rooms.describe()

from sklearn.preprocessing import MinMaxScaler

scalar = MinMaxScaler()
scalar.fit(housing["total_rooms"].values.reshape(-1, 1))
pd.DataFrame(scalar.transform(housing["total_rooms"].values.reshape(-1, 1)), columns=["total_rooms"])["total_rooms"].describe()

from sklearn.preprocessing import StandardScaler

scalar = StandardScaler()
scalar.fit(housing["total_rooms"].values.reshape(-1, 1))
pd.DataFrame(scalar.transform(housing["total_rooms"].values.reshape(-1, 1)), columns=["total_rooms"])["total_rooms"].describe()

"""
Explanation: 2.5.4 Feature Scaling
End of explanation
"""


from sklearn.pipeline import Pipeline

num_pipeline = Pipeline([
    ('imputer', SimpleImputer(strategy="median")),
    ('attr_adder', CombinedAttributesAdder()),
    ('std_scaler', StandardScaler())
])

# I want to verify the pipelined version
# doest the same thing as the separated steps

num_pipeline_stage1 = Pipeline([
    ('imputer', SimpleImputer(strategy="median")),
])

X_pipeline = num_pipeline_stage1.fit_transform(housing_num)
X = imputer.transform(housing_num)
X_pipeline
np.array_equal(X, X_pipeline)

num_pipeline_stage2 = Pipeline([
    ('imputer', SimpleImputer(strategy="median")),
    ('attr_adder', CombinedAttributesAdder()),
])

Y = attr_adder.fit_transform(X)
Y_pipeline = num_pipeline_stage2.fit_transform(housing_num)
np.array_equal(Y, Y_pipeline)

num_pipeline_stage3 = Pipeline([
    ('imputer', SimpleImputer(strategy="median")),
    ('attr_adder', CombinedAttributesAdder()),
    ('std_scaler', StandardScaler())
])

Z = scalar.fit_transform(Y)
Z.std(), Z.mean()
Z_pipeline = num_pipeline_stage3.fit_transform(housing_num)
np.array_equal(Z, Z_pipeline)

from sklearn.base import BaseEstimator, TransformerMixin

class DataFrameSelector(BaseEstimator, TransformerMixin):
    def __init__(self, attribute_names):
        self.attribute_names = attribute_names
    def fit(self, X, y=None):
        return self
    def transform(self, X):
        return X[self.attribute_names].values

class CustomizedLabelBinarizer(BaseEstimator, TransformerMixin):
    def __init__(self, sparse_output=False):
        self.encode = LabelBinarizer(sparse_output = sparse_output)
    def fit(self, X, y=None):
        return self.encode.fit(X)
    def transform(self, X):
        return self.encode.transform(X)


num_attribs = list(housing_num)
cat_attribs = ["ocean_proximity"]

num_pipeline = Pipeline([
    ('selector', DataFrameSelector(num_attribs)),
    ('imputer', SimpleImputer(strategy="median")),
    ('attr_adder', CombinedAttributesAdder()),
    ('std_scaler', StandardScaler()),
]
)

cat_pipeline = Pipeline([
    ('selector', DataFrameSelector(cat_attribs)),
    ('label_binarizer', CustomizedLabelBinarizer()),
]
)

# LabelBinarizer().fit_transform(DataFrameSelector(cat_attribs).fit_transform(housing))
# num_pipeline.fit_transform(housing)
# cat_pipeline.fit_transform(housing)

from sklearn.pipeline import FeatureUnion

full_pipeline = FeatureUnion(transformer_list=[
    ('num_pipeline', num_pipeline),
    ('cat_pipeline', cat_pipeline),
])

housing_prepared = full_pipeline.fit_transform(housing)
print(housing_prepared.shape)
housing_prepared

"""
Explanation: 2.5.5 Transformation Pipeline
End of explanation
"""


from sklearn.linear_model import LinearRegression

lin_reg = LinearRegression()
lin_reg.fit(housing_prepared, housing_labels)

some_data = housing[:5]
some_data

some_labels = housing_labels[:5]
some_labels

some_data_prepared = full_pipeline.transform(some_data)
some_data_prepared

print(f'Prediction:\t{lin_reg.predict(some_data_prepared)}')
print(f'Lables:\t\t{list(some_labels)}')

from sklearn.metrics import mean_squared_error

housing_prediction = lin_reg.predict(housing_prepared)
lin_mse = mean_squared_error(housing_prediction, housing_labels)
lin_rmse = np.sqrt(lin_mse)
lin_rmse

"""
Explanation: 2.6.1 Training and Evaluating on the Training Set
End of explanation
"""


from sklearn.tree import DecisionTreeRegressor

tree_reg = DecisionTreeRegressor()
tree_reg.fit(housing_prepared, housing_labels)
tree_predictions = tree_reg.predict(housing_prepared)
tree_mse = mean_squared_error(tree_predictions, housing_labels)
tree_rmse = np.sqrt(tree_mse)
tree_rmse

"""
Explanation: Tree model
End of explanation
"""


from sklearn.model_selection import cross_val_score
scores = cross_val_score(tree_reg, housing_prepared, housing_labels, scoring="neg_mean_squared_error", cv=10)

rmse_scores = np.sqrt(-scores)
rmse_scores

def display_scores(scores):
    print(f'Scores: {scores}')
    print(f'Mean: {scores.mean()}')
    print(f'STD: {scores.std()}')

display_scores(rmse_scores)

"""
Explanation: 2.6.2 Better Evaluation Using Cross-Validation
End of explanation
"""


from sklearn.ensemble import RandomForestRegressor

forest_reg = RandomForestRegressor()
forest_reg.fit(housing_prepared, housing_labels)
forest_scores = cross_val_score(forest_reg, housing_prepared, housing_labels, scoring="neg_mean_squared_error", cv=10)
forest_rmse_scores = np.sqrt(-forest_scores)
display_scores(forest_rmse_scores)

forest_prediction = forest_reg.predict(housing_prepared)
forest_rmse = np.sqrt(mean_squared_error(forest_prediction, housing_labels))
forest_rmse

"""
Explanation: Random Forest
End of explanation
"""


from sklearn.model_selection import RandomizedSearchCV

from scipy.stats import uniform
from scipy.stats import randint

distributions = {
    'n_estimators': randint(low=3, high=63),
    'max_features': randint(low=2, high=8),
    'bootstrap'   : [True, False],
}

rand_search = RandomizedSearchCV(forest_reg, distributions, random_state=42, cv=5, scoring="neg_mean_squared_error")

rand_search.fit(housing_prepared, housing_labels)

cvres = rand_search.cv_results_
for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
    print(np.sqrt(-mean_score), params)

"""
Explanation: Ex02
Try replacing GridSearchCV with RandomizedSearchCV.
The official document is here
End of explanation
"""


# from sklearn.model_selection import GridSearchCV

# param_grid = [
#     {'n_estimators': [3, 10, 30], 'max_features': [2,4,6,8]},
#     {'bootstrap': [False], 'n_estimators': [3, 10, 30], 'max_features': [2,4,6,8]}
# ]

# forest_reg = RandomForestRegressor()

# grid_search = GridSearchCV(forest_reg, param_grid, cv=5, scoring="neg_mean_squared_error")

# grid_search.fit(housing_prepared, housing_labels)

# grid_search.best_params_

# grid_search.best_estimator_

# cvres = grid_search.cv_results_
# for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):
#     print(np.sqrt(-mean_score), params)

"""
Explanation: These are the results I got, similar to GridSearchCV
python
50865.19350215962 {'bootstrap': True, 'max_features': 5, 'n_estimators': 31}
53821.59623533819 {'bootstrap': True, 'max_features': 4, 'n_estimators': 10}
50296.03902259843 {'bootstrap': True, 'max_features': 6, 'n_estimators': 41}
50762.781955938386 {'bootstrap': False, 'max_features': 4, 'n_estimators': 25}
51365.68817153847 {'bootstrap': True, 'max_features': 4, 'n_estimators': 26}
50555.07511742977 {'bootstrap': True, 'max_features': 5, 'n_estimators': 42}
50750.81686321685 {'bootstrap': False, 'max_features': 4, 'n_estimators': 24}
52003.13807126158 {'bootstrap': True, 'max_features': 3, 'n_estimators': 26}
49766.656263124736 {'bootstrap': False, 'max_features': 7, 'n_estimators': 40}
50347.828040616296 {'bootstrap': False, 'max_features': 5, 'n_estimators': 23}
2.7.1 Grid Search
End of explanation
"""


# feature_importances = grid_search.best_estimator_.feature_importances_
# feature_importances

# extra_attribs = ['rooms_per_hhold', 'pop_per_hhold']

# cat_one_hot_attribs = list(encoder.classes_)
# cat_one_hot_attribs

# attributes = num_attribs + extra_attribs + cat_one_hot_attribs
# attributes, len(attributes)

# sorted(zip(feature_importances, attributes), reverse=True)

"""
Explanation: 2.7.4 Analyze the best models and their errors
End of explanation
"""


# final_model = grid_search.best_estimator_
# X_test = strat_test_set.drop("median_house_value", axis=1)
# y_test = strat_test_set.median_house_value.copy()
# X_test_prepared = full_pipeline.transform(X_test)

# final_predictions = final_model.predict(X_test_prepared)
# final_mse = mean_squared_error(final_predictions, y_test)
# final_rmse = np.sqrt(final_mse)
# final_rmse

"""
Explanation: 2.7.5 Evaluate Your System on the Test Set
End of explanation
"""


