tmp = train.isnull().sum()

# get top 10 results
tmp.sort_values(ascending=False).head(10).plot(kind='bar', figsize=(8,8))

"""
Explanation: We have 1460 observations, and 81 columns.  There quite a few strings too, which we would have to encode later on. 
Let's have a look at the number of missing values.
End of explanation
"""


drop_cols = ['PoolQC','MiscFeature','Alley','Fence']

# write custom transformer to drop these 4 cols for use in Pipeline later
from sklearn.base import BaseEstimator, TransformerMixin

def DropColumnsTransform(BaseEstimator, TransformerMixin):
    def __init__(self, attribs_drop):
        self.attribs_drop = attribs_drop
    def fit(self, X):
        return self
    def transform(self, X):
        return X.drop(self.attribs_drop, axis=1).values

# look at categorical data
train_cat = train.select_dtypes(include=['object'])
train_cat.shape
# use this to impute missing values as "?"
train_cat = train_cat.fillna("?")

print("43/%d or %.2f%% of columns are categorical" % (train.shape[1], 43/train.shape[1]*100))

from sklearn.preprocessing import LabelBinarizer, Imputer

LabelBinarizer = LabelBinarizer()
# loop to apply LB to each column individually, then combine them back together
list_cols = []
for col in list(train_cat.columns):
    x = train_cat[col].values
    x_trans = LabelBinarizer.fit_transform(x)
    list_cols.append(x_trans)
train_cat_transformed = np.concatenate(list_cols,axis=1)
train_cat_transformed

# numerical data now

Imp = Imputer(strategy="median")
train_num = train.select_dtypes(include=['number'])
train_num.shape

# look at correlation
cor = train_num.corr()
f = plt.figure(figsize=(15,15))
sns.heatmap(cor, cmap='plasma')

"""
Explanation: One way to handle this is to drop the first 4, given that almost all observations are missing.
End of explanation
"""


tmp = cor['SalePrice'].sort_values(ascending=False)
tmp[1:11].plot(kind='bar', figsize=(8,8))

# we will have to remove SalePrice before imputing
train_num_wsp = train_num.drop('SalePrice',axis=1)
train_num_tr = Imp.fit_transform(train_num_wsp)
train_num_tr

X = np.concatenate([train_num_tr, train_cat_transformed],axis=1)
y = train_num['SalePrice'].values
print("Shape of X:", X.shape)
print("Shape of y:", y.shape)

"""
Explanation: Many features (e.g. LotArea, GarageCars) are indeed correlated highly with SalePrice.
End of explanation
"""


from sklearn.model_selection import train_test_split
# split into 10% for validation at end
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.1)

# Linear Regression
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_squared_error as mse

linreg = LinearRegression()
scores = cross_val_score(linreg, X_train, y_train, scoring="neg_mean_squared_error", cv=10, verbose=1)

def printscorespretty(scores):
    sc = np.sqrt(-scores)
    print("Scores:", sc)
    print("Mean:", np.mean(sc))
    print("SD:", np.sqrt(np.var(sc)))

printscorespretty(scores)

#Decision Tree Regressor
from sklearn.tree import DecisionTreeRegressor

dtr = DecisionTreeRegressor()
scores = cross_val_score(dtr, X_train, y_train, scoring="neg_mean_squared_error", cv=10, verbose=1)
printscorespretty(scores)

"""
Explanation: Fit Models

Linear Regression
RandomForest
End of explanation
"""


from sklearn.ensemble import RandomForestRegressor

rf = RandomForestRegressor()
scores = cross_val_score(rf, X_train, y_train, scoring="neg_mean_squared_error", cv=10, verbose=1)
printscorespretty(scores)

"""
Explanation: DecisionTree Regressor is performing much better than Linear Regression here, perhaps capturing some non-linearity in data.
End of explanation
"""


# XGBoost
from xgboost import XGBRegressor

XGB = XGBRegressor()
scores = cross_val_score(XGB, X_train, y_train, scoring="neg_mean_squared_error", cv=10, verbose=1)
printscorespretty(scores)

"""
Explanation: Best performance thus far is from RF.
End of explanation
"""


