"""
## Import data and create data sets
"""
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

target = 'SalePrice'
X_train = pd.read_csv('../input/home-data-for-ml-course/train.csv', index_col = 'Id')
X_test = pd.read_csv('../input/home-data-for-ml-course/test.csv', index_col = 'Id')

# Remove rows with missing target data and drop target column
X_train.dropna(axis = 0, subset = [target], inplace = True)
y_train = X_train[target]
X_train.drop([target], axis = 1, inplace = True)
"""
## Choose strategies for handling columns with categorical data
Here, we process categorical values on a column-by-column basis. For each categorical column, we have one of three different approaches (thanks [intermediate machine learning course](https://www.kaggle.com/alexisbcook/categorical-variables#Three-Approaches)!).

### Drop the column
This is done if the cardinality of the column is too high. The definition of "too high" is completely arbitrary for now. This can be improved by running different models with different cardinality thresholds and seeing which one performs the best.

### Ordinal encoding
Do this for values that have a clear ordering. This pretty much only involves columns with "quality" or "condition" ratings.

### One-hot encoding
Do this for the remaining categorical columns.
"""
from matplotlib import pyplot as plt
import numpy as np

# Get columns with categorical data
cols_by_is_object = (X_train.dtypes == 'object')
cols_by_is_object = cols_by_is_object.where(lambda is_object: is_object)
cat_cols = list(cols_by_is_object.dropna().index)

cat_data = X_train[cat_cols]

# Get columns with high cardinality.
cardinality_threshold = 10
high_card_cols = [c for c in cat_data.columns if len(cat_data[c].unique()) > cardinality_threshold]
# Drop the ones that have NaNs
cat_cols_to_drop = [c for c in high_card_cols if X_train[c].isnull().any()] + [c for c in high_card_cols if X_test[c].isnull().any()]
if len(cat_cols_to_drop) > 0:
    print(f'Dropping the columns {cat_cols_to_drop} due to cardinalities that are above {cardinality_threshold}\n')
    X_train.drop(columns = cat_cols_to_drop, inplace = True)
    X_test.drop(columns = cat_cols_to_drop, inplace = True)

# Set columns to ordinal encode. Do this by manually going through the column values and deciding if there's an order to them or not.
cat_cols_to_ordinal_encode = [
    'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'HeatingQC', 'CentralAir', 'KitchenQual',
    'FireplaceQu', 'GarageQual', 'GarageCond', 'PoolQC', 'Fence'
]
print(f'Ordinal encoding the columns {cat_cols_to_ordinal_encode}\n')

cat_cols_to_one_hot_encode = list(set(cat_cols) - set(high_card_cols) - set(cat_cols_to_ordinal_encode))
print(f'Remaining columns to one-hot encode: {cat_cols_to_one_hot_encode}')

# Plot for funsies
fig, ax = plt.subplots(figsize = (8, 8))
ax.pie(
    [len(cat_cols_to_drop), len(cat_cols_to_ordinal_encode), len(cat_cols_to_one_hot_encode)],
    autopct = lambda pct: f'{pct:.0f}%',
    labels = ['Dropped', 'Ordinal Encoded', 'One-Hot Encoded'],
    shadow = True,
    explode = (0.03, 0.03, 0.03)
)
ax.set_title('% of columns that are...')
plt.show()
"""
## Set up preprocessors
"""
from sklearn.pipeline import make_pipeline
from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import OneHotEncoder

# Define transformers for numerical data.
number_transformer_mean = SimpleImputer(strategy = 'mean')
number_transformer_median = SimpleImputer(strategy = 'median')
number_transformer_zero = SimpleImputer(strategy = 'constant', fill_value = 0)

# Most null numerical data is set to zero, with some exceptions.
numerical_cols = (X_train.dtypes != 'object')
numerical_cols = numerical_cols.loc[numerical_cols].index.tolist()

num_cols_to_mean = ['MSSubClass', 'OverallQual', 'OverallCond']
num_cols_to_median = ['YearBuilt', 'YearRemodAdd', 'GarageYrBlt']
num_cols_to_zero = list(set(numerical_cols) - set(num_cols_to_median) - set(num_cols_to_mean))

# Define transformers for categorical data.

# Cross validation can cause some column values to appear in ONLY the training or validation set, causing ordinal encoder to mad.
# Pass in all categories beforehand to fix.
ordinal = X_train[cat_cols_to_ordinal_encode].fillna('');
ordinal_cats = [ordinal[c].unique() for c in ordinal.columns]
ordinal_transformer = make_pipeline(
    SimpleImputer(strategy = 'constant', fill_value = ''),
    # Setting unknown_value to np.nan can cause an error when using the test set.
    OrdinalEncoder(handle_unknown = 'use_encoded_value', unknown_value = -1)
)

onehot_transformer = make_pipeline(
    SimpleImputer(strategy = 'constant', fill_value = 'N/A'),
    OneHotEncoder(handle_unknown = 'ignore')
)

preprocessor = ColumnTransformer(transformers = [
    ('num_cols_mean', number_transformer_mean, num_cols_to_mean),
    ('num_cols_median', number_transformer_median, num_cols_to_median),
    ('num_cols_zero', number_transformer_zero, numerical_cols),
    ('ordinal', ordinal_transformer, cat_cols_to_ordinal_encode),
    ('onehot', onehot_transformer, cat_cols_to_one_hot_encode)
])

"""
# Run and test the model
"""
from sklearn.model_selection import train_test_split, cross_val_score
from xgboost import XGBRegressor

num_trees = [100, 300, 500, 800, 1000]
learning_rates = [0.02, 0.03, 0.05, 0.08, 0.13]

# Results in the form [(n_estimators, learning_rate, score)]
results = []
for trees in num_trees:
    for rate in learning_rates:
        pipeline = make_pipeline(
            preprocessor,
            XGBRegressor(n_estimators = trees, learning_rate = rate)
        )

        scores = -1 * cross_val_score(
            pipeline,
            X_train,
            y_train,
            scoring = 'neg_mean_absolute_error'
        )
        results.append((trees, rate, scores.mean()))

        
n_estimators, learning_rate, score = min(results, key = lambda x: x[2])
print(f'Best parameters:\nn_estimators = {n_estimators}\nlearning_rate = {learning_rate}\nMAE: {score}')

# Run using the best model parameters
pipeline = make_pipeline(
    preprocessor,
    XGBRegressor(n_estimators = n_estimators, learning_rate = learning_rate)
)

pipeline.fit(X_train, y_train)
preds = pipeline.predict(X_test)

output = pd.DataFrame({
    'Id': X_test.index,
    'SalePrice': preds
})
output.to_csv('submission.csv', index = False)