"""
## Hello, my name is George Lolaev and I will be satisfied, if you check my notebook
I was trying to make it beginner friendly, cause I am also beginner in DataScience.

If there is anything to make better in my code, please tell me about it
"""
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
# for cross-validation
from sklearn.model_selection import cross_val_score

# for preprocessing categorical features
from sklearn.preprocessing import OneHotEncoder
# for preprocessing numerical features
from sklearn.impute import SimpleImputer

# the model I'm gonna use
from xgboost import XGBRegressor

# package with statistical functions, it will be helpfull in searching outliers in our train dataset
from scipy import stats
# very useful function for fast checking differently made train-datasets
# I recommend you to save it
# Here it is not as helpful as it be in other cases, but anyway it makes my work faster
def score_dataset(X, y, model = XGBRegressor(random_state=0)):
    score = cross_val_score(model, X, y, scoring = 'neg_mean_squared_error')
    score = -1* score.mean()
    score = np.sqrt(score)
    return score

# also very cool function, use it drop outliers in numerical features
# the idea here is linked with statistal termin z-score
def drop_outliers(col, zscore= 3):
    return col[np.abs(stats.zscore(col))<zscore]
# reading our csv's
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv', index_col='Id')
test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv', index_col='Id')

# that cols are almost empty, so they are useless
cols_to_drop = [ 'Alley', 'PoolQC', 'Fence', 'MiscFeature']
train = train.drop(cols_to_drop, axis=1)

# split into features dataframe and target column
X = train.drop(['SalePrice'], axis=1)
y = train.SalePrice

# the code will be much clearer if we defince categorical and numerical columns explicitly 
cat_cols = X.select_dtypes(include='object').columns
num_cols = X.select_dtypes(exclude='object').columns
# cat-features
cat_X = X[cat_cols]
# num-features
num_X = X[num_cols]

# create onehotencoder transformer to process cat-features
# pay attention to attribute named 'handle_unknown'. It's important if we don't want to get errors with working with test dataset
onehot_encoder = OneHotEncoder(sparse=False, handle_unknown='ignore')
cat_X_prep = onehot_encoder.fit_transform(cat_X)

# filling nans with mean values for every column
for col in num_X:
    num_X[col].fillna(num_X[col].mean(), inplace=True)

# assign refined features to old columns
for col in num_X:
    num_X.loc[:,col] = drop_outliers(num_X[col])
# outliers now are nans
# and we just use dropna method
num_X = num_X.dropna(axis=0)

# concatenate categoricals and numericals
X_prep = np.concatenate((num_X, cat_X_prep[num_X.index]), axis=1)

# our dataset have become smaller and we must adapt target-vector to it
y_prep = y[num_X.index]
# just for tracking changes
# In previous generations of that code the score was around 40'000
# now it's much smaller :)
score_dataset(X_prep,y_prep, XGBRegressor(random_state=0,learning_rate=0.1,n_jobs=-1
                                             ))
"""
## Let's now work with the test data
"""
"""
Prerprocess acordingly the test dataset, but don't drop outliers
"""
X_test = test.drop(columns=cols_to_drop, axis=1)

cat_X_test= X_test[cat_cols]
num_X_test = X_test[num_cols]

# be careful! with test-dataset you should use only 'trasform' method, without fitting
cat_X_test_prep = onehot_encoder.transform(cat_X_test)



for col in num_X_test:
    num_X_test[col].fillna(num_X_test[col].mean(), inplace=True)



X_test_prep = np.concatenate((num_X_test, cat_X_test_prep), axis=1)
# learning_rate defines the speed of learning of our model
# n_jobs is the number of cores of your processor that are used. value '-1' means using all cores
my_model = XGBRegressor(learning_rate=0.1, n_jobs=-1)
my_model.fit(X_prep,y_prep)

# making prediction
y_predicted = my_model.predict(X_test_prep)
# I concatenate the id's of test dataset and predicted values into single dataframe
result = pd.DataFrame({'Id':test.index, 'SalePrice':y_predicted})
# and then just convert it to csv file
result.to_csv('./result.csv', index=False)
"""
**Thanks for watching :)
Now let's do that by yourself !**
"""