"""
# Explanation of this notebook
- This code is a simple solution for those who have taken the Intermediate ML courses in Kaggle.
- This code is not for the explanation of the lecture, but for creating a file for submission with the techniques learned in the course.
- The following steps were used to analyze the data.
1. Create a data set that will be used in this analyses
2. Prepare to use pipeline for analysis
3. Finally, fit and predict
- This code is very simple, but it gave good results(Top5%!).
- If you find any mistakes or problems with this code, please let me know in the comments!
- Lastly, I'd love it if you'd give me a like!
"""
# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
"""
# 0.Import libraries and CSV files
"""
# Import helpful libraries
import pandas as pd
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import train_test_split
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import OrdinalEncoder
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from xgboost import XGBRegressor

#Importing CSV files
X_train_full=pd.read_csv("/kaggle/input/home-data-for-ml-course/train.csv")
X_test_full=pd.read_csv("/kaggle/input/home-data-for-ml-course/test.csv")

"""
# 1.Create a data set that will be used in this analyses
"""
#Check if there are any rows with missing target values, and if so, delete them
#missing_val_count_by_column = (X_train_full.isnull().sum())
#print(missing_val_count_by_column[missing_val_count_by_column > 0])
#X_train_full.dropna(axis=0, subset=['SalePrice'], inplace=True)

#Create y_train,X_train_full
y_train=X_train_full.SalePrice
X_train_full.drop(['SalePrice'], axis=1, inplace=True)

#Separate numeric data columns from categorical data columns
X_train_full_num = X_train_full.select_dtypes(exclude=['object'])
X_train_full_cat = X_train_full.select_dtypes(include=['object'])
X_test_full_num = X_test_full.select_dtypes(exclude=['object'])
X_test_full_cat = X_test_full.select_dtypes(include=['object'])

#Remove columns with missing values in the category data
cols_with_missing1 = [col for col in X_train_full_cat.columns if X_train_full_cat[col].isnull().any()] 
cols_with_missing2 = [col for col in X_test_full_cat.columns if X_test_full_cat[col].isnull().any()] 
bad_cat_list=list(set(cols_with_missing1)|set(cols_with_missing2))
all_cat_list = [col for col in X_train_full_cat.columns]
nomissing_cat_list=list(set(all_cat_list)-set(bad_cat_list))
X_train_full_cat_nomissing=X_train_full_cat[nomissing_cat_list]
X_test_full_cat_nomissing=X_test_full_cat[nomissing_cat_list]

#Finally, we will create a data set that will be used in this analyses.
X_train_full_final=pd.concat([X_train_full_num,X_train_full_cat_nomissing],axis=1)
X_test_full_final=pd.concat([X_test_full_num,X_test_full_cat_nomissing],axis=1)


"""
# 2.Prepare to use pipeline for analysis
"""
# Preprocessing for numerical data
numerical_transformer = SimpleImputer(strategy='mean')

# Preprocessing for categorical data
categorical_transformer = Pipeline(steps=[('ordinal',OrdinalEncoder())])

# Bundle preprocessing for numerical and categorical data
numerical_cols = [cname for cname in X_train_full_num.columns if X_train_full[cname].dtype in ['int64', 'float64']]
categorical_cols = [cname for cname in X_train_full_cat_nomissing.columns if X_train_full[cname].dtype == "object"]
preprocessor = ColumnTransformer(transformers=[('num', numerical_transformer, numerical_cols),('cat', categorical_transformer, categorical_cols)])

# Define model
#This model has already been selected as the best one in the first intermediate course
#So, using cross-validation may give better results (that's my future to-do)
model = XGBRegressor(n_estimators=1000,learning_rate=0.05,n_jobs=4,random_state=0)

# Bundle preprocessing and modeling code in a pipeline
clf = Pipeline(steps=[('preprocessor', preprocessor),('model', model)])

"""
# 3. Finally, fit and predict
"""
clf.fit(X_train_full_final,y_train)
preds=clf.predict(X_test_full_final)
output = pd.DataFrame({'Id': X_test_full.Id,'SalePrice': preds})
output.to_csv('submission.csv', index=False)