# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in 

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.
# For example, runninga this (by clicking run or pressing Shift+Enter) will list the files in the input directory

from subprocess import check_output
print(check_output(["ls", "../input"]).decode("utf8"))
"""
## Importing the required libraries for data analysis and visualization
"""
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import ensemble, tree, linear_model
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import r2_score, mean_squared_error
from sklearn.utils import shuffle

%matplotlib inline
import warnings
warnings.filterwarnings('ignore')
"""
Reading the training and test data
"""
train = pd.read_csv('../input/train.csv')
test = pd.read_csv('../input/test.csv')

train.describe()
train.head(3)
test.head(3)
"""
We can see there are so many NAN  values in both train and test sets which needs to treated well.

"""
train.shape,test.shape
"""
Check for duplicate ids
"""
#check for dupes for Id
idsUnique = len(set(train.Id))
idsTotal = train.shape[0]
idsdupe = idsTotal - idsUnique
print("No. of duplicate IDs present: " + str(idsdupe))
#drop id col
train.drop(['Id'],axis =1,inplace=True)
"""
## **Visualization of features coorelation with each other**
"""
#correlation matrix
corrmat = train.corr()
f, ax = plt.subplots(figsize=(20, 9))
sns.heatmap(corrmat, vmax=.8, annot=True);
"""
### Most correlated features
"""
corrmat = train.corr()
top_corr_features = corrmat.index[abs(corrmat["SalePrice"])>0.5]
plt.figure(figsize=(10,10))
g = sns.heatmap(train[top_corr_features].corr(),annot=True,cmap="RdYlGn")
"""
**Most of the features are correlated with each other like Garage Cars and Garage Area, isnt it?**
- OverallQual is highly correlated with target feature SalePrice 0.79 can you see. we'll see how it effected the saleprice in below graph.
"""
sns.barplot(train.OverallQual,train.SalePrice)
"""
Here we can see how each feature is correlated  with SalePrice.

"""
sns.set()
cols = ['SalePrice', 'OverallQual', 'GrLivArea', 'GarageCars', 'TotalBsmtSF', 'FullBath', 'YearBuilt']
sns.pairplot(train[cols], size = 2.5)
plt.show();
"""
**SalePrice** is the variable we need to predict. So let's do some analysis on this variable first.

We will use a distribution plot for the SalePrice along with its mean (mu) and standard deviation (sigma)
"""
from scipy import stats
from scipy.stats import norm, skew #for some statistics

sns.distplot(train['SalePrice'] , fit=norm);

# Get the fitted parameters used by the function
(mu, sigma) = norm.fit(train['SalePrice'])
print( '\n mu = {:.2f} and sigma = {:.2f}\n'.format(mu, sigma))
plt.legend(['Normal dist. ($\mu=$ {:.2f} and $\sigma=$ {:.2f} )'.format(mu, sigma)],
            loc='best')
plt.ylabel('Frequency')
plt.title('SalePrice distribution')

fig = plt.figure()
res = stats.probplot(train['SalePrice'], plot=plt)
plt.show()
"""
Since, the SalePrice distribution is skewed, we can use the log function to get rid of its skewness
"""
train.SalePrice = np.log1p(train.SalePrice )
y = train.SalePrice
"""
### Pre processing
"""
plt.scatter(y =train.SalePrice,x = train.GrLivArea,c = 'black')
plt.show()
#we can see the outlier in the below image
"""
**Checking how many values are missing for each feature in the training data**
"""
train_nas = train.isnull().sum()
train_nas = train_nas[train_nas>0]
train_nas.sort_values(ascending=False)
"""
**Checking how many values are missing for each feature in the test data**
"""
test_nas = test.isnull().sum()
test_nas = test_nas[test_nas>0]
test_nas.sort_values(ascending = False)
print("Find most important features relative to target i.e SalePrice")
corr = train.corr()
corr.sort_values(["SalePrice"], ascending = False, inplace = True)
print(corr.SalePrice)
#this you can see at the time of heatmap also.
"""
**Differentiate numerical features (excluding the SalePrice) and categorical features**
"""
categorical_features = train.select_dtypes(include=['object']).columns
categorical_features
numerical_features = train.select_dtypes(exclude = ["object"]).columns
numerical_features
categorical_features = train.select_dtypes(include = ["object"]).columns
numerical_features = train.select_dtypes(exclude = ["object"]).columns
numerical_features = numerical_features.drop("SalePrice")
print("Numerical features : " + str(len(numerical_features)))
print("Categorical features : " + str(len(categorical_features)))
train_num = train[numerical_features]
train_cat = train[categorical_features]
"""
**We will use median() to fill NaN (i.e missing) values**
"""
# Handle remaining missing values for numerical features by using median as replacement
print("NAs for numerical features in train : " + str(train_num.isnull().values.sum()))
train_num = train_num.fillna(train_num.median())
print("Remaining NAs for numerical features in train : " + str(train_num.isnull().values.sum()))
"""
**Check for skewness in the dataset features**
"""
from scipy.stats import skew 
skewness = train_num.apply(lambda x: skew(x))
skewness.sort_values(ascending=False)
"""
**We will consider those features skewed which will be having a positive or negative skewness greater than 0.5**
"""
skewness = skewness[abs(skewness)>0.5]
skew_features = train[skewness.index]
skew_features.columns
"""
**We can treat skewness of a feature with the help of log transformation, so we'll apply the same here.**
"""
skew_features = np.log1p(skew_features)
"""
### Create dummy features for categorical values via one-hot encoding
"""
print("Original train dataset shape: " + str(train_cat.shape))
train_cat = pd.get_dummies(train_cat)
print("New train dataset shape after one-hot encoding: " + str(train_cat.shape))
train_cat.head()
"""
**Checking for missing values in categorical features**
"""
str(train_cat.isnull().values.sum())
"""
**Modeling**
"""
import pandas as pd
import numpy as np
from sklearn.model_selection import cross_val_score, train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LinearRegression, RidgeCV, LassoCV, ElasticNetCV
from sklearn.metrics import mean_squared_error, make_scorer
import matplotlib.pyplot as plt
import seaborn as sns
"""
Earlier we split the train set into categorical and numerical features.  

Now after transformation(preprocessing) we'll join them to get the whole train set back.
"""
train = pd.concat([train_cat,train_num],axis=1)
train.shape
"""
### Split the data to train the model 
"""
X_train,X_test,y_train,y_test = train_test_split(train,y,test_size = 0.3,random_state= 0)
X_train.shape,X_test.shape,y_train.shape,y_test.shape
X_train.head(3)
"""
### Defining  cross_val_score function for both train and test sets separately

**We will use Root Mean Square Error to get an estimate of the model's performance.**
"""
n_folds = 5
from sklearn.metrics import make_scorer
from sklearn.model_selection import KFold
scorer = make_scorer(mean_squared_error,greater_is_better = False)
def rmse_CV_train(model):
    kf = KFold(n_folds,shuffle=True,random_state=42).get_n_splits(train.values)
    rmse = np.sqrt(-cross_val_score(model,X_train,y_train,scoring ="neg_mean_squared_error",cv=kf))
    return (rmse)
def rmse_CV_test(model):
    kf = KFold(n_folds,shuffle=True,random_state=42).get_n_splits(train.values)
    rmse = np.sqrt(-cross_val_score(model,X_test,y_test,scoring ="neg_mean_squared_error",cv=kf))
    return (rmse)
"""
###  Linear Regression model without Regularization
"""
lr = LinearRegression()
lr.fit(X_train,y_train)
test_pre = lr.predict(X_test)
train_pre = lr.predict(X_train)
print('rmse on train',rmse_CV_train(lr).mean())
print('rmse on test',rmse_CV_test(lr).mean())

"""
It might sound confusing  what is validation data? 

We are still working on train data only, which split was into x_train,x_test.  Here x_test is the validation set we call because 

We are checking how model is performing on our own data(x_test).
"""
# Plot predictions - Real values
plt.scatter(train_pre, y_train, c = "blue",  label = "Training data")
plt.scatter(test_pre, y_test, c = "black",  label = "Validation data")
plt.title("Linear regression")
plt.xlabel("Predicted values")
plt.ylabel("Real values")
plt.legend(loc = "upper left")
plt.plot([10.5, 13.5], [10.5, 13.5], c = "red")
plt.show()
"""
### Regularization is a very useful method to filter out noise from data, and eventually prevent overfitting. 
### The concept behind regularization is to introduce additional information (bias) to penalize extreme parameter weights. 
"""
"""
**Here, we will be using Ridge Regularization to improve upon the existing Linear regression model**

**We will pass different values of alpha to the RigeCV() and it will try to find the most optimum value of alpha depending on the model performance.**
"""
ridge = RidgeCV(alphas = [0.01, 0.03, 0.06, 0.1, 0.3, 0.6, 1, 3, 6, 10, 30, 60])
ridge.fit(X_train,y_train)
alpha = ridge.alpha_
print('best alpha',alpha)

print("Try again for more precision with alphas centered around " + str(alpha))
ridge = RidgeCV(alphas = [alpha * .6, alpha * .65, alpha * .7, alpha * .75, alpha * .8, alpha * .85, 
                          alpha * .9, alpha * .95, alpha, alpha * 1.05, alpha * 1.1, alpha * 1.15,
                          alpha * 1.25, alpha * 1.3, alpha * 1.35, alpha * 1.4],cv = 5)
ridge.fit(X_train, y_train)
alpha = ridge.alpha_
print("Best alpha :", alpha)
print("Ridge RMSE on Training set :", rmse_CV_train(ridge).mean())
print("Ridge RMSE on Test set :", rmse_CV_test(ridge).mean())
y_train_rdg = ridge.predict(X_train)
y_test_rdg = ridge.predict(X_test)
"""
**Her, we can have a look at how many features our model seleced and how many it eliminated**
"""
coef = pd.Series(ridge.coef_, index = X_train.columns)

print("Ridge picked " + str(sum(coef != 0)) + " variables and eliminated the other " +  str(sum(coef == 0)) + " variables")

print('rmse on train',rmse_CV_train(ridge).mean())
print('rmse on test',rmse_CV_test(ridge).mean())
# Plot predictions - Real values
plt.scatter(y_train_rdg, y_train, c = "blue",  label = "Training data")
plt.scatter(y_test_rdg, y_test, c = "black",  label = "Validation data")
plt.title("Linear regression with Ridge regularization")
plt.xlabel("Predicted values")
plt.ylabel("Real values")
plt.legend(loc = "upper left")
plt.plot([10.5, 13.5], [10.5, 13.5], c = "red")
plt.show()
"""
### Save the finalized model
"""
import pickle
pickle.dump(ridge, open('house_price_model', 'wb'))
"""
### Loading the model
"""
model = pickle.load(open('house_price_model', 'rb'))
"""
# Conclusion
"""
"""
**The final Linear Regression model trained with Ridge Regularization gave an average RMSE score of about 0.17 on the test set, showing that the real and the predicted values were actually pretty close!**
"""