# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from matplotlib import pyplot as plt
%matplotlib inline
import matplotlib
matplotlib.rcParams["figure.figsize"] = (20,10)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
df1=pd.read_csv("/kaggle/input/bengaluru-house-price-data/Bengaluru_House_Data.csv")
df1.shape
df1.head()

df1.info()
df1.groupby("area_type")["area_type"].agg("count")
df2=df1.drop(["area_type","society","balcony","availability"],axis="columns")
df2.head()
df2.isnull().sum()
df3=df2.dropna()
df3.isnull().sum()
df3.shape
df3["size"].unique()
df3['bhk']=df3['size'].apply(lambda x: int(x.split(' ')[0]))
#df3.head()
df3['bhk'].unique()
df3[df3.bhk>20]
df3.total_sqft.unique()
#to see variations are float or not.
def is_float(x):
    try:
        float(x)
    except:
        return False
    return True
        
# '~'' is for negate operation 
df3[~df3['total_sqft'].apply(is_float)].head(12)
#creating a function to take the average of the range of total_sqft
def sqft_to_num(x):
    num=x.split('-')
    if len(num)==2:
        return (float(num[0])+float(num[1]))/2
    try:
        return float(x)
    except:
        return None
    
df4=df3.copy()
df4["total_sqft"] = df4["total_sqft"].apply(sqft_to_num)
df4.head()

import seaborn as sns
sns.histplot(df4["total_sqft"])
sns.boxplot(df4["total_sqft"])
df4["total_sqft"].skew()
df4.loc[30]
df4.head()
df5=df4.copy()
#creating a feature(feature engineering) which can be helpful for 
#outlier detection and removal in the later stage
#price*100000(price is in lakh rupees)/totalsqft
df5['price_per_sqft'] =df5['price']*100000/df5['total_sqft']
df5.head()

#location is categorical data. there is 1304 data which is big.
#Usually to handle text data we convert it into dummy columns using one hot encoding
len(df5.location.unique())
df5.location = df5.location.apply(lambda x: x.strip())
location_stats = df5.groupby('location')['location'].agg('count').sort_values(ascending=False)
location_stats
len(location_stats[location_stats<=10])
location_stats_below_10 = location_stats[location_stats<=10]
location_stats_below_10
len(df5.location.unique())
df5.location = df5.location.apply(lambda x: 'other' if x in location_stats_below_10 else x)
len(df5.location.unique())
df5.head(10)
"""
OUTLIER REMOVAL
-
1020 sqft house with 6 bedroom will be an error

"""
df5[df5.total_sqft/df5.bhk<300].head()
df5.shape
df6 = df5[~(df5.total_sqft/df5.bhk<300)]
df6.shape
"""
##### 
"""
df6.price_per_sqft.describe()
# 267 price per sqft in banglore is too low and very unlikely.
#so we have to remove these.We will use standard deviation
"""
here taking dataframe as an input and grouping them by location first, and per location this sub dataframe for which mean m and standard deviation st. Them filtering all the points which are beyond the standard deviation.
"""
def remove_PricePerSqft_outliers(df):
    df_out = pd.DataFrame()
    for key, subdf in df.groupby('location'):
        m = np.mean(subdf.price_per_sqft)
        st = np.std(subdf.price_per_sqft)
        reduced_df = subdf[(subdf.price_per_sqft>(m-st)) & (subdf.price_per_sqft<=(m+st))]
        df_out = pd.concat([df_out,reduced_df],ignore_index=True)
    return df_out 
df7 = remove_PricePerSqft_outliers(df6)
df7.shape
def plt_scatter_chart(df,location):
    bhk2 = df[(df.location==location) & (df.bhk==2)]
    bhk3 = df[(df.location==location) & (df.bhk==3)]
    matplotlib.rcParams['figure.figsize'] = (15,10)
    plt.scatter(bhk2.total_sqft,bhk2.price_per_sqft,color='blue',label='2 BHK', s=50)
    plt.scatter(bhk3.total_sqft,bhk3.price_per_sqft,marker='+',color='green',label='3 BHK', s=50)
    plt.xlabel('Total Square Feet Area')
    plt.ylabel("Price per Square Feet")
    plt.title(location)
    plt.legend()
    
plt_scatter_chart(df7,'Rajaji Nagar')    
    
    
    
    
    
def plt_scatter_chart(df,location):
    bhk2 = df[(df.location==location) & (df.bhk==2)]
    bhk3 = df[(df.location==location) & (df.bhk==3)]
    matplotlib.rcParams['figure.figsize'] = (15,10)
    plt.scatter(bhk2.total_sqft,bhk2.price,color='blue',label='2 BHK', s=50)
    plt.scatter(bhk3.total_sqft,bhk3.price,marker='+',color='green',label='3 BHK', s=50)
    plt.xlabel('Total Square Feet Area')
    plt.ylabel("Price")
    plt.title(location)
    plt.legend()
    
plt_scatter_chart(df7,'Hebbal')  
def remove_bhk_outlier(df):
    exclude_indices = np.array([])
    for location,location_df in df.groupby('location'):
        bhk_stats = {}
        for bhk, bhk_df in location_df.groupby('bhk'):
            bhk_stats[bhk] = {
                'mean' : np.mean(bhk_df.price_per_sqft),
                'std' : np.std(bhk_df.price_per_sqft),
                'count' : bhk_df.shape[0]
            }
        for bhk, bhk_df in location_df.groupby('bhk'):
            stats = bhk_stats.get(bhk-1)
            if stats and stats['count']>5:
                exclude_indices = np.append(exclude_indices,bhk_df[bhk_df.price_per_sqft<(stats['mean'])].index.values)
    return df.drop(exclude_indices,axis='index')
df8 = remove_bhk_outlier(df7)
df8.shape
plt_scatter_chart(df8,"Hebbal")

import matplotlib
matplotlib.rcParams['figure.figsize'] = (20,10)
plt.hist(df8.price_per_sqft,rwidth=0.8)
plt.xlabel("Price per Square Feet")
plt.ylabel("Count")
import seaborn as sns
sns.histplot(df8["price_per_sqft"])
df8['bath'].unique()
df8[df8['bath']>10]
sns.histplot(df8["bath"])
df8[df8['bath']>df8['bhk']+2]
df9=df8[df8['bath']<df8['bhk']+2]
df9.shape
df10=df9.drop(['size','price_per_sqft'],axis='columns')
df10.head()
#we have to convert text data to numerical ,here we use 'one hot encoding'
dummies=pd.get_dummies(df10['location'])
dummies.head(3)
df11=pd.concat([df10,dummies.drop('other',axis='columns')],axis='columns')
df11.head(3)
df12=df11.drop('location',axis='columns')
df12.head(3)
X=df12.drop('price',axis='columns')
X.head()
y=df12['price']
y.head()
from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.2,random_state=0)
from sklearn.linear_model import LinearRegression
lr_clf = LinearRegression()
lr_clf.fit(X_train,y_train)
lr_clf.score(X_test,y_test)
from sklearn.model_selection import ShuffleSplit
from sklearn.model_selection import cross_val_score

cv=ShuffleSplit(n_splits = 5, test_size=0.2, random_state=0)
cross_val_score(LinearRegression(),X,y,cv=cv)
from sklearn.model_selection import GridSearchCV
from sklearn.linear_model import Lasso
from sklearn.tree import DecisionTreeRegressor

def find_best_model_using_gridsearchcv(X,y):
    algos = {
        'linear_regression' : {
            'model':LinearRegression(),
            'params': {
                'normalize':[True, False]
            }
        },
        'lasso' : {
            'model':Lasso(),
            'params': {
                'alpha':[1,2],
                'selection':['random','cyclic']
            }
        },
        'decision tree': {
            'model':DecisionTreeRegressor(),
            'params': {
                'criterion' : ['mse','friedman_mse'],
                'splitter' : ['best','random']
            }
        }
        
    }
    scores=[]
    cv = ShuffleSplit(n_splits=5,test_size=0.2,random_state=0)
    for algo_name, config in algos.items():
        gs = GridSearchCV(config['model'],config['params'],cv=cv,return_train_score=False)
        gs.fit(X,y)
        scores.append({
            'model':algo_name,
            'best_score':gs.best_score_,
            'best_params' : gs.best_params_
        })
    
    return pd.DataFrame(scores,columns=['model','best_score','best_params'])
        
find_best_model_using_gridsearchcv(X,y)
X.columns
#np.where(X.columns=='2nd Phase Judicial Layout')[0][0]
def predict_price(location,sqft,bath,bhk):
    loc_index = np.where(X.columns==location)[0][0]
    
    x = np.zeros(len(X.columns))
    x[0] = sqft
    x[1] = bath
    x[2] = bhk
    if loc_index >= 0:
        x[loc_index] = 1
    return lr_clf.predict([x])[0]

predict_price('1st Phase JP Nagar',1000,3,3)
predict_price('1st Phase JP Nagar',1000,2,3)
predict_price('1st Phase JP Nagar', 1000, 2,2)
predict_price('Indira Nagar',1000,2,3)
predict_price('Indira Nagar', 1000, 2,2)
predict_price('Indira Nagar',1000,3,3)