# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
df=pd.read_csv('/kaggle/input/cardataset/data.csv')

df
df.columns=df.columns.str.lower().str.replace(' ','_')
df.head()

df.dtypes
df.dtypes[df.dtypes=="object"].index
strings=list(df.dtypes[df.dtypes=="object"].index)
# Loop over the strings list
for i in strings:
    df[i]=df[i].str.lower().str.replace(' ','_')
    
df.head()
df.dtypes
for col in df.columns:
    print(col)
    print(df[col].unique()[:4],'\n')
    print(df[col].nunique())
# lets try determining unique values
# how many they are
import matplotlib.pyplot as plt
import seaborn as sns

%matplotlib inline

sns.histplot(df.msrp, bins=60)
sns.histplot(df.msrp[df.msrp<=100000], bins=50)

price_logs=np.log1p(df.msrp)
price_logs

sns.histplot(price_logs, bins=50)
"""
#### Missing vaues
"""
df.isnull().sum()
n=len(df)

n_val=int(n*0.2)
n_test=int(n*0.2)
#  But this is not sustainable becase n reduces by 2 due to rounding off with fxn int()
# we just subtract
n_train=n-n_val-n_test

# thus they are the same
n, n_val+n_test+n_train
n, n_val,n_test, n_train
df.iloc[:10]
df_train=df.iloc[:n_train]

df_val=df.iloc[n_train:n_train+n_val]

df_test=df.iloc[n_train+n_val:]


idx=np.arange(n)
np.random.seed(2)
np.random.shuffle(idx)
df.iloc[idx[:10]]
df_train= df.iloc[idx[:n_train]]

df_val= df.iloc[idx[n_train:n_train+n_val]]

df_test= df.iloc[idx[n_train+n_val:]]


df_train=df_train.reset_index(drop=True)
df_val=df_val.reset_index(drop=True)
df_test=df_test.reset_index(drop=True)

len(df_train), len(df_val), len(df_test)
"""
Log transformation of our target variable y. Use .values to remain with a numpy array and not a Pandas series
"""
y_train=np.log1p(df_train.msrp.values)
y_val=np.log1p(df_val.msrp.values)
y_test=np.log1p(df_test.msrp.values)
df.head()
del df_train ['msrp']
del df_val ['msrp']
del df_test ['msrp']
len(y_train)
"""
# 2. LINEAR REGRESSION

A model used to solve regression problems. Used to predict number
"""
df_train.iloc[10]
xi=[453, 11, 86]
w0=7.17 #bias
w=[0.01,0.04,0.002]# weights
#  is the linear regression model
def linear_regression(xi):
    
    n=len(xi)
    
    pred=w0
    
    for j in range(n):
        
        pred=pred+w[j] * xi[j]
 #do something
    return pred
np.expm1(linear_regression(xi))
"""
## 2.1 Linear Regression Vector Form
"""
def dot(xi, w):
    n=len(xi)
    
    res=0.0
    
    for j in range(n):
        
        res=res + xi[j]* w[j]
        
    return res
    
dot(xi, w)
def linear_regression(xi):
    return w0 + dot(xi, w)
linear_regression(xi)
w_new=[w0]+ w

w_new
#  is the linear regression model
def linear_regression(xi):
    
    xi= [1] + xi
    return dot(xi,w_new)
round(linear_regression(xi), 3)
xi=[453, 11, 86]
w0=7.17 #bias
w=[0.01,0.04,0.002]# weights

w_new=[w0]+ w
x1=[1, 148, 24, 1385]
x2=[1, 132, 25, 2031]
x10=[1, 453, 11, 86]

# We then make a list of lists

X=[x1, x2, x10]

X=np.array(X)

X
def linear_regression(X):
    return X.dot(w_new)
"""
## 2.7 Training a linear regression model
"""
X=[[ 148, 24, 1385],
[132, 25, 2031],
[453, 11, 86],
[158, 24, 185],
[172, 25, 201],
[413, 11, 86],
[38, 54, 185],
[142, 25, 431],
[453, 31, 86]]

X=np.array(X)
# ones=np.ones(X.shape[0])
# X= np.column_stack([ones,X])
X

y=[10000,20000,15000,20050,10000,20000,15000,25000,12000]
# gram matrix

XTX=X.T.dot(X)
XTX
XTX_inv= np.linalg.inv(XTX)
XTX_inv
# w_full=XTX_inv.dot(X.T).dot(y)
np.linalg.inv(XTX)
np.linalg.det(XTX_inv)
def train_linear_regression(X, y):
# Iclude a bias term.. helps us identify value of car if we dont know anythig about the car.
    ones=np.ones(X.shape[0])
    X=np.column_stack([ones, X])
    
    XTX=X.T.dot(X)
    XTX_inv= np.linalg.inv(XTX)
    w_full= XTX_inv.dot(X.T).dot(y)
    
    return  w_full[0], w_full[1:]
train_linear_regression(X,y)
"""
## 2.8 Car Price Baseline Model
"""
df_train.dtypes
base=['engine_hp','engine_cylinders','highway_mpg','city_mpg','popularity']



X_train= df_train[base].fillna(0).values
# LEts now train a model
w0, w= train_linear_regression(X_train, y_train)

w0, w
y_pred=w0+ X_train.dot(w)
sns.histplot(y_pred, color='red', alpha=0.4, bins=50)
sns.histplot(y_train, color='blue',alpha=0.4, bins=50)
"""
## 2.9 RMSE

EVALUATION OF OUR REGRESSION MODELS
"""
def rmse(y_pred, y):
    
    error=y_pred- y
    se=error**2
    mse=se.mean()
    return np.sqrt(mse)
    
rmse(y_pred, y_train)
"""
## 2.10 Validating the  model
"""
base=['engine_hp','engine_cylinders','highway_mpg','city_mpg','popularity']
X_train= df_train[base].fillna(0).values
w0, w= train_linear_regression(X_train, y_train)
y_pred=w0+ X_train.dot(w)
def prepare_X(df):
    df_num=df[base]
    df_num=df_num.fillna(0)
    X=df_num.values
    return X

    
X_train=prepare_X(df_train)
w0, w= train_linear_regression(X_train, y_train)

X_val=prepare_X(df_val)
y_pred=w0+ X_val.dot(w)

rmse(y_val, y_pred)

"""
## 2.11 Simple Feature Engineering
"""
df_train["year"].max()# to determine the ages of the cars.

2017-df_train["year"] # we can use this as one of our feeatures in this model
def prepare_X(df):
    df=df.copy()
    
    df['age']=2017-df["year"] 
    
    features= base+['age']
    df_num=df[features]
    df_num=df_num.fillna(0)
    X=df_num.values
    return X

X_train=prepare_X(df_train)
w0, w= train_linear_regression(X_train, y_train)

X_val=prepare_X(df_val)
y_pred=w0+ X_val.dot(w)

rmse(y_val, y_pred)


sns.histplot(y_pred, color='red', alpha=0.4, bins=50)
sns.histplot(y_val, color='blue',alpha=0.4, bins=50)
"""
## 2.12 Categorical Variables
"""
df_train.dtypes #mostly strings(object)
# df_train['num_doors_2']=(df_train.number_of_doors ==2).astype(int)
# df_train['num_doors_3']=(df_train.number_of_doors ==3).astype(int)
# df_train['num_doors_4']=(df_train.number_of_doors ==4).astype(int)
# for v in [2,3,4]:
#     df_train['num_doors_%s' %v]=(df_train.number_of_doors ==v).astype(int)
# df_train
def prepare_X(df):
    df=df.copy()
    
    df['age']=2017-df["year"] 
    
    features= base+['age']
    
    for v in[2,3,4]:
        df['num_doors_%s' %v]=(df.number_of_doors ==v).astype(int)
        features.append('num_doors_%s' %v)
        
    df_num=df[features]
    df_num=df_num.fillna(0)
    X=df_num.values
    return X
prepare_X(df_train)
X_train=prepare_X(df_train)
w0, w= train_linear_regression(X_train, y_train)

X_val=prepare_X(df_val)
y_pred=w0+ X_val.dot(w)

rmse(y_val, y_pred)

"""
There is a slight change  .002 which shows that the doors feature is not useful.. Now let us look at the make feature
"""
df.make.nunique()
makes=list(df.make.value_counts().head().index)
def prepare_X(df):
    df=df.copy()
    
    df['age']=2017-df["year"] 
    
    features= base+['age']
    
    for v in[2,3,4]:
        df['num_doors_%s' %v]=(df.number_of_doors ==v).astype(int)
        features.append('num_doors_%s' %v)
    for m in makes:
        df['make_%s' %m]=(df.make ==m).astype(int)
        features.append('make_%s' %m)
        
    df_num=df[features]
    df_num=df_num.fillna(0)
    X=df_num.values
    return X
prepare_X(df_train)
X_train=prepare_X(df_train)
w0, w= train_linear_regression(X_train, y_train)

X_val=prepare_X(df_val)
y_pred=w0+ X_val.dot(w)

rmse(y_val, y_pred)

df.dtypes
# We can now use all the categorical variables

categorical_var=['make',
    "engine_fuel_type",'transmission_type','driven_wheels','market_category','market_category'  
     ,'vehicle_style',]
# We use a dictionary
categories={}

for c in categorical_var:
    categories[c]=list(df[c].value_counts().head().index)

categories
def prepare_X(df):
    df=df.copy()
    
    df['age']=2017-df["year"] 
    
    features= base+['age']
    
    for v in[2,3,4]:
        df['num_doors_%s' %v]=(df.number_of_doors ==v).astype(int)
        features.append('num_doors_%s' %v)
    for c, values in categories.items():
        for v in values:
            df['%s_%s' %(c,v)]=(df[c] ==v).astype(int)
            features.append('%s_%s' %(c,v))
        
    df_num=df[features]
    df_num=df_num.fillna(0)
    X=df_num.values
    return X
X_train=prepare_X(df_train)
w0, w= train_linear_regression(X_train, y_train)

X_val=prepare_X(df_val)
y_pred=w0+ X_val.dot(w)

rmse(y_val, y_pred)

w0, w
"""
## 2.13 Regularization
"""
def train_linear_regression_reg(X, y, r=0.001):
# Iclude a bias term.. helps us identify value of car if we dont know anythig about the car.
    ones=np.ones(X.shape[0])
    X=np.column_stack([ones, X])
    
    XTX=X.T.dot(X)
    
    XTX=XTX+ r* np.eye(XTX.shape[0])
    XTX_inv= np.linalg.inv(XTX)
    w_full= XTX_inv.dot(X.T).dot(y)
    
    return  w_full[0], w_full[1:]
X_train=prepare_X(df_train)
w0, w= train_linear_regression_reg(X_train, y_train, r=0.001)

X_val=prepare_X(df_val)
y_pred=w0+ X_val.dot(w)

rmse(y_val, y_pred)
"""
## 2.14 Model Tuning
"""
for r in [0.0, 0.00001,0.0001,0.001,0.1,1,10]:

    X_train=prepare_X(df_train)
    w0, w= train_linear_regression_reg(X_train, y_train, r=r)

    X_val=prepare_X(df_val)
    y_pred=w0+ X_val.dot(w)

    score=rmse(y_val, y_pred)
    
    print(f"for {r} the bias term is  {w0}  and score is  {score}")
r=0.001
X_train=prepare_X(df_train)
w0, w= train_linear_regression_reg(X_train, y_train, r=r)

X_val=prepare_X(df_val)
y_pred=w0+ X_val.dot(w)

score=rmse(y_val, y_pred)
score
"""
## 2.15 Using the Model on Test Data
"""
df_full_train=pd.concat([df_train, df_val])
df_full_train=df_full_train.reset_index(drop=True)
df_full_train
X_full_train=prepare_X(df_full_train)
X_full_train
y_full_train= np.concatenate([y_train, y_val])

w0, w= train_linear_regression_reg(X_full_train, y_full_train, r=0.001)

w0, w
X_test=prepare_X(df_test)
y_pred=w0+ X_test.dot(w)

score=rmse(y_test, y_pred)
score
car=df_test.iloc[20].to_dict()

car
df_small=pd.DataFrame([car])

X_small=prepare_X(df_small)

y_pred=w0+ X_small.dot(w)

y_pred=y_pred[0]
y_pred
np.expm1(y_pred)
(np.expm1(y_test[20])- np.expm1(y_pred))
"""
Our model shows that there is a slight difference between the predicted and actual price of the car model pick. A difference of $1692.
"""