# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

df = pd.read_csv("house_prices.csv")

y = df.SalePrice
df.drop("SalePrice",axis=1, inplace=True)
# now we have our X,y set. X is DF and y is our saleprice! NICE!

# ## removing features with small variance

# +
from sklearn.feature_selection import VarianceThreshold

vt = VarianceThreshold(0.1)
df_transformed = vt.fit_transform(df)
# -

# #### As mentioned last week, fit_transform() in sklearn transforms an object from DataFrame to numpy.array and we are losing column names, so we need to do some tricks to get them back!

# columns we have selected
# get_support() is method of VarianceThreshold and stores boolean of each variable in the numpy array.
selected_columns = df.columns[vt.get_support()]
# transforming an array back to a data-frame preserves column labels
df_transformed = pd.DataFrame(df_transformed, columns = selected_columns)

# ## removing correlated features

# +
# step 1 calculate a correlation matrix
df_corr = df_transformed.corr().abs()

# step 2 get pairs of highly correlated figures
indices = np.where(df_corr > 0.8) 
indices = [(df_corr.index[x], df_corr.columns[y]) 
for x, y in zip(*indices)
    if x != y and x < y]

# step 3 remove correlated columns
for idx in indices: #each pair
    try:
        df_transformed.drop(idx[1], axis = 1, inplace=True)
    except KeyError:
        pass
# -

print(indices) # highly correlated pairs

# ## forward regression

from sklearn.feature_selection import f_regression, SelectKBest
skb = SelectKBest(f_regression, k=10)
X = skb.fit_transform(df_transformed, y)

# this will give us the position of top 10 columns
skb.get_support()
# column names
df_transformed.columns[skb.get_support()]
X = pd.DataFrame(X,columns=df_transformed.columns[skb.get_support()]) # back to DataFrame

# # statsmodel

import statsmodels.api as sm

X = sm.add_constant(X) # adding a constant

lin_reg = sm.OLS(y,X) # LEAST SQUARES "OLS"

model = lin_reg.fit()
print_model = model.summary()
print(print_model)

# # sklearn

from sklearn.linear_model import LinearRegression
regressor = LinearRegression()
regressor.fit(X,y)

print(regressor.coef_)

regressor.score(X,y)



StarcoderdataJupyterScriptsDedupFiltered