# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/aidanvu1992/DS-Unit-2-Linear-Models/blob/master/Anh_Vu__assignment_regression_classification_3.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="-1n9MiopHRRf" colab_type="text"
# Lambda School Data Science
#
# *Unit 2, Sprint 1, Module 3*
#
# ---

# + [markdown] colab_type="text" id="7IXUfiQ2UKj6"
# # Ridge Regression
#
# ## Assignment
#
# We're going back to our other **New York City** real estate dataset. Instead of predicting apartment rents, you'll predict property sales prices.
#
# But not just for condos in Tribeca...
#
# Instead, predict property sales prices for **One Family Dwellings** (`BUILDING_CLASS_CATEGORY` == `'01 ONE FAMILY DWELLINGS'`). 
#
# Use a subset of the data where the **sale price was more than \\$100 thousand and less than $2 million.** 
#
# The [NYC Department of Finance](https://www1.nyc.gov/site/finance/taxes/property-rolling-sales-data.page) has a glossary of property sales terms and NYC Building Class Code Descriptions. The data comes from the [NYC OpenData](https://data.cityofnewyork.us/browse?q=NYC%20calendar%20sales) portal.
#
# - [ ] Do train/test split. Use data from January — March 2019 to train. Use data from April 2019 to test.
# - [ ] Do one-hot encoding of categorical features.
# - [ ] Do feature selection with `SelectKBest`.
# - [ ] Do [feature scaling](https://scikit-learn.org/stable/modules/preprocessing.html).
# - [ ] Fit a ridge regression model with multiple features.
# - [ ] Get mean absolute error for the test set.
# - [ ] As always, commit your notebook to your fork of the GitHub repo.
#
#
# ## Stretch Goals
# - [ ] Add your own stretch goal(s) !
# - [ ] Instead of `Ridge`, try `LinearRegression`. Depending on how many features you select, your errors will probably blow up! 💥
# - [ ] Instead of `Ridge`, try [`RidgeCV`](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.RidgeCV.html).
# - [ ] Learn more about feature selection:
#     - ["Permutation importance"](https://www.kaggle.com/dansbecker/permutation-importance)
#     - [scikit-learn's User Guide for Feature Selection](https://scikit-learn.org/stable/modules/feature_selection.html)
#     - [mlxtend](http://rasbt.github.io/mlxtend/) library
#     - scikit-learn-contrib libraries: [boruta_py](https://github.com/scikit-learn-contrib/boruta_py) & [stability-selection](https://github.com/scikit-learn-contrib/stability-selection)
#     - [_Feature Engineering and Selection_](http://www.feat.engineering/) by Kuhn & Johnson.
# - [ ] Try [statsmodels](https://www.statsmodels.org/stable/index.html) if you’re interested in more inferential statistical approach to linear regression and feature selection, looking at p values and 95% confidence intervals for the coefficients.
# - [ ] Read [_An Introduction to Statistical Learning_](http://faculty.marshall.usc.edu/gareth-james/ISL/ISLR%20Seventh%20Printing.pdf), Chapters 1-3, for more math & theory, but in an accessible, readable way.
# - [ ] Try [scikit-learn pipelines](https://scikit-learn.org/stable/modules/compose.html).

# + colab_type="code" id="o9eSnDYhUGD7" colab={}
# %%capture
import sys

# If you're on Colab:
if 'google.colab' in sys.modules:
    DATA_PATH = 'https://raw.githubusercontent.com/LambdaSchool/DS-Unit-2-Applied-Modeling/master/data/'
    # !pip install category_encoders==2.*

# If you're working locally:
else:
    DATA_PATH = '../data/'
    
# Ignore this Numpy warning when using Plotly Express:
# FutureWarning: Method .ptp is deprecated and will be removed in a future version. Use numpy.ptp instead.
import warnings
warnings.filterwarnings(action='ignore', category=FutureWarning, module='numpy')

# + colab_type="code" id="QJBD4ruICm1m" colab={}
import pandas as pd
import pandas_profiling

# Read New York City property sales data
df = pd.read_csv(DATA_PATH+'condos/NYC_Citywide_Rolling_Calendar_Sales.csv')

# Change column names: replace spaces with underscores
df.columns = [col.replace(' ', '_') for col in df]

# SALE_PRICE was read as strings.
# Remove symbols, convert to integer
df['SALE_PRICE'] = (
    df['SALE_PRICE']
    .str.replace('$','')
    .str.replace('-','')
    .str.replace(',','')
    .astype(int)
)

# + id="m31CM-x6HRRn" colab_type="code" colab={}
# BOROUGH is a numeric column, but arguably should be a categorical feature,
# so convert it from a number to a string
df['BOROUGH'] = df['BOROUGH'].astype(str)

# + id="5zov_VXaHRRp" colab_type="code" colab={}
# Reduce cardinality for NEIGHBORHOOD feature

# Get a list of the top 10 neighborhoods
top10 = df['NEIGHBORHOOD'].value_counts()[:10].index

# At locations where the neighborhood is NOT in the top 10, 
# replace the neighborhood with 'OTHER'
df.loc[~df['NEIGHBORHOOD'].isin(top10), 'NEIGHBORHOOD'] = 'OTHER'

# + id="elG1IGkBHRRr" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 434} outputId="15ee5119-bcb3-49fa-e565-711b29791c34"
df.head()

# + id="qm0W69YpSK26" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 218} outputId="372b7966-b772-4fb1-ba03-f15e420a6da5"
df['SALE_PRICE'].value_counts()

# + id="_NWtCt8WSbc-" colab_type="code" colab={}
condition = (df['SALE_PRICE'] > 100000) & (df['SALE_PRICE'] < 2000000)

# + id="TFREPgloVhRE" colab_type="code" colab={}
df = df.loc[condition]

# + id="dyEFJAywVnwe" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 218} outputId="9b963c4e-907c-4fdc-961a-69e7ea61190e"
df['SALE_PRICE'].value_counts()

# + id="nHUSdpOQWRS7" colab_type="code" colab={}
OFD = df[df['BUILDING_CLASS_CATEGORY'] == '01 ONE FAMILY DWELLINGS']

# + id="rdigZAhyMbxV" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 640} outputId="98f7db33-9b30-4d88-edf9-d90485e63b6e"
OFD

# + id="W7NaJmn8TvVa" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="d06126c6-885b-4088-d79a-ac2c41dc4960"
OFD.shape

# + id="jvFpf9xbHoyK" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 386} outputId="d8bcf77a-5cd2-41da-f64b-1f15bb8b4170"
OFD.dtypes

# + id="d_ilLoLDMViT" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 118} outputId="8a05f9ac-6da9-4b3c-ccb1-fed7fbbe461b"
# Create cutoff day
OFD['SALE_DATE'] = pd.to_datetime(OFD['SALE_DATE'], infer_datetime_format = True)
cutoff = pd.to_datetime('04/01/2019')

# + id="W0vMvYfWNNK2" colab_type="code" colab={}
#Train and test
train = OFD[OFD['SALE_DATE'] < cutoff]
test = OFD[OFD['SALE_DATE'] >= cutoff]

# + id="ncfKfqcDOC1T" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 269} outputId="e8b859bd-f5c5-4a4b-a309-3c5eb5b62e93"
train.head(), train.tail()

# + id="KH7TuNDJOO7B" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 269} outputId="16b1fc9d-7baa-4be5-c969-28098bc8bef9"
test.head(), test.tail()

# + id="pyHWzNeKOphO" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="c90858f8-21de-4ae5-965f-96b643de1774"
print(train.shape, test.shape)

# + id="4HRsws7AOtak" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 343} outputId="e63aa88e-e88d-4937-cb2b-e5752c878b7d"
# Check for unique values of object columns
train.select_dtypes(exclude = 'number').describe().T

# + id="MpCxXeMtNl5x" colab_type="code" colab={}
# Create features and target for one-hot coding

target = 'SALE_PRICE'
high_cardinality = ['ADDRESS', 'LAND_SQUARE_FEET', 'LAND_SQUARE_FEET', 'SALE_DATE']
NaN = ['EASE-MENT', 'APARTMENT_NUMBER']
features = train.columns.drop([target] + high_cardinality + NaN)

X_train = train[features]
y_train = train[target]
X_test = test[features]
y_test = test[target]

# + id="z0ditX0nP4dz" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 316} outputId="70d5ee41-d8c2-4830-f875-34011c237e88"
print(X_train.shape)
X_train.head()

# + id="CcKRdSzbP_I1" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 316} outputId="a5e57ac7-56d0-4548-ca2c-df0a0a60dedb"
print(X_test.shape)
X_test.head()

# + id="EOFzjtMsQECw" colab_type="code" colab={}
# One-hot encoder
import category_encoders as ce 

encoder = ce.OneHotEncoder(use_cat_names = True)
X_train = encoder.fit_transform(X_train)
X_test = encoder.transform(X_test)

# + id="acV_VPiOQoUg" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 249} outputId="3984d3c5-e61a-4654-b939-7018775d7540"
print(X_train.shape)
X_train.head()

# + id="1iIdCE7WQtlO" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 249} outputId="20d88c7d-9768-4f69-8063-a4166afa2a6f"
print(X_test.shape)
X_test.head()

# + id="r5j3gFRPSOGC" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 185} outputId="4b844b05-fa15-40e4-fc99-b1eb791bc388"
# Perform feature selection with SelectKBest

from sklearn.feature_selection import f_regression, SelectKBest

selector = SelectKBest(score_func = f_regression, k = 20)
X_train_selected = selector.fit_transform(X_train, y_train)
X_test_selected = selector.transform(X_test)

# + id="olDyggn-TZRv" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="a6e965ee-ca9c-49a9-a26b-6e62a9cb02f2"
print(X_train_selected.shape, X_test_selected.shape)

# + id="MMGqtbyOTih8" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 874} outputId="66da9f8e-628a-4b5e-9a02-8ff1cf920370"
# Which features were selected?

all_names = X_train.columns
selected_mask = selector.get_support()
selected_names = all_names[selected_mask]
unselected_names = all_names[~selected_mask]

print('Selected Features')
for name in selected_names:
  print(name)

print('')
print('Features Not Selected:')
for name in unselected_names:
  print(name)

# + id="Cibuz-t5UyKm" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 1000} outputId="d3e43f95-2d72-4750-97f5-43690b895af1"
# Scaling features

from sklearn.linear_model import LinearRegression as LR
from sklearn.metrics import mean_absolute_error as MAE 

for k in range(1, len(X_train.columns) + 1):
  print(f'{k} features:')

  selector = SelectKBest(score_func = f_regression, k = k)
  X_train_selected = selector.fit_transform(X_train, y_train)
  X_test_selected = selector.transform(X_test)

  model = LR()
  model.fit(X_train_selected, y_train)
  y_pred = model.predict(X_test_selected)

  mae = MAE(y_test, y_pred)
  print(f'Test MAE: ${mae:,.0f} \n')

# + id="dx0Ms_oaWsFL" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 1000} outputId="9f257cdb-0d41-419f-d8b9-dee846361cec"
# Fit Ridge Regression and find MAE for test set

# %matplotlib inline
from IPython.display import display, HTML
from ipywidgets import interact
import matplotlib.pyplot as plt
from sklearn.linear_model import Ridge
from sklearn.preprocessing import StandardScaler

for alpha in [10**1, 10**2, 10**3, 10**4, 10**5, 10**6]:

  # Scale data before doing Ridge Regression
  scaler = StandardScaler()
  X_train_scaled = scaler.fit_transform(X_train)
  X_test_scaled = scaler.transform(X_test)

  # Fit Ridge Regression model
  display(HTML(f'Ridge Regression, with alpha = {alpha}'))
  model = Ridge(alpha = alpha)
  model.fit(X_train_scaled, y_train)
  y_pred = model.predict(X_test_scaled)

  # Get Test MAE
  mae = MAE(y_test, y_pred)
  display(HTML(f'Test Mean Absolute Error: ${mae:,.0f}'))

  # Plot coefficients
   # Plot coefficients
  coefficients = pd.Series(model.coef_, X_train.columns)
  plt.figure(figsize=(16,8))
  coefficients.sort_values().plot.barh(color='grey')
  plt.xlim(-1000,1000)
  plt.show()

StarcoderdataJupyterScriptsDedupFiltered