# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/n8mcdunna/DS-Unit-2-Linear-Models/blob/master/module3-ridge-regression/Nathan_213_assignment.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="l-CQB0NFgWxA"
# Lambda School Data Science
#
# *Unit 2, Sprint 1, Module 3*
#
# ---

# + [markdown] id="7IXUfiQ2UKj6"
# # Ridge Regression
#
# ## Assignment
#
# We're going back to our other **New York City** real estate dataset. Instead of predicting apartment rents, you'll predict property sales prices.
#
# But not just for condos in Tribeca...
#
# - [ ] Use a subset of the data where `BUILDING_CLASS_CATEGORY` == `'01 ONE FAMILY DWELLINGS'` and the sale price was more than 100 thousand and less than 2 million.
# - [ ] Do train/test split. Use data from January — March 2019 to train. Use data from April 2019 to test.
# - [ ] Do one-hot encoding of categorical features.
# - [ ] Do feature selection with `SelectKBest`.
# - [ ] Fit a ridge regression model with multiple features. Use the `normalize=True` parameter (or do [feature scaling](https://scikit-learn.org/stable/modules/preprocessing.html) beforehand — use the scaler's `fit_transform` method with the train set, and the scaler's `transform` method with the test set)
# - [ ] Get mean absolute error for the test set.
# - [ ] As always, commit your notebook to your fork of the GitHub repo.
#
# The [NYC Department of Finance](https://www1.nyc.gov/site/finance/taxes/property-rolling-sales-data.page) has a glossary of property sales terms and NYC Building Class Code Descriptions. The data comes from the [NYC OpenData](https://data.cityofnewyork.us/browse?q=NYC%20calendar%20sales) portal.
#
#
# ## Stretch Goals
#
# Don't worry, you aren't expected to do all these stretch goals! These are just ideas to consider and choose from.
#
# - [ ] Add your own stretch goal(s) !
# - [ ] Instead of `Ridge`, try `LinearRegression`. Depending on how many features you select, your errors will probably blow up! 💥
# - [ ] Instead of `Ridge`, try [`RidgeCV`](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.RidgeCV.html).
# - [ ] Learn more about feature selection:
#     - ["Permutation importance"](https://www.kaggle.com/dansbecker/permutation-importance)
#     - [scikit-learn's User Guide for Feature Selection](https://scikit-learn.org/stable/modules/feature_selection.html)
#     - [mlxtend](http://rasbt.github.io/mlxtend/) library
#     - scikit-learn-contrib libraries: [boruta_py](https://github.com/scikit-learn-contrib/boruta_py) & [stability-selection](https://github.com/scikit-learn-contrib/stability-selection)
#     - [_Feature Engineering and Selection_](http://www.feat.engineering/) by Kuhn & Johnson.
# - [ ] Try [statsmodels](https://www.statsmodels.org/stable/index.html) if you’re interested in more inferential statistical approach to linear regression and feature selection, looking at p values and 95% confidence intervals for the coefficients.
# - [ ] Read [_An Introduction to Statistical Learning_](http://faculty.marshall.usc.edu/gareth-james/ISL/ISLR%20Seventh%20Printing.pdf), Chapters 1-3, for more math & theory, but in an accessible, readable way.
# - [ ] Try [scikit-learn pipelines](https://scikit-learn.org/stable/modules/compose.html).

# + id="o9eSnDYhUGD7"
# %%capture
import sys

# If you're on Colab:
if 'google.colab' in sys.modules:
    DATA_PATH = 'https://raw.githubusercontent.com/LambdaSchool/DS-Unit-2-Applied-Modeling/master/data/'
    # !pip install category_encoders==2.*

# If you're working locally:
else:
    DATA_PATH = '../data/'
    
# Ignore this Numpy warning when using Plotly Express:
# FutureWarning: Method .ptp is deprecated and will be removed in a future version. Use numpy.ptp instead.
import warnings
warnings.filterwarnings(action='ignore', category=FutureWarning, module='numpy')

# + id="8Y-aT700kLcQ"
import numpy as np

# + id="QJBD4ruICm1m"
import pandas as pd
import pandas_profiling

# Read New York City property sales data
df = pd.read_csv(DATA_PATH+'condos/NYC_Citywide_Rolling_Calendar_Sales.csv', parse_dates= ['SALE DATE'], index_col= 'SALE DATE')

# Change column names: replace spaces with underscores
df.columns = [col.replace(' ', '_') for col in df]

# SALE_PRICE was read as strings.
# Remove symbols, convert to integer
df['SALE_PRICE'] = (
    df['SALE_PRICE']
    .str.replace('$','')
    .str.replace('-','')
    .str.replace(',','')
    .astype(int)
)

# + id="1ksiUuHZgWxQ"
# BOROUGH is a numeric column, but arguably should be a categorical feature,
# so convert it from a number to a string
df['BOROUGH'] = df['BOROUGH'].astype(str)

# + id="l4JwfVWlgWxV"
# Reduce cardinality for NEIGHBORHOOD feature

# Get a list of the top 10 neighborhoods
top10 = df['NEIGHBORHOOD'].value_counts()[:10].index

# At locations where the neighborhood is NOT in the top 10, 
# replace the neighborhood with 'OTHER'
df.loc[~df['NEIGHBORHOOD'].isin(top10), 'NEIGHBORHOOD'] = 'OTHER'

# + id="LZBxoyNqhLiA"
# Subsetting the dataframe
condition1 = df['BUILDING_CLASS_CATEGORY'] == '01 ONE FAMILY DWELLINGS'
condition2 = df['SALE_PRICE'] > 100000
condition3 = df['SALE_PRICE'] < 2000000
df = df[condition1 & condition2 & condition3]

# + id="KmdF_Ym71E2z" outputId="ab732e30-c828-4fa1-cb57-c5e6573ad755" colab={"base_uri": "https://localhost:8080/", "height": 391}
print(df.shape)
df.isnull().sum()

# + id="E3OAh2l01Kdn" outputId="d8c6529b-7bc7-4edb-d7f3-2523ce05e8b7" colab={"base_uri": "https://localhost:8080/", "height": 357}
# Removing useless columns filled with NaNs
df.dropna(axis= 1, inplace= True)
print(df.shape)
df.isnull().sum()

# + id="Qu-dLeyDl_3K" outputId="ebcdfef8-fbc7-49ab-abcc-4cbfc0b10215" colab={"base_uri": "https://localhost:8080/", "height": 170}
# Find categorical variables
df.select_dtypes('object').nunique()

# + id="2k8c6KxQmmtE"
# Drop high cardinality columns
hc_cols = [col for col in df.select_dtypes('object').columns if df[col].nunique() > 12]
df.drop(hc_cols, inplace= True, axis = 1)

# + id="0SFU5taMhiUl"
# Train/test split
target = 'SALE_PRICE'
y = df[target]
X = df.drop(columns= target)

cutoff = '2019-04-01'
mask = X.index < cutoff
X_train, y_train = X.loc[mask], y.loc[mask]
X_test, y_test = X.loc[~mask], y.loc[~mask]

# + id="MRmKyhGYldYe" outputId="836f7186-700f-41b8-dd15-17f99c076bda" colab={"base_uri": "https://localhost:8080/", "height": 105}
# One hot encoding categorical variables
from category_encoders import OneHotEncoder
ohe = OneHotEncoder(use_cat_names= True)

XT_train = ohe.fit_transform(X_train)
XT_test = ohe.transform(X_test)

# + id="7NJl-Oo90hlM" outputId="1ae9ca17-f643-46df-a9dc-ae1cbc530644" colab={"base_uri": "https://localhost:8080/", "height": 85}
print(XT_train.shape)
print(XT_test.shape)
print(y_train.shape)
print(y_test.shape)

# + id="YhaICBTNjOP-" outputId="ae3c570d-0deb-4ae2-80cd-185c5bc8b3c2" colab={"base_uri": "https://localhost:8080/", "height": 374}
XT_train.columns

# + id="wN8GGXXskBkb" outputId="920d3cf0-54b7-4f57-9f80-28a3c28958ec" colab={"base_uri": "https://localhost:8080/", "height": 119}
# Feature selection with selectkbest
from sklearn.feature_selection import SelectKBest, f_regression
selector = SelectKBest(k= 15)
XT_train_selected = selector.fit_transform(XT_train, y_train)
XT_test_selected = selector.transform(XT_test)

# + id="Ejo7LhK2sLER" outputId="246e6d11-d9d5-41b3-ad54-a1560144955c" colab={"base_uri": "https://localhost:8080/", "height": 51}
print(XT_train_selected.shape)
print(XT_test_selected.shape)

# + id="I3oHbHAX2BFe" outputId="f8ab8f31-eecf-4d15-db24-b003cf609632" colab={"base_uri": "https://localhost:8080/", "height": 272}
selected_mask = selector.get_support()
selected_names = XT_train.columns[selected_mask]
[print(name) for name in selected_names];

# + id="KqbHaTVM2Nik" outputId="4d3e7fe1-ec1a-4080-d692-a1cb0b7bba5b" colab={"base_uri": "https://localhost:8080/", "height": 51}
# Modeling using linear and ridge regression
from sklearn.linear_model import Ridge, LinearRegression
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score

LR_model = LinearRegression()
LR_model.fit(XT_train_selected, y_train)

ridge_model = Ridge(alpha = 5, normalize= False)
ridge_model.fit(XT_train_selected, y_train)

# + id="iXDWuIdH2Q0i" outputId="29b6489e-593d-419f-d4b3-257db416fec5" colab={"base_uri": "https://localhost:8080/", "height": 119}
print('Linear Reg training MAE: ', mean_absolute_error(y_train, LR_model.predict(XT_train_selected)))
print('Linear Reg testing MAE:  ', mean_absolute_error(y_test, LR_model.predict(XT_test_selected)))
print('Ridge training MAE:      ', mean_absolute_error(y_train, ridge_model.predict(XT_train_selected)))
print('Ridge testing MAE:       ', mean_absolute_error(y_test, ridge_model.predict(XT_test_selected)))
print('Ridge R^2:               ', r2_score(y_train, ridge_model.predict(XT_train_selected)))
print('Ridge R^2:               ', r2_score(y_test, ridge_model.predict(XT_test_selected)))

# + id="DESyrC1zAEob" outputId="8daa1e26-cd60-44e5-8e71-3c82d5d91c6b" colab={"base_uri": "https://localhost:8080/", "height": 657}
X_train

# + id="nzqMMphM94Co" outputId="7f8cf168-6f25-4f18-b557-14354a6b8df7" colab={"base_uri": "https://localhost:8080/", "height": 1000}
maes = []
Rsquareds = []

for k in range(1, len(XT_train.columns)+1):

    selector = SelectKBest(score_func=f_regression, k=k)
    X_train_selected = selector.fit_transform(XT_train, y_train)
    X_test_selected = selector.transform(XT_test)

    #model = LinearRegression()
    model = Ridge(alpha = 5, normalize= False)
    model.fit(X_train_selected, y_train)
    y_pred = model.predict(X_test_selected)
    mae = mean_absolute_error(y_test, y_pred)
    Rsquare = r2_score(y_test, y_pred)

    maes.append(mae)
    Rsquareds.append(Rsquare)

# + id="OVq1fg5Z5rQf" outputId="a6a4a88a-1413-4102-b805-cf7a11169d99" colab={"base_uri": "https://localhost:8080/", "height": 296}
import matplotlib.pyplot as plt
# %matplotlib inline

x = range(1, len(maes)+1)
y = maes
print(min(maes))

plt.plot(x, y)
plt.xlabel('Number of Features')
plt.ylabel('Mean Absolute Error')
plt.show();

# + id="O6KMo7ZTbE3R" outputId="2ffbd0ed-dd60-4d72-b9ff-5e340e4da448" colab={"base_uri": "https://localhost:8080/", "height": 296}
print(max(Rsquareds))

y = Rsquareds

plt.plot(x, y)
plt.xlabel('Number of Features')
plt.ylabel('R^2')
plt.show();

# + id="oFNobrQ_dusJ"


# + id="sdwbp-Ohc8M2"
157492.24827086728 for Linear Regression

StarcoderdataJupyterScriptsDedupFiltered