# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="ePh4rRYS9G3-" colab_type="text"
# Lambda School Data Science
#
# *Unit 2, Sprint 1, Module 3*
#
# ---

# + [markdown] colab_type="text" id="7IXUfiQ2UKj6"
# # Ridge Regression
#
# ## Assignment
#
# We're going back to our other **New York City** real estate dataset. Instead of predicting apartment rents, you'll predict property sales prices.
#
# But not just for condos in Tribeca...
#
# - [ ] Use a subset of the data where `BUILDING_CLASS_CATEGORY` == `'01 ONE FAMILY DWELLINGS'` and the sale price was more than 100 thousand and less than 2 million.
# - [ ] Do train/test split. Use data from January — March 2019 to train. Use data from April 2019 to test.
# - [ ] Do one-hot encoding of categorical features.
# - [ ] Do feature selection with `SelectKBest`.
# - [ ] Fit a ridge regression model with multiple features. Use the `normalize=True` parameter (or do [feature scaling](https://scikit-learn.org/stable/modules/preprocessing.html) beforehand — use the scaler's `fit_transform` method with the train set, and the scaler's `transform` method with the test set)
# - [ ] Get mean absolute error for the test set.
# - [ ] As always, commit your notebook to your fork of the GitHub repo.
#
# The [NYC Department of Finance](https://www1.nyc.gov/site/finance/taxes/property-rolling-sales-data.page) has a glossary of property sales terms and NYC Building Class Code Descriptions. The data comes from the [NYC OpenData](https://data.cityofnewyork.us/browse?q=NYC%20calendar%20sales) portal.
#
#
# ## Stretch Goals
#
# Don't worry, you aren't expected to do all these stretch goals! These are just ideas to consider and choose from.
#
# - [ ] Add your own stretch goal(s) !
# - [ ] Instead of `Ridge`, try `LinearRegression`. Depending on how many features you select, your errors will probably blow up! 💥
# - [ ] Instead of `Ridge`, try [`RidgeCV`](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.RidgeCV.html).
# - [ ] Learn more about feature selection:
#     - ["Permutation importance"](https://www.kaggle.com/dansbecker/permutation-importance)
#     - [scikit-learn's User Guide for Feature Selection](https://scikit-learn.org/stable/modules/feature_selection.html)
#     - [mlxtend](http://rasbt.github.io/mlxtend/) library
#     - scikit-learn-contrib libraries: [boruta_py](https://github.com/scikit-learn-contrib/boruta_py) & [stability-selection](https://github.com/scikit-learn-contrib/stability-selection)
#     - [_Feature Engineering and Selection_](http://www.feat.engineering/) by Kuhn & Johnson.
# - [ ] Try [statsmodels](https://www.statsmodels.org/stable/index.html) if you’re interested in more inferential statistical approach to linear regression and feature selection, looking at p values and 95% confidence intervals for the coefficients.
# - [ ] Read [_An Introduction to Statistical Learning_](http://faculty.marshall.usc.edu/gareth-james/ISL/ISLR%20Seventh%20Printing.pdf), Chapters 1-3, for more math & theory, but in an accessible, readable way.
# - [ ] Try [scikit-learn pipelines](https://scikit-learn.org/stable/modules/compose.html).

# + colab_type="code" id="o9eSnDYhUGD7" colab={}
# %%capture
import sys

# If you're on Colab:
if 'google.colab' in sys.modules:
    DATA_PATH = 'https://raw.githubusercontent.com/LambdaSchool/DS-Unit-2-Applied-Modeling/master/data/'
    # !pip install category_encoders==2.*

# If you're working locally:
else:
    DATA_PATH = '../data/'
    
# Ignore this Numpy warning when using Plotly Express:
# FutureWarning: Method .ptp is deprecated and will be removed in a future version. Use numpy.ptp instead.
import warnings
warnings.filterwarnings(action='ignore', category=FutureWarning, module='numpy')

# + colab_type="code" id="QJBD4ruICm1m" colab={}
import pandas as pd
import pandas_profiling

# Read New York City property sales data
df = pd.read_csv(DATA_PATH+'condos/NYC_Citywide_Rolling_Calendar_Sales.csv')

# Change column names: replace spaces with underscores
df.columns = [col.replace(' ', '_') for col in df]

# SALE_PRICE was read as strings.
# Remove symbols, convert to integer
df['SALE_PRICE'] = (
    df['SALE_PRICE']
    .str.replace('$','')
    .str.replace('-','')
    .str.replace(',','')
    .astype(int)
)

# + id="OT7Fj32LfNRD" colab_type="code" colab={}
# BOROUGH is a numeric column, but arguably should be a categorical feature,
# so convert it from a number to a string
df['BOROUGH'] = df['BOROUGH'].astype(str)

# + id="u3GG2BbEfNRF" colab_type="code" colab={}
# Reduce cardinality for NEIGHBORHOOD feature

# Get a list of the top 10 neighborhoods
top10 = df['NEIGHBORHOOD'].value_counts()[:10].index

# At locations where the neighborhood is NOT in the top 10, 
# replace the neighborhood with 'OTHER'
df.loc[~df['NEIGHBORHOOD'].isin(top10), 'NEIGHBORHOOD'] = 'OTHER'

# + id="mlTz9SdwfNRH" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 445} outputId="a21390a6-2e68-4446-cedd-bb3c1f93b9db"
df.head()

# + id="i1DReq_jfoig" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 476} outputId="b5e1556b-c4c7-44bb-a31a-f4c4fe944e71"
df = df.set_index('SALE_DATE')
df.head()

# + id="GJNNnuTVgLQ_" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 380} outputId="35a0e5a1-96b3-45c6-9475-052660fc8710"
df = df.sort_index()
df.head(3)

# + id="CF3sOqsDgRSg" colab_type="code" colab={}
df.index = pd.to_datetime(df.index)

# + id="vKmpSGZTgdA_" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 476} outputId="f5915cf5-d1f2-46b5-8855-ec6931be5544"
df.info()

# + id="sUvSP2kKgy2Z" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="5677743e-6433-4d4e-84b1-d1063abd3a56"
#Convert appropriate columns to numeric
df['LAND_SQUARE_FEET'] = pd.to_numeric(df['LAND_SQUARE_FEET'], errors='coerce')
df['LAND_SQUARE_FEET'].dtype

# + id="vchXu9j0of8X" colab_type="code" colab={}
#Filter data
#Use a subset of the data where BUILDING_CLASS_CATEGORY == '01 ONE FAMILY DWELLINGS' and the sale price was more than 100 thousand and less than 2 million.
df = df[(df['SALE_PRICE'] > 100000) & (df['SALE_PRICE'] < 2000000) & (df['BUILDING_CLASS_CATEGORY'] == '01 ONE FAMILY DWELLINGS')]

# + id="tJ7TnWC2feno" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 756} outputId="aa5175eb-96d6-48c6-f98b-afea657ef3e4"
#Feature Engineering
#Find average sale price of zip code
temp = df[['ZIP_CODE', 'SALE_PRICE']].groupby(by='ZIP_CODE').mean()
print(temp)
temp2 = df['ZIP_CODE'].map(lambda x: temp['SALE_PRICE'][temp.index == x].values[0])
print(temp2.values)
print(temp2.dtype)
df['ZIP_CODE_MEAN_SALE_PRICE'] = temp2.values
df[['ZIP_CODE_MEAN_SALE_PRICE','ZIP_CODE']].sort_values(by='ZIP_CODE')

# + id="IxQft0VNfx4I" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 357} outputId="0df8ebab-3867-4bc6-9103-34decc69a005"
df.head()

# + id="UahHuXs5YMXn" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 51} outputId="907840b5-7dc4-4931-8d6b-c33851541050"
import numpy as np
#Below is the crappy OneHotEncoder since it can't automatically select columns for OneHotEncoding
#from sklearn.preprocessing import OneHotEncoder
from category_encoders import OneHotEncoder
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_absolute_error
from sklearn.feature_selection import SelectKBest
from sklearn.metrics import mean_squared_error


# + id="AmY7Sz30hWW7" colab_type="code" colab={}
#Model features a one-hot-encoder, Select K Best, and linear regression
def model(df, target, features_arr):
  X = df[features_arr]
  y = df[target]

  #Do train/test split. Use data from January — March 2019 to train. Use data from April 2019 to test.
  mask = df.index.map(lambda x: True if x.month in range(1,4) and x.year == 2019 else False)
  invert_mask = mask.map(lambda x: not x)

  X_train = X[mask]
  y_train = y[mask]

  X_val = X[invert_mask]
  y_val = y[invert_mask]


  transformer = OneHotEncoder(use_cat_names=True)

  #Fits transformer to training data (I'm not totally sure what that means)
  transformer.fit(X_train)

  
  #One hot encodes categorical data
  XT_train = transformer.transform(X_train)
  XT_val = transformer.transform(X_val)
  
  #Do feature selection with SelectKBest.
  #Note: what is the best value for k?
  #num = round(len(features)/2)
  num = round(len(features_arr)/1.5)
  transformer_2 = SelectKBest(k=num) #k is a hyperparameter

  #Fit transformer to training data
  transformer_2.fit(XT_train, y_train)

  #Transform the training data
  XTT_train = transformer_2.transform(XT_train)
  XTT_val = transformer_2.transform(XT_val)

  #Create the model
  model = LinearRegression()

  #Fit model to training data
  model.fit(XTT_train, y_train)

  LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, normalize=False)

  #Training predictions derived from feature data
  y_pred_train = model.predict(XTT_train)

  #Validation predictions derived from feature data
  y_pred = model.predict(XTT_val)

  #Get mean absolute error for the test set.
  print("Training Mean absolute error:", mean_absolute_error(y_train, y_pred_train))
  print("Validation Mean absolute error:", mean_absolute_error(y_val, y_pred), '\n')

  print('Training MAE:', mean_absolute_error(y_train, y_pred_train))
  print('Validation MAE:', mean_absolute_error(y_val, y_pred), '\n')

  print("Traning RMSE:", mean_squared_error(y_train, y_pred_train, squared=False))
  print("Validation RMSE:", mean_squared_error(y_val, y_pred, squared=False), '\n')

  print("Training R-squared score is:", model.score(XTT_train, y_train))
  print("Validation R-squared is:", model.score(XTT_val, y_val), '\n')

# + id="7VARo1DV_8Rf" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 493} outputId="f610824c-5666-4300-f652-4535745278a8"
df.info()

# + id="9g5jP6jLiVb8" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 323} outputId="ede30f98-f671-4989-ca4b-7239cbffe56b"
model(df, 'SALE_PRICE', ['NEIGHBORHOOD', 'TAX_CLASS_AT_PRESENT', 'BUILDING_CLASS_CATEGORY', 'BUILDING_CLASS_AT_PRESENT', 'BOROUGH', 'ZIP_CODE_MEAN_SALE_PRICE'])

# + id="T7MzdgPvhVvX" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 153} outputId="887bd0ba-dd49-4454-957a-af347a71bf48"
print(df.columns)

# + id="7PD8JsMSCY_H" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 425} outputId="835af2db-fede-4a92-eb3d-091afa0bd367"
features = df.columns.map(lambda x: x if x not in ['SALE_PRICE', 'APARTMENT_NUMBER', 'EASE-MENT', 'LAND_SQUARE_FEET', 'ADDRESS', 'LOT', 'BLOCK'] else np.NaN)
features = features.dropna()
print(features)
model(df, 'SALE_PRICE', features)

# + id="ji3nbHbbkKa2" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 374} outputId="c892fbcd-697a-41c9-c832-f8e146ef6ef2"
('BOROUGH', 'NEIGHBORHOOD', 'BUILDING_CLASS_CATEGORY',
       'TAX_CLASS_AT_PRESENT', 'BLOCK', 'LOT', 'EASE-MENT',
       'BUILDING_CLASS_AT_PRESENT', 'ADDRESS', 'APARTMENT_NUMBER', 'ZIP_CODE',
       'RESIDENTIAL_UNITS', 'COMMERCIAL_UNITS', 'TOTAL_UNITS',
       'LAND_SQUARE_FEET', 'GROSS_SQUARE_FEET', 'YEAR_BUILT',
       'TAX_CLASS_AT_TIME_OF_SALE', 'BUILDING_CLASS_AT_TIME_OF_SALE',
       'SALE_PRICE', 'ZIP_CODE_MEAN_SALE_PRICE')

# + id="oXbRiu8OjqtM" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 323} outputId="64294304-a53a-41a6-e142-95177d4a0648"
features = ['YEAR_BUILT', 'ZIP_CODE_MEAN_SALE_PRICE', 'COMMERCIAL_UNITS', 'RESIDENTIAL_UNITS', 'BUILDING_CLASS_AT_PRESENT', 'TAX_CLASS_AT_PRESENT', 'BUILDING_CLASS_CATEGORY', 
            'NEIGHBORHOOD', 'GROSS_SQUARE_FEET']
model(df, 'SALE_PRICE', features)

# + id="0uO5veuMmVIG" colab_type="code" colab={}
from sklearn.linear_model import Ridge
#Model features a one-hot-encoder, Select K Best, and ridge regression
def model_ridge_regression(df, target, features_arr):
  X = df[features_arr]
  y = df[target]

  #Do train/test split. Use data from January — March 2019 to train. Use data from April 2019 to test.
  mask = df.index.map(lambda x: True if x.month in range(1,4) and x.year == 2019 else False)
  invert_mask = mask.map(lambda x: not x)

  X_train = X[mask]
  y_train = y[mask]

  X_val = X[invert_mask]
  y_val = y[invert_mask]

  #Note: instead of cat names you can use cols=[''] to pass in exact columns to encode
  #Do one-hot encoding of categorical features.
  transformer = OneHotEncoder(use_cat_names=True)

  #Fits transformer to training data (I'm not totally sure what that means)
  transformer.fit(X_train)

  #Note: the one hot encoder is smart enough to find columns with 100% string values and encode them (I think 100% string to work)
  #One hot encodes categorical data
  XT_train = transformer.transform(X_train)
  XT_val = transformer.transform(X_val)
  
  #Do feature selection with SelectKBest.
  #Note: what is the best value for k?
  #num = round(len(features)/2)
  num = round(len(features)/2)
  transformer_2 = SelectKBest(k=num) #k is a hyperparameter

  #Fit transformer to training data
  transformer_2.fit(XT_train, y_train)

  #Transform the training data
  XTT_train = transformer_2.transform(XT_train)
  XTT_val = transformer_2.transform(XT_val)

  #Fit a ridge regression model with multiple features. Use the normalize=True parameter (or do feature scaling beforehand — use the scaler's fit_transform method with the train set, and the scaler's transform method with the test set)
  #Create the model
  model = Ridge(alpha=0.2)

  #Fit model to training data
  model.fit(XTT_train, y_train)

  Ridge(alpha=0.2, copy_X=True, fit_intercept=True, max_iter=None, normalize=False, random_state=None, solver='auto', tol=0.001)

  #Training predictions derived from feature data
  y_pred_train = model.predict(XTT_train)

  #Validation predictions derived from feature data
  y_pred = model.predict(XTT_val)

  #Get mean absolute error for the test set.
  print("Training Mean absolute error:", mean_absolute_error(y_train, y_pred_train))
  print("Validation Mean absolute error:", mean_absolute_error(y_val, y_pred), '\n')

  print('Training MAE:', mean_absolute_error(y_train, y_pred_train))
  print('Validation MAE:', mean_absolute_error(y_val, y_pred), '\n')

  print("Traning RMSE:", mean_squared_error(y_train, y_pred_train, squared=False))
  print("Validation RMSE:", mean_squared_error(y_val, y_pred, squared=False), '\n')

  print("Training R-squared score is:", model.score(XTT_train, y_train))
  print("Validation R-squared is:", model.score(XTT_val, y_val), '\n')



# + id="MXcCMrDbnw2X" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 323} outputId="dbb24b69-b618-4853-92b8-895811bee265"
features = ['YEAR_BUILT', 'ZIP_CODE_MEAN_SALE_PRICE', 'COMMERCIAL_UNITS', 'RESIDENTIAL_UNITS', 'BUILDING_CLASS_AT_PRESENT', 'TAX_CLASS_AT_PRESENT', 'BUILDING_CLASS_CATEGORY', 
            'NEIGHBORHOOD', 'GROSS_SQUARE_FEET']
model_ridge_regression(df, 'SALE_PRICE', features)

StarcoderdataJupyterScriptsDedupFiltered