# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # INTRODUCTION
#
# Will be going through AirBnB data for Seattle.
#

# %pylab inline
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from decimal import Decimal
import datetime
from sklearn import linear_model
from sklearn.model_selection import train_test_split
from sklearn.metrics import r2_score
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import RandomForestRegressor
import seaborn as sns
import statsmodels.api as sm
from sklearn.metrics import mean_squared_error
from math import sqrt
from itertools import *
import os
from IPython.display import HTML, display

# # DATA UNDERSTANDING

#Reading the Data from CSV using panda
calendar = pd.read_csv('calendar.csv')
listing = pd.read_csv('listings.csv')
reviews = pd.read_csv('reviews.csv')

calendar.head()

reviews.head()

listing.head()

listing['price'].values


# # DATA PREPARATION
# Creating a function that removes '$' and percentage sing from the columns to work with.

# +
# there is need to have a function that will remove '$' from currency columns

def remove_dollar_sign(data,column_name):
    column =  pd.to_numeric(data[column_name].apply(lambda x: str(x).replace('$', '').replace(',', '')), errors='coerce')
    return column

def remove_percentage_sign(data,column_name):
    column =  pd.to_numeric(data[column_name].apply(lambda x: str(x).replace('%', '').replace(',', '')), errors='coerce')
    return column


# +
calendar = calendar.dropna(axis = 0, subset = ['price'], how = 'any')
reviews = reviews.dropna(axis = 0, subset = ['comments'], how = 'any')
calendar['date'] = pd.to_datetime(calendar['date'])
calendar['month'] = calendar.date.dt.month
calendar['year'] = calendar.date.dt.year
calendar['day'] = calendar.date.dt.day
calendar['price'] = pd.to_numeric(calendar['price'].apply(lambda x: str(x).replace('$', '').replace(',', '')), errors='coerce')

listing['monthly_price'] = remove_dollar_sign(listing,'monthly_price')
listing['weekly_price'] = remove_dollar_sign(listing,'weekly_price')

listing['price'] = remove_dollar_sign(listing,'price')

listing['cleaning_fee'] = remove_dollar_sign(listing,'cleaning_fee')


listing['security_deposit'] = remove_dollar_sign(listing,'security_deposit')

listing['extra_people'] = remove_dollar_sign(listing,'extra_people')
listing = listing.rename(columns = {'id':'listing_id'})




# -

# # DATA REVIEW
# Reviewing each of the data (listing, reviews and calendar), there seems to be some 'NaN' values in calendar and reviews while listing data seems to be more informative. I will take a deep dive into the listing data as it has more accurate information. Also, there is need to remove '$' sign from the price in order to have a floating column to work with.

listing.head()

# Reviewing the columns to know the percentage of missing values.

(listing.isnull().sum()[listing.isnull().sum().nonzero()[0]])/len(listing) 

# The following columns will be neeeded for our insight.
#
# property_type 
# neighbourhood 
# review_scores_value 
# bathrooms 
# bedrooms 
# price 

# +
default_list = listing[['property_type', 'neighbourhood', 'review_scores_value', 
                        'bathrooms', 'bedrooms', 'price','latitude','longitude']]

new_list = default_list.dropna(axis = 0, how = 'any')
new_list


# -

# Using seaborn to plot graph for our data.

# The function will be used to plot seaborn data.
def Plot(cur, data_list):
    plt.figure(figsize=(20,20))
    plt.xticks(rotation=90)
    sns.countplot((data_list)[(cur)],
                 order = data_list[cur].value_counts().index)
    plt.show()


(new_list.property_type.value_counts())/(new_list.property_type.count())

Plot('property_type', new_list)

(new_list.neighbourhood.value_counts())/(new_list.neighbourhood.count())

Plot('neighbourhood', new_list)

# From the above graphs, i noticed that there are propert types that are in mostly listed in the dataset while some are fairly listed. Due to this, there is need to mini or limit the number of features to work with while maintaining data integrity. 

# Reducing the neighbourhood to TOP 27.
new_list_neighbourhood = new_list.groupby('neighbourhood').count()
new_list_top_15_neighbourhood = new_list_neighbourhood.nlargest(27,'property_type')
new_list_top_15_neighbourhood

# creating a list for the reduced neighbouhood.
neighbourhood_list = ['Capitol Hill',
'Ballard',
'Belltown',
'Minor',
'Queen Anne',
'Fremont',
'Wallingford',
'First Hill',
'North Beacon Hill',
'University District',
'Stevens',
'Central Business District',
'Lower Queen Anne',
'Greenwood',
'Columbia City',
'Ravenna',
'Magnolia',
'Atlantic',
'North Admiral',
'Phinney Ridge',
'Green Lake',
'Leschi',
'Mount Baker',
'Eastlake',
'Maple Leaf',
'Madrona',
'Pike Place Market']

# Using the neighbourhood_list above, we filter out some of the data not needed.

# +
#This gives me a list of True/False statements for each row if the value in the neighbourhood columns is in 
#neighbourhood_list above.
true_false_by_neighbourhood = new_list.neighbourhood.isin(neighbourhood_list) 

#I can then put this new list of True/False statements into our origional new_list. This filters new_list down
# to 37 categories of the neighbourhood column while reataining about 80% of the origional data.
filtered_neighborhood = new_list[true_false_by_neighbourhood]
filtered_neighborhood
# -

#Applying the same for 'property_type'
new_list_property_type = new_list.groupby('property_type').count()
new_list_top_16_property_type = new_list_property_type.nlargest(16, 'neighbourhood')
new_list_top_16_property_type

# reducing the 'property_type' as represented by the top listed neighbourhoods.
property_type_list = ['House', 'Apartment', 'Townhouse', 'Condominium', 'Loft', 'Bed & Breakfast', 'Cabin']

# +
#This gives me a list of True/False statements for each row if the value in the property_type column is in 
#the property_type_list above.
true_false_by_property = filtered_neighborhood.property_type.isin(property_type_list) 

#I can then put this new list of True/False statements into our filtered_neighbourhood list. 
#This filters new_list down seven property types while still containing 98.30% of the data of filtered_neighbourhood.
filtered_data = filtered_neighborhood[true_false_by_property]
filtered_data


# -

# Filtering data base on property type.
def Filterlist(cur, data):
    """Description: This function can be used to read the file and filter based on the input.
    
    Arguments: 
    cur: the cursor object.
    data: data file
    
    Returns: 
    The data file that is filtered by the cur object."""
    
    property_type = [cur]
    true_false_by_property = filtered_data.property_type.isin(property_type) 
    List = filtered_data[true_false_by_property]
    return List


# Measuring how expensive each neighbourhood is. The is based the seen correlation between prices and number of bedrooms.

sns.lmplot(data=filtered_data, x='bedrooms', y='price', hue='review_scores_value')

# The graphs shows that there is a correlation between number of rooms and prices. In other to dig deep, there is need to really get the average price per bedroom so that the possiblites of a neighbourhood becoming more expensive than the other base on higher number of rooms will be eliminated.

room_premium = (filtered_data.price)/(filtered_data.bedrooms)
filtered_data['Cost Per Bedroom'] = filtered_data['latitude'].add(room_premium)
filtered_data

# filtering data by house.
House_list = Filterlist('House', filtered_data)
House_list


# # DATA MODELING
# Let's get an insight on the data so far.

def catplot(x_data, y_data, data_list):
    """Description: This function can be used to read the file in the data_list and create a catplot based on the
    x_data and y_data.
    
    Arguments: 
    x_data: Column in data_list that you want to plot on the x axis (Put as string). 
    y_ data: Column in the data_list that you want to plot on the y axis (Put as string).
    data_list: The datalist that contains the x_data and y_data as columns."""
    #sns.catplot(x=x_data, y=y_data, data=data_list, height=8)
    plt.figure(figsize=(10, 8))
    sns.stripplot(x=x_data, y=y_data, data=data_list, size=5)


catplot('Cost Per Bedroom', 'neighbourhood', House_list)
#draw_histogram_data(House_list)

Apartment_list = Filterlist('Apartment', filtered_data)
Apartment_list

catplot('Cost Per Bedroom', 'neighbourhood', Apartment_list)

# It seems that there is much more varability in terms of price for houses depending on the neighbourhood.
#
# Queen Anne, Ballard, Minor, Capitol Hill, and Stevens looked among the most expensive neighbourhoods. However, that is debatable since the varability is so high.
#
# For Apartments there is less varability in the price per room.
#
# Queen Anne, Ballard, Lower Queen Anne, Minor, Capitol Hill, Pike Place Market, Central Business District, Belltown, and Firsthill ranked among the most expensive.
#

# # PRIDICTING PRICE

df = listing[["host_response_rate", "host_acceptance_rate", "host_is_superhost",
               "host_listings_count", "zipcode", "property_type","room_type", "accommodates", "bathrooms", "bedrooms", 
               "beds", "price", "number_of_reviews", "review_scores_rating", "cancellation_policy", 
               "reviews_per_month"]]
df2=df.dropna(axis=0)

non_num_vars = df2.select_dtypes(include=['object']).columns
df2[non_num_vars].head()

#clean data to be used for predictions
pd.options.mode.chained_assignment = None  # default='warn'
df2['host_acceptance_rate'] = remove_percentage_sign(df2,'host_acceptance_rate')
df2['host_response_rate'] = remove_percentage_sign(df2,'host_response_rate')

df2['superhost']=np.where(df2['host_is_superhost']=='t',1,0)
del df2['host_is_superhost']

# select non-numeric variables and create dummies
non_num_vars = df2.select_dtypes(include=['object']).columns
df2[non_num_vars].head()

dummy_vars = pd.get_dummies(df2[non_num_vars])
dummy_vars.head()

# drop non-numeric variables from df2 and add the dummies
df3=df2.drop(non_num_vars,axis=1)
df3 = pd.merge(df3,dummy_vars, left_index=True, right_index=True)
df3.head()

# Some summary stats
df3['price'].hist(bins=50)
plt.ylabel('Count')
plt.xlabel('Listing price in $')
plt.title('Histogram of listing prices')

cols = ['number_of_reviews','host_acceptance_rate','host_listings_count','accommodates',
       'bathrooms','bedrooms','beds','price']

# # Visualizing correlations through heatmap

# heatmap
corrs = np.corrcoef(df3[cols].values.T)
sns.set(font_scale=1)
hm=sns.heatmap(corrs, cbar = True, annot=True, square = True, fmt = '.2f',
              yticklabels = cols, xticklabels = cols)

# # Prediction Examples

# +
# split into test and training data
np.random.seed(1)
indices = np.random.permutation(len(df3))
train_size = int(round(0.8*len(df3)))
test_size = len(df3)-train_size

y = df3['price']
x = df3.drop('price', axis =1)

x.train = x.iloc[indices[0:train_size]]
y.train = y.iloc[indices[0:train_size]]
x.test = x.iloc[indices[train_size+1:]]
y.test = y.iloc[indices[train_size+1:]]

x2 = x.train.as_matrix()
y2 = y.train.as_matrix()

# +
#Applying statistical model on training model.

olsmod = sm.OLS(y2,x2)
olsres = olsmod.fit()
print(olsres.summary())
# -

x0 = x.test.as_matrix()
y0 = y.test.as_matrix()

ypred = olsres.predict(x0) # out of sample prediction
rms_ols = sqrt(mean_squared_error(y0,ypred))

rms_ols

# plot predictions agains true values as function of bedrooms
beds = x.test['bedrooms'].as_matrix()

fig, ax = plt.subplots()
ax.plot(beds,y0, 'o', label = 'True Data')
ax.plot(beds, ypred, 'o', label = 'Prediction')
ax.legend(loc = 'best')

# +
# different method

X_train, X_test, y_train, y_test = train_test_split(x, y, test_size = 0.25, random_state=1)
slr = linear_model.LinearRegression()
slr.fit(X_train, y_train)
y_train_pred = slr.predict(X_train)
y_test_pred = slr.predict(X_test)
# -

plt.scatter(y_train_pred,  y_train_pred - y_train,
            c='blue', marker='o', label='Training data')
plt.scatter(y_test_pred,  y_test_pred - y_test,
            c='lightgreen', marker='s', label='Test data')
plt.xlabel('Predicted values')
plt.ylabel('Residuals')
plt.legend(loc='upper left')
plt.show()

# +

rms_ols2=sqrt(mean_squared_error(y_test,y_test_pred))
print('MSE train: %.3f, test: %.3f' % (
        mean_squared_error(y_train, y_train_pred),
        mean_squared_error(y_test, y_test_pred)))
print('R^2 train: %.3f, test: %.3f' % (
        r2_score(y_train, y_train_pred),
        r2_score(y_test, y_test_pred)))
# -

# # Random forest

# +

forest = RandomForestRegressor(n_estimators=500, 
                               criterion='mse', 
                               random_state=3, 
                               n_jobs=-1)
forest.fit(X_train, y_train)
y_train_pred = forest.predict(X_train)
y_test_pred = forest.predict(X_test)

print('MSE train: %.3f, test: %.3f' % (
        mean_squared_error(y_train, y_train_pred),
        mean_squared_error(y_test, y_test_pred)))
print('R^2 train: %.3f, test: %.3f' % (
        r2_score(y_train, y_train_pred),
        r2_score(y_test, y_test_pred)))
# -

plt.scatter(y_train_pred,  y_train_pred - y_train,
            c='blue', marker='o', label='Training data')
plt.scatter(y_test_pred,  y_test_pred - y_test,
            c='lightgreen', marker='s', label='Test data')
plt.xlabel('Predicted values')
plt.ylabel('Residuals')
plt.legend(loc='upper left')
plt.show()

rmse_randfor=sqrt(mean_squared_error(y_test,y_test_pred))

compare = pd.Series( [rms_ols, rms_ols2, rmse_randfor], index = ["ols1","ols2","random forest"])
compare

StarcoderdataJupyterScriptsDedupFiltered