# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
import os
import io
import re
import numpy as np
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
from collections import Counter
import missingno

import warnings
warnings.filterwarnings('ignore')
# -

#read data, specify index so it's easier to join and search using loc
path = '../../data/new-york-city-airbnb-open-data/'
listings_csv = os.path.join(path,'selected_columns_listings.csv')
df = pd.read_csv(listings_csv)

df.columns


df.describe()

df.drop(columns=['listing_url','name','host_name','host_url','host_url', 'host_since','host_location','host_listings_count',
          'host_listings_count','street','state','market','calculated_host_listings_count','calculated_host_listings_count_entire_homes',
        'calculated_host_listings_count_private_rooms','calculated_host_listings_count_shared_rooms' ], inplace=True)

host_count  = df['host_id'].value_counts()

fil=df['host_total_listings_count']>1
df[fil].head()

df['host_id'].value_counts()

df.groupby('host_total_listings_count').size().sort_values(ascending = False)[fil]

host_count_20=host_count[:35,]
plt.figure(figsize=(10,5))
sns.barplot(host_count_20.index, host_count_20.values, alpha=0.8)
plt.title('Number of Listing per host- Top 35')
plt.ylabel('Number of Occurrences', fontsize=12)
plt.xlabel('Host_id', fontsize=12)
plt.xticks(rotation=50)
plt.show()


host_count = host_count[:1000]
plt.figure(figsize=(10,5))
sns.distplot(host_count,color='g')
plt.title('Number of Listing per host')
plt.ylabel('Number of listing per host', fontsize=12)
plt.xlabel('Host_id', fontsize=12)
plt.show()

missingno.bar(df, labels=True) 

missingno.matrix(df)

df.isna().sum()

#visualization for type of room
sns.catplot(x='room_type', kind='count' ,data=df)
fig = plt.gcf()
fig.set_size_inches(5, 5)

#visualization for neighbourhood
sns.catplot(x='neighbourhood_group_cleansed', kind='count' ,data=df)
fig = plt.gcf()
fig.set_size_inches(5, 5)

df['price'] = pd.to_numeric(df.price.str.replace('$',''),errors='coerce')
df['host_response_rate'] = pd.to_numeric(df.host_response_rate.str.replace('%',''),errors='coerce')
df['host_acceptance_rate'] = pd.to_numeric(df.host_acceptance_rate.str.replace('%',''),errors='coerce')
df['host_is_superhost']=df.host_is_superhost.replace(['t','f'],[1,0])
df['host_identity_verified']=df.host_identity_verified.replace(['t','f'],[1,0])
df['is_location_exact']=df.is_location_exact.replace(['t','f'],[1,0])
df['security_deposit'] = pd.to_numeric(df.security_deposit.str.replace('$',''),errors='coerce')
df['cleaning_fee'] = pd.to_numeric(df.cleaning_fee.str.replace('$',''),errors='coerce') 
df['extra_people'] = pd.to_numeric(df.extra_people.str.replace('$',''),errors='coerce')
df['instant_bookable']=df.instant_bookable.replace(['t','f'],[1,0])

df.host_response_rate.fillna(df.host_response_rate.mean(), inplace=True)
df.host_acceptance_rate.fillna(df.host_acceptance_rate.mean(), inplace=True)
df.host_is_superhost.fillna(df.host_is_superhost.mean(), inplace=True)
df.host_identity_verified.fillna(df.host_identity_verified.mean(), inplace=True)
df.is_location_exact.fillna(df.is_location_exact.mean(), inplace=True)
df.security_deposit.fillna(df.security_deposit.mean(),inplace=True)
df.cleaning_fee.fillna(df.cleaning_fee.mean(),inplace=True)
df.extra_people.fillna(df.extra_people.mean(),inplace=True)

#Drop other type of bed - keep just "Real Bed"
df=df.set_index("bed_type")
df.drop(["Airbed","Couch","Futon","Pull-out Sofa"], axis=0,inplace=True)
#Drop Hotel Room from room_type
df=df[df.room_type != 'Hotel room']

df.reset_index(drop=True, inplace=True)

pd.crosstab(df.neighbourhood_group_cleansed,df.room_type,margins=True)

g_1 = sns.catplot(x="price", y="room_type", hue="neighbourhood_group_cleansed", data=df,
              height=5, kind="bar", palette="RdBu")
g_1.despine(left=False)
g_1.set_ylabels("Room_type")

pd.set_option('display.max_columns', 500)
pd.set_option('display.max_rows', 500)

corr=df.corr()



StarcoderdataJupyterScriptsDedupFiltered