# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="NqqLNJZjfi8U"
#  0. 학습 환경 설정하기
#  1. 데이터셋 불러오기
#  2. EDA
#  3. 모델 학습을 위한 데이터 전처리
#  4. 모델 학습하기
#  5. 모델 평가하기
#  6. 모델 학습 결과 심화 분석하기
#
# 출처 : 신제용 강사 (패스트캠퍼스)
#

# + [markdown] pycharm={"name": "#%% md\n"}
# ## 0 . 학습 환경 설정하기

# + id="g49RuFGrBvt7"
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import os

from sklearn.model_selection import train_test_split
from xgboost import XGBRegressor
from sklearn.metrics import mean_absolute_error, mean_squared_error
from math import sqrt

# + id="mMKFOC0OBtHO"
# os.environ을 이용하여 Kaggle API Username, Key 세팅하기
os.environ['KAGGLE_USERNAME'] = 'fastcampuskim'
os.environ['KAGGLE_KEY'] = 'c939a1e37f5ca93b6406a66fc8bb08e5'
# -

# ## 1. 데이터셋 불러오기
# * 데이터 출처: https://www.kaggle.com/dgomonov/new-york-city-airbnb-open-data
# * 컬럼
#     id: 항목의 ID
#          name: 항목의 이름 (타이틀)
#          host_id: 호스트 ID
#          host_name: 호스트의 이름
#          neighbourhood_group: 방이 있는 구역 그룹
#          neighbourhood: 방이 있는 구역
#          latitude: 방이 위치한 위도
#          longitude: 방이 위치한 경도
#          room_type: 방의 종류
#          price: 가격 (미 달러)
#          minimum_nights: 최소 숙박 일수
#          number_of_reviews: 리뷰의 개수
#          last_review: 마지막 리뷰 일자
#          reviews_per_month: 월별 리뷰 개수
#          calculated_host_listings_count: 호스트가 올린 방 개수
#          availability_365: 365일 중 가능한 일수

# + id="JSblp2NsCGbh" colab={"base_uri": "https://localhost:8080/"} outputId="153ad19e-92fa-4d97-9a82-dffbcaa55024"
# Linux 명령어로 Kaggle API를 이용하여 데이터셋 다운로드하기 (!kaggle ~)
# Linux 명령어로 압축 해제하기
# !rm *.*
# !kaggle datasets download -d dgomonov/new-york-city-airbnb-open-data
# !unzip '*.zip'

# + id="RnJv-4YwCMSx"
df = pd.read_csv('AB_NYC_2019.csv')

# + colab={"base_uri": "https://localhost:8080/", "height": 864} id="x3U_iHpNfYOF" outputId="2cee770b-3f7d-439c-888c-4e4dc62891eb"
df

# + [markdown] id="9L3BNVM7tHN5"
# ## 2. EDA
#

# + id="YcR9BX23DIFW" colab={"base_uri": "https://localhost:8080/", "height": 411} outputId="a5f8ee35-6780-4c5a-bd5f-0e6987864109"
df.head()

# + colab={"base_uri": "https://localhost:8080/"} id="_x9R5pzlniRF" outputId="123aeb0b-f9a0-4b7a-af43-9567392cda44"
df['room_type'].value_counts()

# + colab={"base_uri": "https://localhost:8080/"} id="89i38feMmx_e" outputId="fb376c96-10d5-4013-b761-7e2f136102f9"
df.info()

# + colab={"base_uri": "https://localhost:8080/"} id="ub117b7Do1H4" outputId="1f93b1a4-5753-4033-9e7d-d56d0252001f"
df.isna().sum()

# + colab={"base_uri": "https://localhost:8080/"} id="vii-v4FIn2BK" outputId="7af781e4-a392-4d6f-f32e-cb7abd11c734"
(df['reviews_per_month'].isna() & df['last_review'].isna()).sum()

# + colab={"base_uri": "https://localhost:8080/"} id="yEPkZwtyoLUy" outputId="039dabe5-b1b9-4af2-f8ac-fd88d712aebf"
df['reviews_per_month'].isna().sum()

# + colab={"base_uri": "https://localhost:8080/"} id="_1en1a5ppZck" outputId="74e8c71a-5f26-403c-ce25-7f763f572631"
(df['number_of_reviews'] == 0).sum()

# + colab={"base_uri": "https://localhost:8080/", "height": 282} id="w7Vpx5iModag" outputId="03eb3c62-7899-4a98-9626-f0a703763947"
df['availability_365'].hist()

# + colab={"base_uri": "https://localhost:8080/"} id="6GyC17vdojIY" outputId="84942c57-ac42-426b-f1ea-ea8c58f92bd5"
(df['availability_365'] == 0).sum()

# + colab={"base_uri": "https://localhost:8080/", "height": 317} id="WIoofJDVmzF1" outputId="2c93a36b-33e7-478d-8255-2e6e2786a86c"
df.describe()

# + colab={"base_uri": "https://localhost:8080/"} id="lF-wjmG2pxpb" outputId="fa91fce1-63ff-4cb4-82f9-0d0573fea01b"
df.columns

# + id="VMtkp1CYpypz"
df.drop(['id', 'name', 'host_name', 'latitude', 'longitude'], axis=1, inplace=True)

# + colab={"base_uri": "https://localhost:8080/", "height": 275} id="YwrtZAEIp7-5" outputId="f5076543-37ae-4819-ccee-3075bd86fcfb"
df.head()

# + id="b_zcrUDF7khF" colab={"base_uri": "https://localhost:8080/"} outputId="385e5ff6-0212-4a0e-d3f9-e412a990725a"
df.columns

# + colab={"base_uri": "https://localhost:8080/", "height": 458} id="4qMUlI0XqMV4" outputId="69b10dca-060a-4612-d892-82140b0064cf"
sns.jointplot(x='host_id', y='price', data=df, kind='hex')

# + colab={"base_uri": "https://localhost:8080/", "height": 458} id="JihglMQnqny9" outputId="def7f70f-0dd4-4d06-f0ed-9574d2a07c3a"
sns.jointplot(x='reviews_per_month', y='price', data=df, kind='hex')

# + id="GZXSBFPyDh6R" colab={"base_uri": "https://localhost:8080/", "height": 428} outputId="81d330b6-013a-4195-cb09-e3adb7451406"
sns.heatmap(df.corr(), annot=True, cmap='YlOrRd')

# + id="MPmzv61vTFiw" colab={"base_uri": "https://localhost:8080/"} outputId="915deba6-b822-4b86-97d3-8965d1579ffc"
df.columns

# + colab={"base_uri": "https://localhost:8080/", "height": 297} id="7gcHiC_JrupY" outputId="11a5cbc5-de03-4f37-8e7b-00d9154f8adf"
sns.boxplot(x='neighbourhood_group', y='price', data=df)

# + colab={"base_uri": "https://localhost:8080/", "height": 297} id="OzN-Dw5Yr7aP" outputId="8e067200-c703-460d-88bd-5baa07bbce99"
sns.boxplot(x='room_type', y='price', data=df)

# + colab={"base_uri": "https://localhost:8080/"} id="nWZj4hKAsw9c" outputId="3ed92a9d-3eef-47e4-c314-4288fff2089a"
df.columns

# + colab={"base_uri": "https://localhost:8080/"} id="gx7NTZ_zszN0" outputId="70f4b911-d0ff-4a73-bf98-fcaa5f6bb6e7"
df.isna().sum()

# + colab={"base_uri": "https://localhost:8080/"} id="IrGafHOYtBfy" outputId="56f39e35-5a79-4766-ebf3-b3ed3a235df1"
df['neighbourhood_group'].value_counts()

# + colab={"base_uri": "https://localhost:8080/", "height": 282} id="ZHji8nlLtJwz" outputId="fc99bbaf-abf5-4cb9-c540-b21a277e8b8f"
neigh = df['neighbourhood'].value_counts()
plt.plot(range(len(neigh)), neigh)

# + id="DFNFgntWtcFB"
df['neighbourhood'] = df['neighbourhood'].apply(lambda s: s if str(s) not in neigh[50:] else 'others')

# + colab={"base_uri": "https://localhost:8080/"} id="xAr9FZ7ttsU_" outputId="f2a15b5b-10ec-42aa-e3ee-c0099958a9f8"
df['neighbourhood'].value_counts()

# + colab={"base_uri": "https://localhost:8080/"} id="MJWdyw3Kt3q3" outputId="ed69d0bf-494c-4d84-8eaa-d608fe140574"
df['room_type'].value_counts()

# + colab={"base_uri": "https://localhost:8080/"} id="cy1jXTEQt79X" outputId="ed3e991a-b429-4229-e9b4-f52f4568accc"
df.columns

# + colab={"base_uri": "https://localhost:8080/", "height": 296} id="pCjl92shuShT" outputId="578ad294-4b5d-4c40-9430-32bcc7a63951"
sns.rugplot(x='price', data=df, height=1)

# + colab={"base_uri": "https://localhost:8080/"} id="G6PmHDEcugBE" outputId="62818e85-8eba-4d76-b044-ba59403c2745"
print(df['price'].quantile(0.95))
print(df['price'].quantile(0.005))

# + colab={"base_uri": "https://localhost:8080/", "height": 297} id="ILmJBKuavIJ4" outputId="d9a44ce9-9b57-4622-8037-811346ee00b4"
sns.rugplot(x='minimum_nights', data=df, height=1)

# + colab={"base_uri": "https://localhost:8080/"} id="-y9AtGJcvRB4" outputId="33e586d9-c2e5-453d-e1c9-bcb43e466478"
print(df['minimum_nights'].quantile(0.98))

# + colab={"base_uri": "https://localhost:8080/", "height": 297} id="9DXcLeJPvdlA" outputId="ceb12c31-96ef-4bec-b4bb-ab99fc1c5bcf"
sns.rugplot(x='availability_365', data=df, height=1)

# + colab={"base_uri": "https://localhost:8080/"} id="C0YXy8RKvl84" outputId="133a6283-b2b7-4591-cf70-4bfc6049bb59"
print(df['availability_365'].quantile(0.3))

# + id="8HbUtvqs9C-C" colab={"base_uri": "https://localhost:8080/"} outputId="8f9ebb68-8f29-4466-9acc-319fbaa64790"
# quantile(), drop() 등 메소드를 이용하여 outlier 제거하고 통계 재분석하기
p1 = df['price'].quantile(0.95)
p2 = df['price'].quantile(0.005)
print(p1, p2)

# + id="iAyiIv88wEGM"
df = df[(df['price'] < p1) & (df['price'] > p2)]

# + colab={"base_uri": "https://localhost:8080/", "height": 282} id="xHVfBeOQwNUt" outputId="85c43436-c1cb-4b63-d06c-08920ac86760"
df['price'].hist()

# + colab={"base_uri": "https://localhost:8080/"} id="EB6s7-qawSRT" outputId="b7e9f5ee-e1be-422b-f59a-7ef23edd8d4d"
mn1 = df['minimum_nights'].quantile(0.98)
print(mn1)

# + id="v2KhzoXmwddb"
df = df[df['minimum_nights'] < mn1]

# + colab={"base_uri": "https://localhost:8080/", "height": 282} id="7e9l9co0w5mt" outputId="aeb7b77c-d0be-42d7-97d2-ff110529ec52"
df['minimum_nights'].hist()

# + id="SxcndS2ExTmU"
df['is_avail_zero'] = df['availability_365'].apply(lambda x: 'Zero' if x==0 else 'Nonzero')

# + id="NW-QEym6lgtX"
# fill(), dropna() 등으로 미기입된 데이터를 처리하기
df['review_exists'] = df['reviews_per_month'].isna().apply(lambda x: 'No' if x is True else 'Yes')

# + id="m8fGPMwVyyi_"
df.fillna(0, inplace=True)

# + colab={"base_uri": "https://localhost:8080/"} id="JgWVgoKZy4Zm" outputId="7a405436-81b4-4f25-dcf2-8da09b86d076"
df.isna().sum()

# + [markdown] id="FRfd7ABjepBS"
# ## 3. 모델 학습을 위한 데이터 전처리

# + colab={"base_uri": "https://localhost:8080/"} id="Q8hDQjWizbCV" outputId="bddf8a90-821b-483e-f7ce-3d7def8c4a9a"
df.columns

# + id="wVmEa1ChlrTc"
X_cat = df[['neighbourhood_group', 'neighbourhood', 'room_type', 'is_avail_zero', 'review_exists']]
X_cat = pd.get_dummies(X_cat)

# + id="_k_SDCh5xMgD"
from sklearn.preprocessing import StandardScaler

# + colab={"base_uri": "https://localhost:8080/"} id="XxBgGzGYzxh8" outputId="bf4252dc-7770-4dcc-b78c-f6faf21938f9"
df.columns

# + id="W3EO22NCE3wG"
# StandardScaler를 이용해 수치형 데이터를 표준화하기
scaler = StandardScaler()
X_num = df.drop(['neighbourhood_group', 'neighbourhood', 'room_type', 'price', 
                 'last_review', 'is_avail_zero', 'review_exists'], axis=1)

scaler.fit(X_num)
X_scaled = scaler.transform(X_num)
X_scaled = pd.DataFrame(X_scaled, index=X_num.index, columns=X_num.columns)

X = pd.concat([X_scaled, X_cat], axis=1)
y = df['price']

# + colab={"base_uri": "https://localhost:8080/", "height": 241} id="ddgwL7Q90kAh" outputId="75e2d7b3-3f47-4423-d7c8-c4463eb316d8"
X.head()

# + id="F07QjOFwFNEw"
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=1)

# + [markdown] id="RrWrE8Z4exup"
# ## 4. 모델 학습하기

# + id="LSSNqFUrGM6R" colab={"base_uri": "https://localhost:8080/"} outputId="d9b22b82-6a2b-4da7-a374-30a17c63a8aa"
model_reg = XGBRegressor()
model_reg.fit(X_train, y_train)

# + [markdown] id="gUo8NmHkfIpf"
# ## 5. 모델 평가하기

# + id="KDVy7fFGfUP1" colab={"base_uri": "https://localhost:8080/"} outputId="bc729969-4d2a-4a78-abff-bc6d09553a1c"
pred = model_reg.predict(X_test)
print(mean_absolute_error(y_test, pred))
print(sqrt(mean_squared_error(y_test, pred)))

# + [markdown] id="DTqb-HqPtc4I"
# ## 6. 모델 학습 결과 심화 분석하기
#

# + id="kKEP06-OmrBs" colab={"base_uri": "https://localhost:8080/", "height": 282} outputId="dc44dbfa-ec1a-4581-b11c-7d33e7bc0659"
# y_test vs. pred Scatter 플랏으로 시각적으로 분석하기
plt.scatter(x=y_test, y=pred, alpha=0.1)
plt.plot([0,350], [0, 350], 'r-')

# + id="WLnyYNJwGRgd" colab={"base_uri": "https://localhost:8080/", "height": 279} outputId="34b35d2c-f6c1-4e3b-f25f-07077aae1b61"
# err의 히스토그램으로 에러율 히스토그램 확인하기
err = (pred - y_test) / y_test
sns.histplot(err)
plt.grid()

# + colab={"base_uri": "https://localhost:8080/", "height": 279} id="7hhxUrZ12STu" outputId="fc23057a-78e3-4206-8b2f-2075f78297dc"
# err의 히스토그램으로 에러율 히스토그램 확인하기
err = pred - y_test
sns.histplot(err)
plt.grid()

# + [markdown] id="0w8vUO602pOk" pycharm={"name": "#%% md\n"}
# ## 참고
# * 퀴즈처럼 풀면서 배우는 파이썬 머신러닝 300제

# + pycharm={"name": "#%%\n"}


StarcoderdataJupyterScriptsDedupFiltered