# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="AO1C4kqKsN5A" colab_type="code" colab={}
import pandas as pd
import numpy as np
from sklearn.tree import DecisionTreeRegressor
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import cross_val_score

# + id="Gc_NWjpptLnv" colab_type="code" outputId="a219003a-afb5-4af7-c4ff-513bb431dded" executionInfo={"status": "ok", "timestamp": 1581616837329, "user_tz": -60, "elapsed": 847, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 35}
# cd /content/drive/My Drive/Colab Notebooks/dw_matrix

# + id="rpfk2ybqta7X" colab_type="code" outputId="37d6c6b5-11db-4a22-d17c-d3f3911a3c83" executionInfo={"status": "ok", "timestamp": 1581616850536, "user_tz": -60, "elapsed": 3850, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 35}
# ls data

# + id="ulsLnqIZtdWq" colab_type="code" colab={}
df = pd.read_csv('data/men_shoes.csv', low_memory=False)

# + id="mNU22YiAtqoA" colab_type="code" outputId="b7b34293-c4f7-4f81-bf71-5d6b305ebc8a" executionInfo={"status": "ok", "timestamp": 1581616970409, "user_tz": -60, "elapsed": 947, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 35}
df.shape

# + id="QNWeIO5wttpn" colab_type="code" outputId="50efcc42-5fdc-46a2-f8c6-73f0f9583518" executionInfo={"status": "ok", "timestamp": 1581617011735, "user_tz": -60, "elapsed": 962, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 235}
df.columns

# + id="nTSMrKHauB9r" colab_type="code" outputId="adf71b3e-4e0e-4c9a-b7bc-d8aadbe8e715" executionInfo={"status": "ok", "timestamp": 1581617168343, "user_tz": -60, "elapsed": 1144, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 35}
mean_price = np.mean(df['prices_amountmin'])
mean_price

# + id="-giX2gt4ufhy" colab_type="code" colab={}
y_true = df['prices_amountmin']
y_pred = [mean_price] * y_true.shape[0]

# + id="HtOa2jPYvSUK" colab_type="code" outputId="8ba4c65d-2e6e-4867-cad7-d3a2a3c7ffb5" executionInfo={"status": "ok", "timestamp": 1581617395691, "user_tz": -60, "elapsed": 885, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 35}
mean_absolute_error(y_true, y_pred)

# + id="-0rngDxsvjOB" colab_type="code" outputId="173ee0f1-702c-4642-8ef2-a96d2cee206b" executionInfo={"status": "ok", "timestamp": 1581617535428, "user_tz": -60, "elapsed": 851, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 283}
np.log1p(df['prices_amountmin']).hist(bins=100)

# + id="IjMu-UzTv0R-" colab_type="code" outputId="cb982f24-2b4c-4e6c-ed1b-ae0e2597ae24" executionInfo={"status": "ok", "timestamp": 1581617695243, "user_tz": -60, "elapsed": 1644, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 35}
y_true = df['prices_amountmin']
y_pred = [np.median(y_true)] * y_true.shape[0]
mean_absolute_error(y_true, y_pred)

# + id="_qSDb01ywsJ_" colab_type="code" outputId="317a5d85-58a7-41d5-8c18-334aa158237a" executionInfo={"status": "ok", "timestamp": 1581617711815, "user_tz": -60, "elapsed": 962, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 35}
np.median(y_true)

# + id="g94yhYpbwwY6" colab_type="code" outputId="0437b895-cfb3-45f0-c847-be9409c33c1a" executionInfo={"status": "ok", "timestamp": 1581618080376, "user_tz": -60, "elapsed": 936, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 35}
y_true = df['prices_amountmin']
price_log_mean = np.expm1(np.mean(np.log1p(y_true)))
y_pred = [price_log_mean] * y_true.shape[0]

mean_absolute_error(y_true, y_pred)

# + id="36TxNsT3yKWY" colab_type="code" outputId="187d3013-67f4-4d4f-8f73-0389f45613d9" executionInfo={"status": "ok", "timestamp": 1581618224656, "user_tz": -60, "elapsed": 868, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 235}
df.brand.value_counts()

# + id="czQxownsytmB" colab_type="code" outputId="7703f17c-fba0-4f21-e5a1-9bafc09c84ee" executionInfo={"status": "ok", "timestamp": 1581618323280, "user_tz": -60, "elapsed": 1364, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 35}
df['brand'].factorize()[0]

# + id="00HPYgSQzFjG" colab_type="code" colab={}
df['brand_cat']=df['brand'].factorize()[0]

# + id="2esFTQDVzSKn" colab_type="code" outputId="42b3420b-01ca-4127-f56a-f045449e27a2" executionInfo={"status": "ok", "timestamp": 1581618732657, "user_tz": -60, "elapsed": 1125, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 35}
feats = ['brand_cat']
X = df[feats].values
y = df['prices_amountmin'].values

model = DecisionTreeRegressor(max_depth=5)

scores = cross_val_score(model, X, y, scoring='neg_mean_absolute_error')
np.mean(scores), np.std(scores)


# + id="FoTTBlTc0pj0" colab_type="code" colab={}
def run_model(feats):
  X = df[feats].values
  y = df['prices_amountmin'].values

  model = DecisionTreeRegressor(max_depth=5)

  scores = cross_val_score(model, X, y, scoring='neg_mean_absolute_error')
  return np.mean(scores), np.std(scores)


# + id="QGc9Rtlm1P6X" colab_type="code" outputId="f5c41b15-7d4d-4729-a8b5-5119bde6ef0a" executionInfo={"status": "ok", "timestamp": 1581618915605, "user_tz": -60, "elapsed": 2033, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 35}
run_model(['brand_cat'])

# + id="OY-Y9iUp1V_w" colab_type="code" colab={}
df['manufacturer_cat']=df['manufacturer'].factorize()[0]

# + id="E4jdjKkX2KR3" colab_type="code" outputId="5e067ac8-0536-4c49-e91d-6ef299dbbc18" executionInfo={"status": "ok", "timestamp": 1581619136933, "user_tz": -60, "elapsed": 904, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 235}
df['manufacturer_cat']

# + id="uPixYsDC2MVg" colab_type="code" outputId="e30e6f29-d641-4df7-ba34-49c3dc9d55e3" executionInfo={"status": "ok", "timestamp": 1581619231064, "user_tz": -60, "elapsed": 1030, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 35}
run_model(['manufacturer_cat'])

# + id="ZFRMRADk2jRe" colab_type="code" outputId="c4e78da4-3f32-4ec8-cb1a-4db3455b699f" executionInfo={"status": "ok", "timestamp": 1581619334387, "user_tz": -60, "elapsed": 1339, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}} colab={"base_uri": "https://localhost:8080/", "height": 35}
run_model(['brand_cat', 'manufacturer_cat'])

# + id="-xQopl5k2wRW" colab_type="code" colab={}
# !git config --global user.email "<EMAIL>"
# !git config --global user.name "betanot"

# + id="0ur3fLz93atW" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="a1dec328-59ca-43f4-e447-84ee2e94923e" executionInfo={"status": "ok", "timestamp": 1581619606874, "user_tz": -60, "elapsed": 4378, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "00951016543650194842"}}
# !git add dw_matrix/day4.ipynb

# + id="KQYx8Z4y3jnz" colab_type="code" colab={}
pwd


StarcoderdataJupyterScriptsDedupFiltered