# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="b-AcezaqHBBJ" colab_type="code" colab={}
import pandas as pd
import numpy as np
from sklearn.tree import DecisionTreeRegressor # model który zawiera szereg warunków
from sklearn.metrics import mean_absolute_error # w jaki spoób mierzymy sukces, czyli funkcja sprawdzjąca czy posuwamy sie do przodu czy stoimy w miejscu
from sklearn.model_selection import cross_val_score

# + id="b2tOynoKIX7t" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="c3625771-6ce4-402b-a1f0-c77dc9540ce0" executionInfo={"status": "ok", "timestamp": 1581623951462, "user_tz": -60, "elapsed": 832, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
# cd /content/drive/My\ Drive/Colab Notebooks/Matrix_DW

# + id="Yj4dxuejIjwT" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="323f4cc3-2fc5-4b36-f360-cebcc9806034" executionInfo={"status": "ok", "timestamp": 1581623962602, "user_tz": -60, "elapsed": 2346, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
# ls data

# + id="NjjpBQZRImCE" colab_type="code" colab={}
df = pd.read_csv('data/shoes_prices.csv', low_memory=False)

# + id="NkuiRAnCJG9J" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="12e76473-01bb-4dbc-c43a-6590ee008edb" executionInfo={"status": "ok", "timestamp": 1581624149662, "user_tz": -60, "elapsed": 809, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
df.shape

# + id="O9R7CqLLJUDa" colab_type="code" colab={}
df_usd = df[df.prices_currency == 'USD'].copy()

# + id="z4i2XZqwKalP" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="422635bd-a047-41c3-fc0b-484ccf165d47" executionInfo={"status": "ok", "timestamp": 1581624452435, "user_tz": -60, "elapsed": 726, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
df_usd.shape

# + id="4njNCoZoWZoB" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 283} outputId="0f208d30-4e28-40b7-c45a-21be15483f4c" executionInfo={"status": "ok", "timestamp": 1581627977198, "user_tz": -60, "elapsed": 1117, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
df_usd['prices_amountmin'] = df_usd.prices_amountmin.astype(np.float)
max_price_filter = np.percentile(df_usd['prices_amountmin'], 99)
max_price_filter
df_usd_filter = df_usd[df_usd['prices_amountmin'] < max_price_filter]
df_usd_filter.prices_amountmin.hist(bins=100)

# + id="ZRJpRBDPYKDn" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="69736298-3d24-41a2-df8f-cc3ea2a4c9d9" executionInfo={"status": "ok", "timestamp": 1581628122194, "user_tz": -60, "elapsed": 3234, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
# ls

# + id="WbJPS_ixKc5Y" colab_type="code" colab={}
df_usd_filter.to_csv('data/valid_men_shoes.csv', index=False)

# + id="trKzBdOvZEFP" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="aa932ff2-0cb4-450b-ff5f-1f747b599661" executionInfo={"status": "ok", "timestamp": 1581628378271, "user_tz": -60, "elapsed": 3074, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
# ls data/

# + id="bWoZA0fzL_w1" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="74778600-69bc-4115-95d9-8e2978ff9c6b" executionInfo={"status": "ok", "timestamp": 1581628399261, "user_tz": -60, "elapsed": 1546, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
df = pd.read_csv('data/valid_men_shoes.csv', low_memory=False)
df.shape

# + id="biqdh4Kcp8IY" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 233} outputId="bfd5b208-3885-4466-b0c9-a64ca900a61e" executionInfo={"status": "ok", "timestamp": 1581632708197, "user_tz": -60, "elapsed": 1543, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
df.columns

# + id="D6v2x5STMhz8" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="c2f1193b-2e33-4f2b-d32c-592b8817e10e" executionInfo={"status": "ok", "timestamp": 1581628410524, "user_tz": -60, "elapsed": 698, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
mean_price = np.mean(df['prices_amountmin'])
mean_price

# + id="pmPywes_SA_u" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="0c602993-fcb3-47b0-fcc3-08f79d99b9ea" executionInfo={"status": "ok", "timestamp": 1581628418656, "user_tz": -60, "elapsed": 702, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
y_true = df['prices_amountmin']
y_pred = [mean_price] * y_true.shape[0]
mean_absolute_error(y_true, y_pred)

# + id="YsKMl8UNTNrg" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 283} outputId="9456820f-0041-4f12-ed74-5eb25868cea1" executionInfo={"status": "ok", "timestamp": 1581628423299, "user_tz": -60, "elapsed": 1068, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
df['prices_amountmin'].hist(bins=100)

# + id="bqpaA3EcVAAh" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 283} outputId="99ed7efa-8097-42aa-c6fe-322642b4fa80" executionInfo={"status": "ok", "timestamp": 1581629029612, "user_tz": -60, "elapsed": 871, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
np.log(df['prices_amountmin']+1).hist(bins=100)  #dodajemy jedynke bo wynika to z wlasciwosci logarytmow - log(0) daje minus nieskonczoność a dopiero log(1) daje nam zero
# np.log1p(<value>) -> działa taks samo jak np.log(<value> +1)

# + id="BQaedTNQa1P-" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="0991cb00-160f-41c5-9b7b-24c9d54ae77b" executionInfo={"status": "ok", "timestamp": 1581629656828, "user_tz": -60, "elapsed": 873, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
y_true = df['prices_amountmin']
median_value = np.median(y_true) # mediana mowi nam ze 50% butow kosztuje 64.95 USD(wynik mediany) lub mniej. Dzieki temu czesciej trafiamy w wiekszośćprzypadków (odcinamy sie od pewnych skrajnych danych)
# median_value
y_pred = [median_value] * y_true.shape[0]
mean_absolute_error(y_true, y_pred)

# + id="Wn5ryY1_cyhI" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="fdd794d1-1457-49db-9364-308656611af4" executionInfo={"status": "ok", "timestamp": 1581630030467, "user_tz": -60, "elapsed": 723, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
y_true = df['prices_amountmin']
# np.mean(np.log1p(y_true)) -> 4.157759197235119
price_log_mean = np.expm1(np.mean(np.log1p(y_true))) # 62.92811169273113
y_pred = [price_log_mean] * y_true.shape[0]
mean_absolute_error(y_true, y_pred)

# + id="xFKuMnbzfRnE" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 233} outputId="0da0e9ff-3b80-4061-9cb3-83c0d6f5c1aa" executionInfo={"status": "ok", "timestamp": 1581632515785, "user_tz": -60, "elapsed": 1984, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
df.columns

# + id="bFSfVSTHpHZ6" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 143} outputId="8250fdf4-2e55-40e6-81f1-4c3da9bc1554" executionInfo={"status": "ok", "timestamp": 1581632547019, "user_tz": -60, "elapsed": 846, "user": {"displayName": "<NAME>0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
df.brand.value_counts() # zlicza ilości poszczegolnych marek
# ML potrzebuje liczb i nie mozemy mu przekazac takich danych jak poniżej np.NIKE. W tym celu musimy przypisac 
# poszcegolnym brandom jakies id.
#Mozna to zrobic ręcznie {'NIKE': 1, ...} ale mamy do tego gotowa funkcje:
df.brand.factorize() # pierwsza lista to id a druga przypisane wartości (klucz->wartosc). id jest przypisane w sposob spojny

# + id="24IgLVsRpWP4" colab_type="code" colab={}
# tworzymy nowa kolumne 'brand_category' ktora jest id'kami dla brandow 
# NIKE to brand a np. 0 to brand_category
df['brand_category'] = df.brand.factorize()[0] # array([  0,   0,   1, ..., 411, 411, 411])

# + id="X_V7OPI-pbpE" colab_type="code" colab={}
#tutaj nowo utworzona kolumne przypisujemy do zmiennej i wyciagamy wartosci z tej kolumny, czyli nasze powyższe id'ki
feats = ['brand_category']
X = df[feats].values # wazne jest DUZE X
y = df['prices_amountmin'].values

model = DecisionTreeRegressor(max_depth=5)

# + id="aK_9zSqbpl7_" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 55} outputId="715593d1-6833-4629-e211-5bb68566cf5d" executionInfo={"status": "ok", "timestamp": 1581632614308, "user_tz": -60, "elapsed": 1087, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
# stad bierzemy zmienna scoring
import sklearn
sklearn.metrics.SCORERS.keys()

# + id="5fJjThs3pmqf" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="b82f9d2a-a40a-4988-b9ea-9b7499b21ac0" executionInfo={"status": "ok", "timestamp": 1581632630285, "user_tz": -60, "elapsed": 734, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
scores = cross_val_score(model, X, y, scoring='neg_mean_absolute_error')
# array([-57.89947522, -57.73227208, -52.50719535, -58.06843816, -65.72540391])
np.mean(scores), np.std(scores)


# + id="loUjDQGWpqpO" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="1f3882c6-73b8-4a1d-a9fa-5dcb12a67ff5" executionInfo={"status": "ok", "timestamp": 1581632978447, "user_tz": -60, "elapsed": 762, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
# zapisanie powyższych wyliczen dla nowej kolumny do funkcji
def run_model(feats):
  X = df[feats].values
  y = df['prices_amountmin'].values

  model = DecisionTreeRegressor(max_depth=5)
  scores = cross_val_score(model, X, y, scoring='neg_mean_absolute_error')
  return np.mean(scores), np.std(scores)

run_model(['brand_category'])

# + id="CiEBxS-tq_nw" colab_type="code" colab={}
df['name_category'] = df.name.factorize()[0]

# + id="UsMflagYskMY" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="5d86e857-3b18-42ad-87bf-ba705fd8dce7" executionInfo={"status": "ok", "timestamp": 1581634116147, "user_tz": -60, "elapsed": 699, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
run_model(['name_category'])

# + id="crRrpqPJsqGi" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="2313c978-5d29-417e-c024-3d8653020636" executionInfo={"status": "ok", "timestamp": 1581634126365, "user_tz": -60, "elapsed": 738, "user": {"displayName": "<NAME>\u0142\u0105zka", "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mDyS7mGyX9BQobokNUgLgh6YtiGZ6q5q2riNKFBsQ=s64", "userId": "00171644978004066615"}}
run_model(['brand_category', 'name_category'])

# + id="ZDyym7QuuO0u" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="9000d8b5-2e1d-4f90-c376-b160c377a128"
# ls

StarcoderdataJupyterScriptsDedupFiltered