# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="LKmvBxbbqkBl" colab_type="code" outputId="20273003-d1ba-4e4e-8115-bd8722d2b658" executionInfo={"status": "ok", "timestamp": 1589316596595, "user_tz": -120, "elapsed": 8029, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 34}
# %tensorflow_version  2.x
# ! pip install -q keras==2.3.0

# + id="Myjgcxy8O7Lq" colab_type="code" colab={}
# Per sapere che GPU è assegnata 

from tensorflow.python.client import device_lib
device_lib.list_local_devices()

# + id="ipf_kroDqtIb" colab_type="code" outputId="af060099-1d97-4393-decc-e06bc3154f43" executionInfo={"status": "ok", "timestamp": 1589316598698, "user_tz": -120, "elapsed": 10117, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 87}
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

from sklearn.metrics import make_scorer, classification_report
from sklearn.model_selection import cross_val_score, KFold, train_test_split
from sklearn.preprocessing import LabelEncoder, LabelBinarizer
from sklearn.multiclass import OneVsRestClassifier


from xgboost.sklearn import XGBClassifier
import xgboost

from keras import models, layers, optimizers, metrics, callbacks


from keras.wrappers.scikit_learn import KerasClassifier

# + id="lHYvxTCmqvb2" colab_type="code" outputId="9d428b8d-6dea-4ec0-ddc1-f483ea5eefe6" executionInfo={"status": "ok", "timestamp": 1589316616785, "user_tz": -120, "elapsed": 28191, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 121}
from google.colab import drive
drive.mount('/content/drive')


# + id="x8NTlaL5ac6X" colab_type="code" colab={}
# Funzione per standardizzare
def standardize(data, columns):
    for i in columns:
        data[i] = (data[i] - data[i].mean()) / data[i].std()
    return data


# + id="MC52r5QoL6iW" colab_type="code" colab={}
def remove_rare_values_inplace(df_frame, column_list, threshold):

    insignificant_population = int(np.floor(threshold * len(df_frame)))
    for cat in column_list:
        freqs = collections.Counter(df_frame[cat])
        other = [i for i in freqs if freqs[i] < insignificant_population]
        for i in other:
            df_frame[cat].replace(i, 'other', inplace=True)


# + [markdown] id="VdF7xH49t8km" colab_type="text"
# # Load Data

# + id="R1lcY5Npqy_f" colab_type="code" colab={}
data = pd.read_csv('/content/drive/My Drive/progetto_aml/dataset/train_users_2.csv')
#test = pd.read_csv("/content/drive/My Drive/progetto_aml/dataset/test_users.csv")
sessions = pd.read_csv("/content/drive/My Drive/progetto_aml/dataset/sessions.csv")
#age_gender = pd.read_csv("/content/drive/My Drive/progetto_aml/dataset/age_gender_bkts.csv")

# + id="Fb1pGfThq3oX" colab_type="code" outputId="ef005ac4-1eba-45a9-86ab-38845a1a02b9" executionInfo={"status": "ok", "timestamp": 1589316629814, "user_tz": -120, "elapsed": 41197, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 249}
data.head()

# + id="zUH1QjhvrN06" colab_type="code" outputId="2292c312-22b9-407d-d659-a2a37b6ebb4c" executionInfo={"status": "ok", "timestamp": 1589316629815, "user_tz": -120, "elapsed": 41186, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 195}
sessions.head()

# + [markdown] id="cf5IN7JMt6b7" colab_type="text"
# # Preprocess

# + id="B24VFXo4uCMn" colab_type="code" colab={}
# Scartiamo la feature "date_first_booking" inutile

data.drop("date_first_booking", axis = 1, inplace = True)

# + [markdown] id="cwI5PLVkzqU1" colab_type="text"
# ## Classe obiettivo

# + id="FuZi_I7uzsVE" colab_type="code" outputId="aa5f4e44-33b3-48f1-9c0a-7a2a107fe26d" executionInfo={"status": "ok", "timestamp": 1589317630915, "user_tz": -120, "elapsed": 1043, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 393}
data.country_destination.value_counts().plot(kind='bar', rot = 45,color=plt.cm.tab20c(np.arange(len(data.country_destination.unique()))))
plt.xlabel("Country Destination")
plt.ylabel("Numero User")
plt.title("Sbilanciamento variabile target")
plt.savefig("prova.svg", format="svg")

# Classe obietivo è sbilanciata, la maggior parte degli utenti non prenotatimestamp_first_active

# + [markdown] id="J_CDml1fuAmB" colab_type="text"
# ## Age

# + id="8g-UWSLbrPcT" colab_type="code" outputId="c0c482c7-11d9-41ce-e616-0534b10da5d7" executionInfo={"status": "ok", "timestamp": 1589316630289, "user_tz": -120, "elapsed": 40478, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 168}
data["age"].describe()

# + id="hRXJTnIou9__" colab_type="code" outputId="698ce00f-30f3-4eef-92fe-3e2cfb999aea" executionInfo={"status": "ok", "timestamp": 1589316630289, "user_tz": -120, "elapsed": 39932, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 218}
data["age"].value_counts(ascending=False, dropna=False)

# + id="sWTqCgANjhwp" colab_type="code" outputId="bf82247f-329d-46de-9e63-ca0a067c989b" executionInfo={"status": "ok", "timestamp": 1589317656533, "user_tz": -120, "elapsed": 866, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 377}
data[data["age"] > 100].age.hist(grid = 0)
plt.xlabel("Age")
plt.ylabel("Numero User")
plt.title("Distribuzione variabile Age > 100")
plt.savefig("prova_age.svg", format = "svg")

# + [markdown] id="u_YwEdR4vADD" colab_type="text"
# Ci sono persone iscritte con meno di 18 anni, il cui va contre le regole di Airbnb, e persone che superano di granlunga i 100 anni.
#
# Alcuni user probabilmente hanno inserito l'anno di nascita (es. 1990) altri potrebbero essere errori o non volontà di comunicare la vera età (es. 230).

# + id="fFqpDmGEvbA3" colab_type="code" colab={}
# Si prova a sistemare gli user che hanno inserito l'anno di nascita (il dataset risale al 2015)

user_with_year_age_mask = data['age'] > 1000
data.loc[user_with_year_age_mask, 'age'] = 2015 - data.loc[user_with_year_age_mask, 'age']

# + id="PJ1V7X6Uvo1U" colab_type="code" colab={}
# Si pongono dei limiti all'età: sia assume che sotto i 18 anni e sopra i 100 siano errori


data.loc[(data['age'] > 100) | (data['age'] < 18), 'age'] = np.nan
data["age"] = (data["age"] - data["age"].mean()) / data["age"].std()

# + [markdown] id="QrJVRDxsyZro" colab_type="text"
# ## Sistemazione date

# + id="izcZvvPUxz0S" colab_type="code" colab={}
data["date_account_created"] = pd.to_datetime(data["date_account_created"])
data["timestamp_first_active"] = pd.to_datetime(data["timestamp_first_active"], format='%Y%m%d%H%M%S')

# + id="KY3_XFYc2mBL" colab_type="code" colab={}
date_account_created = pd.DatetimeIndex(data['date_account_created'])
date_first_active = pd.DatetimeIndex(data['timestamp_first_active'])

# + id="kg1AAwQp04ek" colab_type="code" colab={}
# Split date in giorno, settimana, mese, anno

# data['day_account_created'] = date_account_created.day # correlata con "day_first_active"
# data['weekday_account_created'] = date_account_created.weekday # correlata con "weekday_first_active"
# data['week_account_created'] = date_account_created.week # altamente correlata al "month_account_created"
# data['month_account_created'] = date_account_created.month # correlata con "month_first_active"
# data['year_account_created'] = date_account_created.year # correlata con "year_first_active"
data['day_first_active'] = date_first_active.day
data['weekday_first_active'] = date_first_active.weekday
# data['week_first_active'] = date_first_active.week # correlata con "month_first_active"
data['month_first_active'] = date_first_active.month
data['year_first_active'] = date_first_active.year

# + id="q5a-6ZD82xdK" colab_type="code" colab={}
# Diffrenza tra data creazione account e prima attività eseguita
data["time_lag"] = (date_account_created - date_first_active).total_seconds().astype(int).values

# + id="QVGLCpuFVa5B" colab_type="code" colab={}
#Features cicliche
data['month_first_active_sin'] = np.sin((data.month_first_active-1)*(2.*np.pi/12))
data['month_first_active_cos'] = np.cos((data.month_first_active-1)*(2.*np.pi/12))

data['day_first_active_sin'] = np.sin((data.day_first_active-1)*(2.*np.pi/31))
data['day_first_active_cos'] = np.cos((data.day_first_active-1)*(2.*np.pi/31))

data['weekday_first_active_sin'] = np.sin((data.weekday_first_active-1)*(2.*np.pi/7))
data['weekday_first_active_cos'] = np.cos((data.weekday_first_active-1)*(2.*np.pi/7))

# + id="DGOR0juZPJKD" colab_type="code" colab={}
data_corr = data[['age', #'nans',
                  'month_first_active_sin', 'month_first_active_cos',
                  'day_first_active_sin', 'day_first_active_cos', 'weekday_first_active_sin',
                  'weekday_first_active_cos', 'year_first_active', 'time_lag']]

# + id="gNeOgJM0x5ag" colab_type="code" outputId="de305625-c6f0-4c6e-f24f-c92933dc80d0" executionInfo={"status": "ok", "timestamp": 1584123619998, "user_tz": -60, "elapsed": 2314, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 712}
#sn.set(font_scale=1.4)
plt.figure(figsize=(15,10))
sns.heatmap(data_corr.corr(), annot=True, vmax = 1, vmin = -1)
plt.show()

# + id="9KX5imRDSYSI" colab_type="code" outputId="4fcd2e72-cdd5-44e7-d87a-82e98c2a8cae" executionInfo={"status": "ok", "timestamp": 1584203029115, "user_tz": -60, "elapsed": 513, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 522}
standardize(data, ["year_first_active", "time_lag"])

# + id="SpQdFjV55k8T" colab_type="code" colab={}
# Drop colonne duplicate

data.drop(["date_account_created", "timestamp_first_active",
           "day_first_active", "weekday_first_active",
           "month_first_active"], axis = 1, inplace = True)

# + [markdown] id="uwe1xJ21CYiL" colab_type="text"
# ## Sistemazione valori poco presenti nel dataset

# + id="GxFS7YrO-0yR" colab_type="code" outputId="bf523d57-418f-47ca-9eee-7a30327d896e" executionInfo={"status": "ok", "timestamp": 1584096676377, "user_tz": -60, "elapsed": 570, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 151}
data.columns

# + id="NToX4bEF-pQv" colab_type="code" outputId="f73dd11e-27e3-4c94-d777-c9c4de124bf5" executionInfo={"status": "ok", "timestamp": 1584096677053, "user_tz": -60, "elapsed": 364, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 84}
data["signup_method"].value_counts()

# + id="LiFDc39Y-wh5" colab_type="code" outputId="8b1eac38-d2d5-48d4-98bd-44e156ef8e2d" executionInfo={"status": "ok", "timestamp": 1584096679270, "user_tz": -60, "elapsed": 429, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 319}
data["signup_flow"].value_counts()

# + id="0M-kePgG_DHP" colab_type="code" outputId="6c557151-f72a-4bec-f96c-4b5242692612" executionInfo={"status": "ok", "timestamp": 1583883464083, "user_tz": -60, "elapsed": 177366, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 935}
data["language"].value_counts()

# + id="HVH-YmZKBdYQ" colab_type="code" outputId="f7e74837-55ab-43c3-a206-1cc0ff8a5dc6" executionInfo={"status": "ok", "timestamp": 1583883464083, "user_tz": -60, "elapsed": 177353, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 287}
data["first_affiliate_tracked"].value_counts()

# + id="1hAXIM-0BpUI" colab_type="code" outputId="fa9f8d82-5acc-42d3-c45b-364ea52aed6d" executionInfo={"status": "ok", "timestamp": 1583883464084, "user_tz": -60, "elapsed": 177342, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 179}
data["signup_app"].value_counts()

# + id="eBoQkt6xBtUH" colab_type="code" outputId="6953245b-d228-4b13-b345-6263441155d4" executionInfo={"status": "ok", "timestamp": 1583883464084, "user_tz": -60, "elapsed": 177330, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 359}
data["first_device_type"].value_counts()

# + id="5pIRljsbBzDY" colab_type="code" outputId="495ef423-4089-49e2-ddac-30be9e5eea66" executionInfo={"status": "ok", "timestamp": 1583883464084, "user_tz": -60, "elapsed": 177317, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 1000}
data["first_browser"].value_counts()

# + id="tZwUU6eOA0lT" colab_type="code" colab={}
# Alcuni valori pochi presenti nel dataset vengono cambiati in "other"

data["language"] = data["language"].replace(data["language"].value_counts()[data["language"].value_counts() < 10].index.to_list(), "other")

data["first_browser"] = data["first_browser"].replace(data["first_browser"].value_counts()[data["first_browser"].value_counts() < 60].index.to_list(), "other")

data["affiliate_provider"] = data["affiliate_provider"].replace(data["affiliate_provider"].value_counts()[data["affiliate_provider"].value_counts() < 100].index.to_list(), "other")

# + [markdown] id="5gsMVNjf7neY" colab_type="text"
# ## Sessions Information

# + id="fg_KUj07NmPq" colab_type="code" outputId="d2efa894-e67b-4b13-f953-159ae1ad584e" executionInfo={"status": "ok", "timestamp": 1584096693180, "user_tz": -60, "elapsed": 2402, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 134}
sessions.isna().sum()

# + id="vMW2bbBS7pxn" colab_type="code" colab={}
sessions.rename(columns = {'user_id': 'id'}, inplace=True)

# + id="YPyjU32ZLevi" colab_type="code" outputId="5c32bb15-1a66-4396-b980-0c9ed74fce1a" executionInfo={"status": "ok", "timestamp": 1584129958005, "user_tz": -60, "elapsed": 800, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 195}
sessions.head()

# + id="x3qDxTGfCpQL" colab_type="code" outputId="34a23e27-120a-426e-d535-bd6f2e4690e1" executionInfo={"status": "ok", "timestamp": 1583885377615, "user_tz": -60, "elapsed": 1560, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 233}
sessions["action"].value_counts()

# + id="NP4Dq2MnPQb4" colab_type="code" outputId="f29601c3-1512-4bdb-e306-16460c64ac45" executionInfo={"status": "ok", "timestamp": 1583883466702, "user_tz": -60, "elapsed": 179882, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 431}
sessions["action_detail"].value_counts()

# + id="h-QGpdRRP2ZC" colab_type="code" outputId="412dfc55-c731-47c9-8b51-e12d4cc4d364" executionInfo={"status": "ok", "timestamp": 1583883467656, "user_tz": -60, "elapsed": 180821, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 539}
sessions["device_type"].value_counts()

# + id="qv9X-DRpC2Ge" colab_type="code" colab={}
sessions["action"] = sessions["action"].replace(sessions["action"].value_counts()[sessions["action"].value_counts() < 1000].index.to_list(), "other")

sessions["action_detail"] = sessions["action_detail"].replace(sessions["action_detail"].value_counts()[sessions["action_detail"].value_counts() < 1000].index.to_list(), "other")

# + id="ooc-8RKJkOuP" colab_type="code" outputId="52b3b3c7-8ca4-437b-cb1f-91535ba24fd0" executionInfo={"status": "ok", "timestamp": 1584131018073, "user_tz": -60, "elapsed": 1718, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 195}
sessions.head()

# + id="RGyvsZbRJwA7" colab_type="code" outputId="cac6d0aa-78fd-424d-da17-00d34d2f05e7" executionInfo={"status": "ok", "timestamp": 1584203131106, "user_tz": -60, "elapsed": 76820, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "10319363990719344506"}} colab={"base_uri": "https://localhost:8080/", "height": 171}
action_count = sessions.groupby(['id', 'action'])['secs_elapsed'].count().unstack()
action_type_count = sessions.groupby(['id', 'action_type'])['secs_elapsed'].count().unstack()
action_detail_count = sessions.groupby(['id', 'action_detail'])['secs_elapsed'].count().unstack()
device_type_sum = sessions.groupby(['id', 'device_type'])['secs_elapsed'].sum().unstack()

sessions_data = pd.concat([action_count, action_type_count, action_detail_count, device_type_sum],axis=1)
sessions_data.columns = sessions_data.columns.map(lambda x: str(x) + '_count')


#Standardize and join
sessions_data = (sessions_data - sessions_data.mean()) / sessions_data.std()


# Device più usato
sessions_data['most_used_device'] = sessions.groupby('id')['device_type'].max()

data = data.join(sessions_data, on='id')

# + id="YatQ13Wx9l-n" colab_type="code" colab={}
# Calcolo di diverse feature su "secs_elapsed"

a = sessions.groupby('id')['secs_elapsed'].agg(
    
        secs_elapsed_sum = np.sum,
        secs_elapsed_mean = np.mean,
        secs_elapsed_min = np.min,
        secs_elapsed_max = np.max,
        secs_elapsed_median = np.median,
        secs_elapsed_std = np.std,
        day_pauses = lambda x: (x > 86400).sum(),
        long_pauses = lambda x: (x > 300000).sum(),
        short_pauses = lambda x: (x < 3600).sum(),
        session_length = np.count_nonzero
)

a["secs_elapsed_skew"] = sessions.groupby("id")["secs_elapsed"].skew()
# Standardize
a = (a - a.mean()) / a.std()

# + id="K7AgcQr5DL8K" colab_type="code" colab={}
data = data.join(a, on='id')

# + id="IxOAh0ilNbTq" colab_type="code" colab={}
categorical_features = [
    'gender', 'signup_method', 'signup_flow', 'language',
    'affiliate_channel', 'affiliate_provider', 'first_affiliate_tracked',
    'signup_app', 'first_device_type', 'first_browser', 'most_used_device'
]
data = pd.get_dummies(data, columns=categorical_features)

# + id="64Chssy0O45q" colab_type="code" colab={}
data.fillna(-99, inplace = True)

# + [markdown] id="_vWWUmMoPEMH" colab_type="text"
# # Scrittura file

# + id="6RLCco0tyLx5" colab_type="code" colab={}
# salvo csv sistemato
data.to_csv("/content/drive/My Drive/progetto_aml/dataset/train_prepocessed.csv" ,index = False)

StarcoderdataJupyterScriptsDedupFiltered