# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="0JBO0iNRRTcO" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 122} outputId="a90a4c10-dd4a-4c1d-c1ed-47e81c7f35ef" executionInfo={"status": "ok", "timestamp": 1556947824223, "user_tz": -600, "elapsed": 24324, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "14614405936594177066"}}
from google.colab import drive
drive.mount('/content/a')

# + id="TCEmmFvyRc4e" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="8c6da496-79e5-4bc8-88c7-ab1830156d05" executionInfo={"status": "ok", "timestamp": 1556947826264, "user_tz": -600, "elapsed": 26338, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "14614405936594177066"}}
import pandas as pd
from time import time
import numpy as np
from IPython.display import Image

import keras
import keras.backend as K
from keras.engine.topology import Layer, InputSpec
from keras.layers import Dense, Input, LSTM, RepeatVector
from keras.models import Model
from keras.optimizers import SGD
from keras import callbacks
from keras.initializers import VarianceScaling
from keras.utils import plot_model
from keras.callbacks import ModelCheckpoint,EarlyStopping

from sklearn.cluster import KMeans, DBSCAN

# + id="9GF92xvGVUhH" colab_type="code" colab={}
filePath = 'a/My Drive/Research Data/TOP/probData/Prob_weekendaft.csv'
df_prob = pd.read_csv(filePath)
# df_prob

# + id="wMElhxJTVXa6" colab_type="code" colab={}
label = df_prob.iloc[:, 0]
feature_input = df_prob.iloc[:, 1:]

# + id="yGVbHNgXVVEq" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 88} outputId="843c9dbd-f808-45e2-a546-8c2072a8b446" executionInfo={"status": "ok", "timestamp": 1556799157242, "user_tz": -600, "elapsed": 834, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "14614405936594177066"}}
dims = [feature_input.shape[-1], 300, 500, 2000, 10]
init = VarianceScaling(scale=1. / 3., mode='fan_in', distribution='uniform')
pretrain_optimizer = SGD(lr=1, momentum=0.9)
pretrain_epochs = 300
batch_size = 256


# + id="-_SlWGh1RxII" colab_type="code" colab={}
def autoencoder(dims, act='relu', init='glorot_uniform'):
    """
    Fully connected auto-encoder model, symmetric.
    Arguments:
        dims: list of number of units in each layer of encoder. dims[0] is input dim, dims[-1] is units in hidden layer.
            The decoder is symmetric with encoder. So number of layers of the auto-encoder is 2*len(dims)-1
        act: activation, not applied to Input, Hidden and Output layers
    return:
        (ae_model, encoder_model), Model of autoencoder and model of encoder
    """
    n_stacks = len(dims) - 1
    # input
    input_feature = Input(shape=(dims[0],), name='input')
    x = input_feature
    # internal layers in encoder
    for i in range(n_stacks-1):
        x = Dense(dims[i + 1], activation=act, kernel_initializer=init, name='encoder_%d' % i)(x)

    # hidden layer
    encoded = Dense(dims[-1], kernel_initializer=init, name='encoder_%d' % (n_stacks - 1))(x)  # hidden layer, features are extracted from here

    x = encoded
    # internal layers in decoder
    for i in range(n_stacks-1, 0, -1):
        x = Dense(dims[i], activation=act, kernel_initializer=init, name='decoder_%d' % i)(x)

    # output
    x = Dense(dims[0], kernel_initializer=init, name='decoder_0')(x)
    decoded = x
    return Model(inputs=input_feature, outputs=decoded, name='AE'), Model(inputs=input_feature, outputs=encoded, name='encoder')


# + id="GmrI0eSfYecb" colab_type="code" colab={}
autoencoder, encoder = autoencoder(dims, init=init)

# + id="2oW2kE3-Yxv5" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 976} outputId="54ca9915-fc87-41e8-c506-41345d177131" executionInfo={"status": "ok", "timestamp": 1556777558836, "user_tz": -600, "elapsed": 1651, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "14614405936594177066"}}
plot_model(autoencoder, to_file='autoencoder.png', show_shapes=True)
Image(filename='autoencoder.png')

# + id="Kk73sHttZdu5" colab_type="code" colab={}
autoencoder.compile(optimizer=pretrain_optimizer, loss='mse')
autoencoder.fit(feature_input, feature_input, batch_size=batch_size, epochs=pretrain_epochs, verbose = 2) #, callbacks=cb)
autoencoder.save_weights('a/My Drive/Research Data/TOP/AutoEncoder Clustering/ae_weights.h5')

# + id="iAmcUeMDZ5Y1" colab_type="code" colab={}
autoencoder.load_weights('a/My Drive/Research Data/TOP/AutoEncoder Clustering/ae_weights.h5')

# + id="9xRMQ8effx3O" colab_type="code" colab={}
y_pred = encoder.predict(feature_input)

# + [markdown] id="-LeWUnHRBPok" colab_type="text"
# # **Kmeans**

# + id="a1dnYvIzf2Zb" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="d45845eb-c2db-4ad5-ed92-03cbaeec3a32" executionInfo={"status": "ok", "timestamp": 1556776011861, "user_tz": -600, "elapsed": 841, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "14614405936594177066"}}
kmeans = KMeans(n_clusters=5, random_state=0).fit(y_pred)
kmeans.labels_

# + id="EBLZq1YMgPUX" colab_type="code" colab={}
result = pd.Series(kmeans.labels_)
df_result = label.to_frame()
df_result.loc[: ,"result"] = result

# + id="9dCD69nTgT1D" colab_type="code" colab={}
df_result.to_csv('a/My Drive/Research Data/TOP/AutoEncoder Clustering/result/AE+Kmeans/Cluster_label_weekdaymor.csv')

# + id="faPtgDamgo1c" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 102} outputId="2250730b-d2d1-4d6f-a3cd-5ef31da22691" executionInfo={"status": "ok", "timestamp": 1556769192837, "user_tz": -600, "elapsed": 336829, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "14614405936594177066"}}
prob_dict = {}

for cluster_num in range(5):
    temp_list = np.zeros(shape=500)
    df_temp = df_result.loc[df_result.result==cluster_num]
    num_of_device = df_temp.shape[0]
    print(num_of_device)
    for marker in df_temp.label.values.tolist():
        temp_device = df_prob.loc[df_prob.label==marker].iloc[0, 1:].values
        temp_list = temp_list + temp_device
    prob_dict[str(cluster_num)] = temp_list/num_of_device

df_cluster_prob = pd.DataFrame(prob_dict)
df_cluster_prob.T.to_csv('a/My Drive/Research Data/TOP/AutoEncoder Clustering/result/AE+Kmeans/Cluster_prob_weekdaymor.csv')

# + id="vqdfgc3Vmhbv" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 233} outputId="e315b451-1d92-4651-8e27-d84782e149da" executionInfo={"status": "ok", "timestamp": 1556701352490, "user_tz": -600, "elapsed": 789, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "14614405936594177066"}}
df_cluster_prob.T

# + [markdown] id="8lZ-Rsn9Bbrz" colab_type="text"
# # **DBSCAN**

# + id="DK4Di7emBa9f" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="c313a1a3-ea24-496d-947e-3de2249b5bf7" executionInfo={"status": "ok", "timestamp": 1556777891872, "user_tz": -600, "elapsed": 318739, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "14614405936594177066"}}
dbscan = DBSCAN(eps=0.5, min_samples=7).fit(y_pred)
dbscan.labels_

# + id="6eQfzVcgEe5s" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 34} outputId="126d25dd-136e-4b2a-9b43-56057897b62c" executionInfo={"status": "ok", "timestamp": 1556777891873, "user_tz": -600, "elapsed": 318497, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "14614405936594177066"}}
a = set(dbscan.labels_.tolist())
a

# + id="wvjP2WqoCxoh" colab_type="code" colab={}
result = pd.Series(dbscan.labels_)
df_result = label.to_frame()
df_result.loc[: ,"result"] = result
df_result.to_csv('a/My Drive/Research Data/TOP/AutoEncoder Clustering/result/AE+DBSCAN/Cluster_label_weekendaft.csv')

# + id="1UZd38-XC17r" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 68} outputId="1a4f9a6c-486a-433a-deff-a8097c35b69b" executionInfo={"status": "ok", "timestamp": 1556777896967, "user_tz": -600, "elapsed": 323072, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "14614405936594177066"}}
prob_dict = {}

for cluster_num in a:
    temp_list = np.zeros(shape=500)
    df_temp = df_result.loc[df_result.result==cluster_num]
    num_of_device = df_temp.shape[0]
    print(num_of_device)
    for marker in df_temp.label.values.tolist():
        temp_device = df_prob.loc[df_prob.label==marker].iloc[0, 1:].values
        temp_list = temp_list + temp_device
    prob_dict[str(cluster_num)] = temp_list/num_of_device

df_cluster_prob = pd.DataFrame(prob_dict)
df_cluster_prob.T.to_csv('a/My Drive/Research Data/TOP/AutoEncoder Clustering/result/AE+DBSCAN/Cluster_prob_weekendaft.csv')

# + [markdown] id="OLATSTwXBUE1" colab_type="text"
# # **LSTM Test**

# + id="Mw5k5NWPaAA9" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 459} outputId="2820f37e-c1c2-4eaf-8bdf-eec0393bae7a" executionInfo={"status": "ok", "timestamp": 1556799163527, "user_tz": -600, "elapsed": 1152, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "14614405936594177066"}}
latent_dim = 10
timesteps = 500
input_dim = 10
inputs = Input(shape=(timesteps, input_dim))
encoded = LSTM(latent_dim)(inputs)

decoded = RepeatVector(timesteps)(encoded)
decoded = LSTM(input_dim, return_sequences=True)(decoded)

sequence_autoencoder = Model(inputs, decoded)
encoder = Model(inputs, encoded)
encoder.summary()
sequence_autoencoder.summary()

# + id="inPPzTpjECje" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 51} outputId="d406053c-f071-49e5-9685-eef6f1e0910b" executionInfo={"status": "ok", "timestamp": 1556799180935, "user_tz": -600, "elapsed": 854, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "14614405936594177066"}}
print(feature_input.shape)
lstm_input = feature_input.values[:, :, np.newaxis]
print(lstm_input.shape)

# + id="DipeV8DiDJIF" colab_type="code" colab={}
adam_opti = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
sequence_autoencoder.compile(optimizer=adam_opti, loss='mse')
sequence_autoencoder.fit(lstm_input, lstm_input, batch_size=batch_size, epochs=pretrain_epochs, verbose = 2) #, callbacks=cb)
sequence_autoencoder.save_weights('a/My Drive/Research Data/TOP/AutoEncoder Clustering/lstm_weights.h5')

# + id="GHMD52ljdmF3" colab_type="code" colab={}
sequence_autoencoder.load_weights('a/My Drive/Research Data/TOP/AutoEncoder Clustering/lstm_weights.h5')
y_pred = encoder.predict(lstm_input)

# + [markdown] id="e9VPR5Ad5_Jb" colab_type="text"
# # **LSTM script**

# + id="GRA01eeU58l4" colab_type="code" colab={}
input_file_path = 'a/My Drive/Research Data/TOP/probData/LSTM/Prob/Prob_{}_{}.csv'
kmeans_out = 'a/My Drive/Research Data/TOP/probData/LSTM/LSTM+Kmeans/{}_LSTM_Kmeans_{}_{}.csv'
dbscan_out = 'a/My Drive/Research Data/TOP/probData/LSTM/LSTM+DBSCAN/{}_LSTM_DBSCAN_{}_{}.csv'

weekList = ['weekday_mor', 'weekday_aft', 'weekend_mor', 'weekend_aft']
monthList = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct']
cluster_num = [5]#[6, 7]
eps_List = [0.2, 0.3, 1.3]

porb_in_Path = 'a/My Drive/Research Data/TOP/probData/M10/{}.csv'


# + id="ccSERoxn9lTa" colab_type="code" colab={}
def auto_LSTM(input_file_path, monthList, weekName):
    dfList = []
    for monthName in monthList:
        df_temp = pd.read_csv(input_file_path.format(monthName, weekName), index_col=0)
        dfList.append(df_temp)
        
    lstm_input = np.empty([2717, 10, 500])
    for ii in range(2717):
        for iii in range(10):
            lstm_input[ii, iii] = dfList[iii].iloc[ii, :].values
    lstm_input = lstm_input.transpose((0, 2, 1))
    
    # create model
    latent_dim = 10
    timesteps = 500
    input_dim = 10
    inputs = Input(shape=(timesteps, input_dim))
    encoded = LSTM(latent_dim)(inputs)

    decoded = RepeatVector(timesteps)(encoded)
    decoded = LSTM(input_dim, return_sequences=True)(decoded)

    sequence_autoencoder = Model(inputs, decoded)
    encoder = Model(inputs, encoded)
    encoder.summary()
    sequence_autoencoder.summary()
    
    pretrain_epochs = 200
    batch_size = 256
    adam_opti = keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
    sequence_autoencoder.compile(optimizer=adam_opti, loss='mse')
    sequence_autoencoder.fit(lstm_input, lstm_input, batch_size=batch_size, epochs=pretrain_epochs, verbose = 2) #, callbacks=cb)
    sequence_autoencoder.save_weights('a/My Drive/Research Data/TOP/AutoEncoder Clustering/lstm_weights.h5')
    
    sequence_autoencoder.load_weights('a/My Drive/Research Data/TOP/AutoEncoder Clustering/lstm_weights.h5')
    y_pred = encoder.predict(lstm_input)
    
    return y_pred

def auto_kmeans(y_pred, cluster_core, label_output, prob_output, label, df_prob):
    kmeans = KMeans(n_clusters=cluster_core, random_state=0).fit(y_pred)
    result = pd.Series(kmeans.labels_)
    df_label = label.to_frame()
    df_label.loc[: ,"result"] = result
    df_label.to_csv(label_output)
    prob_dict = {}

    for cluster_num in range(cluster_core):
        temp_list = np.zeros(shape=500)
        df_temp = df_label.loc[df_label.result==cluster_num]
        num_of_device = df_temp.shape[0]
        print(num_of_device)
        for marker in df_temp.street_marker.values.tolist():
            temp_device = df_prob.loc[df_prob.street_marker==marker].iloc[0, 1:].values
            temp_list = temp_list + temp_device
        prob_dict[str(cluster_num)] = temp_list/num_of_device

    df_cluster_prob = pd.DataFrame(prob_dict)
    df_cluster_prob.T.to_csv(prob_output)    
    
def auto_dbscan(y_pred, eps_val, label_output, prob_output, label, df_prob):
    dbscan = DBSCAN(eps=eps_val, min_samples=7).fit(y_pred)
    a = set(dbscan.labels_.tolist())
    
    result = pd.Series(dbscan.labels_)
    df_label = label.to_frame()
    df_label.loc[: ,"result"] = result
    df_label.to_csv(label_output)

    prob_dict = {}
    for cluster_num in a:
        temp_list = np.zeros(shape=500)
        df_temp = df_label.loc[df_label.result==cluster_num]
        num_of_device = df_temp.shape[0]
        print(num_of_device)
        for marker in df_temp.street_marker.values.tolist():
            temp_device = df_prob.loc[df_prob.street_marker==marker].iloc[0, 1:].values
            temp_list = temp_list + temp_device
        prob_dict[str(cluster_num)] = temp_list/num_of_device

    df_cluster_prob = pd.DataFrame(prob_dict)
    df_cluster_prob.T.to_csv(prob_output)


# + id="3Xm7JO_-_yQN" colab_type="code" colab={"base_uri": "https://localhost:8080/", "height": 29770} outputId="fe658c94-f05d-4602-b65a-03a1a060c097" executionInfo={"status": "ok", "timestamp": 1556954005327, "user_tz": -600, "elapsed": 6151844, "user": {"displayName": "<NAME>", "photoUrl": "", "userId": "14614405936594177066"}}
for weekName in weekList:
    reference_path = porb_in_Path.format(weekName)

    print("Start reading file: " + reference_path)
    df_prob = pd.read_csv(reference_path)
    label = df_prob.iloc[:, 0]
    feature_input = df_prob.iloc[:, 1:]

    y_pred = auto_LSTM(input_file_path, monthList, weekName)

    for cluster_core in cluster_num:
        kmeans_label_output = kmeans_out.format('label', weekName, cluster_core)
        kmeans_prob_output = kmeans_out.format('prob', weekName, cluster_core)
        auto_kmeans(y_pred, cluster_core, kmeans_label_output, kmeans_prob_output, label, df_prob)
        
    for eps_val in eps_List:   
        dbscan_label_output = dbscan_out.format('label', weekName, eps_val)
        dbscan_prob_output = dbscan_out.format('prob', weekName, eps_val)
        auto_dbscan(y_pred, cluster_core, dbscan_label_output, dbscan_prob_output, label, df_prob)

    print("Finish processing file: " + reference_path)

# + id="hrXHa7Ab8udS" colab_type="code" colab={}


StarcoderdataJupyterScriptsDedupFiltered