# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + colab={} colab_type="code" id="ndaf2je-jTzv"

import pandas as pd
import numpy as np
np.random.seed(5151)
from keras.models import Sequential
from keras.layers import LSTM, TimeDistributed, GRU, Activation, Conv2D,AveragePooling2D,Flatten, TimeDistributed,concatenate, Input,BatchNormalization
from keras.layers import Dense
from keras.layers import Add
from keras.models import Model
from keras import regularizers
from sklearn.preprocessing import MinMaxScaler
from sklearn.feature_selection import RFE
from matplotlib import pyplot as plt

# + colab={"base_uri": "https://localhost:8080/", "height": 212} colab_type="code" id="TkrMZwOpjT0M" outputId="3b6ddcb3-9de6-4d6c-e83a-b7f3914c8536"

url='https://raw.githubusercontent.com/NayantaraPrem/EthereumPricePrediction/master/final_dataset.csv'
df = pd.read_csv(url)
df.head(4)

# + colab={"base_uri": "https://localhost:8080/", "height": 55} colab_type="code" id="LdsllcaZjT0o" outputId="b50c833a-6ef1-4018-ba47-00cd9e6b359a"
output_column = 'Price'
output_column_feature = 'Price'
X = df.drop(['Date(UTC)', 'UnixTimeStamp'], axis=1)

#drop rows with NaN values
X = X.fillna(0) 
features =['AddressCount','MarketCap']
from_cols = [col for col in X.columns if 'from' in col]
to_cols = [col for col in X.columns if 'to' in col]
print(len(to_cols))
print(len(from_cols))

X['Price']=X['Price'].diff()
X['AddressCount']=X['AddressCount'].diff()
X['MarketCap']=X['MarketCap'].diff()
X=X.dropna()



n_timesteps = 7
n_features = len(features)
n_outputs = 1 # no. of days to predict
N = len(X)
print(features)

# + colab={"base_uri": "https://localhost:8080/", "height": 35} colab_type="code" id="B5cPw_jpjT1B" outputId="17382e63-48a4-4a82-f002-b57b86e31bbf"
train_split = 0.8 #20% test, 80% train
X_train = X[0:round(N*train_split)]
X_test = X[round(N*train_split):]
N_test = X_test.shape[0]
N_train = X_train.shape[0]
len(X_train)

# + colab={"base_uri": "https://localhost:8080/", "height": 243} colab_type="code" id="UxkXOJtTjT1V" outputId="5f87188f-3de0-498f-d83a-4662002c99e2"
#scale training data
# input = 3 features. Column 4 shifted is output
input_scaler = MinMaxScaler(feature_range = (0, 1))
X_train_scaled = pd.DataFrame(input_scaler.fit_transform(X_train), columns=X_train.columns)
output_scaler = MinMaxScaler(feature_range = (0, 1))
output_scaler.fit(np.array(X_train[output_column]).reshape((N_train,1))) # scaling the output column
X_train_scaled.head()


# + colab={"base_uri": "https://localhost:8080/", "height": 35} colab_type="code" id="pDsUWJ8hjT1r" outputId="e48c31ef-cb2c-4039-8f85-d42a367e43d6"
#input for other features
X_train_windows = []
y_train_windows = []
x_cnn = np.zeros((n_timesteps,4,15))
for i in range(N_train - n_timesteps):
    for j in range(n_timesteps):
        for k in range(15):
                    x_cnn[j,0,k] = X_train_scaled.loc[i+j, to_cols[k*2]]
                    x_cnn[j,1,k] = X_train_scaled.loc[i+j, to_cols[k*2+1]]
                    if (k<2):
                        x_cnn[j,2,k] = X_train_scaled.loc[i+j, from_cols[k*2]]
                        x_cnn[j,3,k] = X_train_scaled.loc[i+j, from_cols[k*2+1]]
                    if (k==2):
                        x_cnn[j,2,k] = 0
                        x_cnn[j,3,k] = 0
                    if (k>2):
                        x_cnn[j,2,k] = X_train_scaled.loc[i+j, from_cols[(k-1)*2]]
                        x_cnn[j,3,k] = X_train_scaled.loc[i+j, from_cols[(k-1)*2+1]]

    X_train_windows.append(np.array(x_cnn))
    y_train_windows.append(np.array(X_train_scaled.loc[i+n_timesteps, X_train_scaled.columns == output_column]))

X_train_windows = np.array(X_train_windows)
X_train_windows=np.reshape(X_train_windows,(N_train - n_timesteps,n_timesteps,4,15,1))
y_train_windows = np.array(y_train_windows)
X_train_windows.shape, y_train_windows.shape


# +
#input for CNNs
X_train_windows2 = []

for i in range(N_train - n_timesteps): 
    X_train_windows2.append(np.array(X_train_scaled.loc[i:i+n_timesteps-1, features]))
    
X_train_windows2 = np.array(X_train_windows2)
X_train_windows2.shape

# + colab={} colab_type="code" id="LwAfqGj2jT18"
#define the model

input1 = Input(shape = (n_timesteps,4,15,1))
model=(TimeDistributed(Conv2D(16, (1, 15),  activation="relu")))(input1)
model=(TimeDistributed(Conv2D(16, (4, 1), activation="relu")))(model)
model=(TimeDistributed(BatchNormalization(axis = -1)))(model)
model=(TimeDistributed(Flatten()))(model)


input2 =  Input(shape = (n_timesteps,len(features) ))

model_merge= concatenate([model, input2])

model_merge=LSTM(
    10,
    activation='relu',
    return_sequences=True)(model_merge)

model_merge=LSTM(10, activation='relu')(model_merge)
model_merge=(Activation('softmax'))(model_merge)
output=(Dense(1 ,activation='sigmoid'))(model_merge)

final_model = Model(inputs=[input1, input2], outputs=[output])


final_model.compile(optimizer='adam', loss='mse')


# train model
final_model.fit([X_train_windows, X_train_windows2], y_train_windows, epochs=200, verbose=0)

# + colab={} colab_type="code" id="wvxkdZmujT2J"
# test prediction

#scale the input to [0,1]
X_test_scaled = pd.DataFrame(input_scaler.fit_transform(X_test), columns=X_test.columns)

#input for CNN

X_test_windows = []
y_test_windows = []
x_cnn = np.zeros((n_timesteps,4,15))
for i in range(N_test - n_timesteps):
    for j in range(n_timesteps):
        for k in range(15):
            x_cnn[j,0,k] = X_test_scaled.loc[i+j, to_cols[k*2]]
            x_cnn[j,1,k] = X_test_scaled.loc[i+j, to_cols[k*2+1]]
            if (k<2):
                x_cnn[j,2,k] = X_test_scaled.loc[i+j, from_cols[k*2]]
                x_cnn[j,3,k] = X_test_scaled.loc[i+j, from_cols[k*2+1]]
            if (k==2):
                x_cnn[j,2,k] = 0
                x_cnn[j,3,k] = 0
            if (k>2):
                x_cnn[j,2,k] = X_test_scaled.loc[i+j, from_cols[(k-1)*2]]
                x_cnn[j,3,k] = X_test_scaled.loc[i+j, from_cols[(k-1)*2+1]]

    X_test_windows.append(np.array(x_cnn))
    y_test_windows.append(np.array(X_test_scaled.loc[i+n_timesteps, X_test_scaled.columns == output_column]))

X_test_windows = np.array(X_test_windows)
X_test_windows=np.reshape(X_test_windows,(N_test - n_timesteps,n_timesteps,4,15,1))

y_test_windows = np.array(y_test_windows)
X_test_windows.shape, y_test_windows.shape

# +
#input for other features
X_test_windows2 = []

for i in range(N_test - n_timesteps): 
    X_test_windows2.append(np.array(X_train_scaled.loc[i:i+n_timesteps-1, features]))
    
X_test_windows2 = np.array(X_test_windows2)
X_test_windows2.shape

# +
#run predictions on each window
y_pred = []


y_pred = final_model.predict([X_test_windows,X_test_windows2])


# rescale the predictions back to the original values
y_pred_scaled = output_scaler.inverse_transform(y_pred)
y_actual_scaled = output_scaler.inverse_transform(y_test_windows)

# + colab={} colab_type="code" id="wEC5So1wjT2z"
plt.plot(y_pred_scaled, color='red', label='Predicted Prices')
plt.plot(y_actual_scaled, color='green', label='Actual Prices')
plt.legend()
plt.show()


# + colab={} colab_type="code" id="2Wjs3h9YjT3B"
def make_df(y_pred,y_true):
    y_pred.name = 'y_pred'
    y_true.name = 'y_true'
    
    df = pd.concat([y_pred,y_true],axis=1)

    df['sign_pred'] = df.y_pred.apply(np.sign)
    df['sign_true'] = df.y_true.apply(np.sign)
    df['is_correct'] = 0
    df.loc[df.sign_pred * df.sign_true > 0 ,'is_correct'] = 1 # only registers 1 when prediction was made AND it was correct
    df['is_incorrect'] = 0
    df.loc[df.sign_pred * df.sign_true < 0,'is_incorrect'] = 1 # only registers 1 when prediction was made AND it was wrong
    df['is_predicted'] = df.is_correct + df.is_incorrect
    df['result'] = df.sign_pred * df.y_true 
    return df
def calc_scorecard(df):
    scorecard = pd.Series()
    # building block metrics
    scorecard.loc['accuracy'] = df.is_correct.sum()*1. / (df.is_predicted.sum()*1.)*100
    scorecard.loc['edge'] = df.result.mean()
    scorecard.loc['noise'] = df.y_pred.diff().abs().mean()

    # derived metrics
    scorecard.loc['y_true_chg'] = df.y_true.abs().mean()
    scorecard.loc['y_pred_chg'] = df.y_pred.abs().mean()
    scorecard.loc['prediction_calibration'] = scorecard.loc['y_pred_chg']/scorecard.loc['y_true_chg']
    scorecard.loc['capture_ratio'] = scorecard.loc['edge']/scorecard.loc['y_true_chg']*100

    # metrics for a subset of predictions
    scorecard.loc['edge_long'] = df[df.sign_pred == 1].result.mean()  - df.y_true.mean()
    scorecard.loc['edge_short'] = df[df.sign_pred == -1].result.mean()  - df.y_true.mean()

    scorecard.loc['edge_win'] = df[df.is_correct == 1].result.mean()  - df.y_true.mean()
    scorecard.loc['edge_lose'] = df[df.is_incorrect == 1].result.mean()  - df.y_true.mean()

    return scorecard
y_pred_scaled = y_pred_scaled.reshape(y_pred_scaled.shape[0])
y_actual_scaled = y_actual_scaled.reshape(y_actual_scaled.shape[0])
print(y_pred_scaled.shape)
df = make_df(pd.Series(y_pred_scaled-0.5),pd.Series(y_actual_scaled-0.5))

print(calc_scorecard(df))

StarcoderdataJupyterScriptsDedupFiltered