# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# <a href="https://www.kaggle.com/baraknetef/optilver-eda-regression?scriptVersionId=87975424" target="_blank"><img align="left" alt="Kaggle" title="Open in Kaggle" src="https://kaggle.com/static/images/open-in-kaggle.svg"></a>

# + [markdown] papermill={"duration": 0.041289, "end_time": "2022-02-16T09:14:00.970097", "exception": false, "start_time": "2022-02-16T09:14:00.928808", "status": "completed"} tags=[]
# The data:
# 112 stocks book samples over 10 min time buckets
# 112 stocks trade data over 10 min time buckets

# + _cell_guid="b1076dfc-b9ad-4769-8c92-a6c4dae69d19" _kg_hide-input=true _uuid="8f2839f25d086af736a60e9eeb907d3b93b6e0e5" papermill={"duration": 1.047169, "end_time": "2022-02-16T09:14:02.058695", "exception": false, "start_time": "2022-02-16T09:14:01.011526", "status": "completed"} tags=[]
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import os
files=[]
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        files.append(os.path.join(dirname, filename))


# + papermill={"duration": 0.05043, "end_time": "2022-02-16T09:14:02.150954", "exception": false, "start_time": "2022-02-16T09:14:02.100524", "status": "completed"} tags=[]
s='86'
sf=[]
sf=[file for file in files if str('=' + s) in file]

# + papermill={"duration": 0.90678, "end_time": "2022-02-16T09:14:03.097593", "exception": false, "start_time": "2022-02-16T09:14:02.190813", "status": "completed"} tags=[]
a=pd.read_parquet(sf[0])
b=pd.read_parquet(sf[1])
a.info(),b.info()

# + papermill={"duration": 0.062861, "end_time": "2022-02-16T09:14:03.202516", "exception": false, "start_time": "2022-02-16T09:14:03.139655", "status": "completed"} tags=[]
b.head()

# + papermill={"duration": 0.660997, "end_time": "2022-02-16T09:14:03.904799", "exception": false, "start_time": "2022-02-16T09:14:03.243802", "status": "completed"} tags=[]
b[['seconds_in_bucket','time_id']].hist()

# + papermill={"duration": 0.061046, "end_time": "2022-02-16T09:14:04.010086", "exception": false, "start_time": "2022-02-16T09:14:03.94904", "status": "completed"} tags=[]
a.info()

# + papermill={"duration": 1.108175, "end_time": "2022-02-16T09:14:05.162794", "exception": false, "start_time": "2022-02-16T09:14:04.054619", "status": "completed"} tags=[]
b.ask_price1.hist(bins=200)
b.bid_price1.hist(bins=200)

# + papermill={"duration": 1.734906, "end_time": "2022-02-16T09:14:06.94158", "exception": false, "start_time": "2022-02-16T09:14:05.206674", "status": "completed"} tags=[]
import os
import pandas as pd
import numpy as np
from sklearn.metrics import r2_score
import glob
import seaborn as sns
from sklearn.model_selection import train_test_split as split
from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.metrics import mean_squared_log_error as msle
from sklearn.metrics import mean_squared_error as mse
from sklearn.metrics import r2_score


list_order_book_file_train = glob.glob('/kaggle/input/optiver-realized-volatility-prediction/book_train.parquet/*')
list_trade_file_train = glob.glob('/kaggle/input/optiver-realized-volatility-prediction/trade_train.parquet/*')
train = pd.read_csv('../input/optiver-realized-volatility-prediction/train.csv')

def log_return(list_stock_prices):
    return np.log(list_stock_prices).diff() 
def realized_volatility(series_log_return):
    return np.sqrt(np.sum(series_log_return**2))
def rmspe(y, y_pred):
    return  (np.sqrt(np.mean(np.square((y - y_pred) / y))))


# + [markdown] papermill={"duration": 0.046082, "end_time": "2022-02-16T09:14:07.031758", "exception": false, "start_time": "2022-02-16T09:14:06.985676", "status": "completed"} tags=[]
# # Feature creation
# This code creates raw features based on the raw data:
# from the book data, per stock and time bucket:
# - realized volatility
# - sum of log returns
# - sum of bid and ask sizes for each level
# - mean spread
#
# from the trade data, per stock and time bucket:
# - trade size
# - order count
#
# *trade price is left out since it is accaunted for by 'wap'

# + papermill={"duration": 357.332947, "end_time": "2022-02-16T09:20:04.409698", "exception": false, "start_time": "2022-02-16T09:14:07.076751", "status": "completed"} tags=[]
def book_features(file_path, prediction_column_name):
    df_book_data = pd.read_parquet(file_path)
    df_book_data['wap'] =(df_book_data['bid_price1'] * df_book_data['ask_size1']+df_book_data['ask_price1'] * df_book_data['bid_size1'])  / (
                                      df_book_data['bid_size1']+ df_book_data[
                                  'ask_size1'])
    df_book_data['log_return'] = df_book_data.groupby(['time_id'])['wap'].apply(log_return)
    df_book_data = df_book_data[~df_book_data['log_return'].isnull()]
    df_book_data['spread'] = df_book_data['ask_price1'] - df_book_data['bid_price1']
    df_book_features =  pd.DataFrame(df_book_data.groupby(['time_id'])['log_return'].agg(realized_volatility)).reset_index()
    df_book_features = df_book_features.rename(columns = {'log_return':prediction_column_name})
    df_agg_data = pd.DataFrame(df_book_data.groupby(['time_id'])[['log_return','bid_size1','ask_size1','bid_size2','ask_size2']].sum()).reset_index()
    stock_id = file_path.split('=')[1]
    df_book_features['stock_id'] = stock_id
    df_agg_data = df_agg_data.rename(columns = {'log_return':'sum_log_return'})
    df_agg_data['row_id'] = df_agg_data['time_id'].apply(lambda x:f'{stock_id}-{x}')
    df_spread = pd.DataFrame(df_book_data.groupby(['time_id'])['spread'].mean()).reset_index()
    df_spread['row_id'] = df_spread['time_id'].apply(lambda x:f'{stock_id}-{x}')                        
    df_book_features = df_book_features.merge(df_agg_data)
    df_book_features = df_book_features.merge(df_spread)                         
    return df_book_features

df_book_features = pd.DataFrame()
for file in list_order_book_file_train:
    a=book_features(file,prediction_column_name='realized_vol')
    df_book_features=pd.concat([df_book_features,a])

# + [markdown] papermill={"duration": 0.044107, "end_time": "2022-02-16T09:20:04.501953", "exception": false, "start_time": "2022-02-16T09:20:04.457846", "status": "completed"} tags=[]
# **The resulting book features df:**

# + papermill={"duration": 0.174667, "end_time": "2022-02-16T09:20:04.721762", "exception": false, "start_time": "2022-02-16T09:20:04.547095", "status": "completed"} tags=[]
df_book_features.info()


# + papermill={"duration": 10.731448, "end_time": "2022-02-16T09:20:15.502568", "exception": false, "start_time": "2022-02-16T09:20:04.77112", "status": "completed"} tags=[]
def trade_features(file_path):
    df_trade_data = pd.read_parquet(file_path)
    df_trade_features =  pd.DataFrame(df_trade_data.groupby(['time_id'])[['size','order_count']].sum()).reset_index()
    df_trade_features = df_trade_features.rename(columns = {'size':'trade_size'})
    stock_id = file_path.split('=')[1]
    df_trade_features['stock_id'] = stock_id
    df_trade_features['row_id'] = df_trade_features['time_id'].apply(lambda x:f'{stock_id}-{x}')                                        
    return df_trade_features

df_trade_features = pd.DataFrame()
for file in list_trade_file_train:
    a=trade_features(file)
    df_trade_features=pd.concat([df_trade_features,a])

# + [markdown] papermill={"duration": 0.04443, "end_time": "2022-02-16T09:20:15.592082", "exception": false, "start_time": "2022-02-16T09:20:15.547652", "status": "completed"} tags=[]
# **the resulting trade data features df:**

# + papermill={"duration": 0.154811, "end_time": "2022-02-16T09:20:15.791542", "exception": false, "start_time": "2022-02-16T09:20:15.636731", "status": "completed"} tags=[]
df_trade_features.info()

# + papermill={"duration": 0.586851, "end_time": "2022-02-16T09:20:16.42372", "exception": false, "start_time": "2022-02-16T09:20:15.836869", "status": "completed"} tags=[]
df_raw_features = df_book_features.merge(df_trade_features,how='left')

# + papermill={"duration": 0.169499, "end_time": "2022-02-16T09:20:16.638299", "exception": false, "start_time": "2022-02-16T09:20:16.4688", "status": "completed"} tags=[]
df_raw_features.info()

# + papermill={"duration": 0.083688, "end_time": "2022-02-16T09:20:16.767707", "exception": false, "start_time": "2022-02-16T09:20:16.684019", "status": "completed"} tags=[]
df_trade_features.stock_id.nunique()

# + [markdown] papermill={"duration": 0.046893, "end_time": "2022-02-16T09:20:16.860026", "exception": false, "start_time": "2022-02-16T09:20:16.813133", "status": "completed"} tags=[]
# # Feature engeneering

# + [markdown] papermill={"duration": 0.045578, "end_time": "2022-02-16T09:20:16.95165", "exception": false, "start_time": "2022-02-16T09:20:16.906072", "status": "completed"} tags=[]
# First we will calculate the future realized volatility for each time bucket and stock (this is the target).
#
# by doing that we have to depart with all the last time buckets - it won't have future data.

# + papermill={"duration": 0.160626, "end_time": "2022-02-16T09:20:17.15758", "exception": false, "start_time": "2022-02-16T09:20:16.996954", "status": "completed"} tags=[]
df_raw_features['future_vol'] = df_raw_features.groupby(['stock_id'])['realized_vol'].shift(-1)
df_raw_features = df_raw_features[~df_raw_features['future_vol'].isnull()]

# + [markdown] papermill={"duration": 0.044889, "end_time": "2022-02-16T09:20:17.24768", "exception": false, "start_time": "2022-02-16T09:20:17.202791", "status": "completed"} tags=[]
# The bid and ask sizes will be used to claculate the book depth:
# - the ratio between first level bid and ask i.e. 'baratio'
# - the ratio between the bid size and the average time bucket trade size i.e. 'bratio'
# - the ratio between the ask size and the average time bucket trade size i.e. 'aratio'
# - the average order

# + papermill={"duration": 0.268327, "end_time": "2022-02-16T09:20:17.56126", "exception": false, "start_time": "2022-02-16T09:20:17.292933", "status": "completed"} tags=[]
df_raw_features.groupby(['stock_id','time_id'])['trade_size'].sum().unstack().mean(axis=1)

# + papermill={"duration": 0.417669, "end_time": "2022-02-16T09:20:18.024397", "exception": false, "start_time": "2022-02-16T09:20:17.606728", "status": "completed"} tags=[]
df_raw_features['baratio'] = df_raw_features['bid_size1'] / df_raw_features['ask_size1']
sizes = df_raw_features.groupby(['stock_id','time_id'])['trade_size'].sum().unstack().mean(axis=1)
df_raw_features = df_raw_features.merge(sizes.rename('mean_trade_size'), left_on='stock_id', right_on='stock_id')
df_raw_features['bratio']=(df_raw_features['bid_size1'] + df_raw_features['bid_size2'])/df_raw_features['mean_trade_size']
df_raw_features['aratio']=(df_raw_features['ask_size1'] + df_raw_features['ask_size2'])/df_raw_features['mean_trade_size']
df_raw_features['mean_order']=df_raw_features['trade_size']/df_raw_features['order_count']
df_raw_features.head()

# + papermill={"duration": 0.070614, "end_time": "2022-02-16T09:20:18.141316", "exception": false, "start_time": "2022-02-16T09:20:18.070702", "status": "completed"} tags=[]
# df_raw_features['bratio']=(df_raw_features['bid_size1'] + df_raw_features['bid_size2'])/df_raw_features['mean_trade_size']
# df_raw_features['aratio']=(df_raw_features['ask_size1'] + df_raw_features['ask_size2'])/df_raw_features['mean_trade_size']
df_raw_features.head()

# + [markdown] papermill={"duration": 0.046053, "end_time": "2022-02-16T09:20:18.233867", "exception": false, "start_time": "2022-02-16T09:20:18.187814", "status": "completed"} tags=[]
# - most of the features seem to distribute log normaly. so in order for them to fit the scale of our target - I will log them

# + [markdown] papermill={"duration": 0.046022, "end_time": "2022-02-16T09:20:18.326442", "exception": false, "start_time": "2022-02-16T09:20:18.28042", "status": "completed"} tags=[]
# Transformations

# + papermill={"duration": 0.146329, "end_time": "2022-02-16T09:20:18.51936", "exception": false, "start_time": "2022-02-16T09:20:18.373031", "status": "completed"} tags=[]
logged = df_raw_features[['spread','baratio','bratio','aratio','mean_order']]
logged = np.log1p(logged)
#hundreds = df_raw_features[['realized_vol','future_vol']]*100

# + [markdown] papermill={"duration": 0.046138, "end_time": "2022-02-16T09:20:18.612973", "exception": false, "start_time": "2022-02-16T09:20:18.566835", "status": "completed"} tags=[]
# # EDA
# now we will see some analysis of the calculated features

# + papermill={"duration": 0.159977, "end_time": "2022-02-16T09:20:18.820384", "exception": false, "start_time": "2022-02-16T09:20:18.660407", "status": "completed"} tags=[]
df_final_features = df_raw_features.copy()[['time_id','stock_id','row_id','realized_vol','future_vol', 'sum_log_return']]
df_final_features = df_final_features.merge(logged,left_index=True, right_index=True)
#df_final_features = df_final_features.merge(hundreds,left_index=True, right_index=True)
df_final_features.head()

# + [markdown] papermill={"duration": 0.047292, "end_time": "2022-02-16T09:20:18.914769", "exception": false, "start_time": "2022-02-16T09:20:18.867477", "status": "completed"} tags=[]
# # Removing outliars

# + papermill={"duration": 0.165518, "end_time": "2022-02-16T09:20:19.12765", "exception": false, "start_time": "2022-02-16T09:20:18.962132", "status": "completed"} tags=[]
df_final_features = df_final_features[df_final_features['future_vol']<0.06]
df_final_features = df_final_features[df_final_features['realized_vol']<0.06]
df_final_features = df_final_features[df_final_features['spread']<0.012]

# + papermill={"duration": 0.056634, "end_time": "2022-02-16T09:20:19.231624", "exception": false, "start_time": "2022-02-16T09:20:19.17499", "status": "completed"} tags=[]
len(df_final_features)

# + papermill={"duration": 2.522131, "end_time": "2022-02-16T09:20:21.801393", "exception": false, "start_time": "2022-02-16T09:20:19.279262", "status": "completed"} tags=[]
df_final_features.hist(bins=80,figsize=(9,6))

# + [markdown] papermill={"duration": 0.048914, "end_time": "2022-02-16T09:20:21.899524", "exception": false, "start_time": "2022-02-16T09:20:21.85061", "status": "completed"} tags=[]
# - on most cases the features seem to distribute log normaly (except for the returns which are allready logged)

# + papermill={"duration": 0.634648, "end_time": "2022-02-16T09:20:22.583491", "exception": false, "start_time": "2022-02-16T09:20:21.948843", "status": "completed"} tags=[]
sns.heatmap(df_final_features.corr().abs())
df_final_features.corr()

# + papermill={"duration": 4.845853, "end_time": "2022-02-16T09:20:27.479823", "exception": false, "start_time": "2022-02-16T09:20:22.63397", "status": "completed"} tags=[]
#sns.pairplot(df_final_features.iloc[:,2:], height=1.5)
df_final_features.plot.scatter('realized_vol','spread')
df_final_features.plot.scatter('realized_vol','future_vol')
df_final_features.plot.scatter('aratio','bratio')
df_final_features.columns

# + [markdown] papermill={"duration": 0.055201, "end_time": "2022-02-16T09:20:27.59079", "exception": false, "start_time": "2022-02-16T09:20:27.535589", "status": "completed"} tags=[]
# - the spread is heavily correlated with the realized volatility
# - the ask ratio and the bid ratio are correlated, but the scatter isn't so clear. 
# remove one? log them?
# - the other features are not very correlated to each other and to the target. 
# - question - the outliers indicate special trade points. do we leave them? seperate?

# + papermill={"duration": 0.074791, "end_time": "2022-02-16T09:20:27.720065", "exception": false, "start_time": "2022-02-16T09:20:27.645274", "status": "completed"} tags=[]
df_final_features.head()

# + [markdown] papermill={"duration": 0.054979, "end_time": "2022-02-16T09:20:27.829841", "exception": false, "start_time": "2022-02-16T09:20:27.774862", "status": "completed"} tags=[]
# # Linear Regression
# - aratio and bratio are correleted. so for the linear regression I'll use only one 

# + papermill={"duration": 0.14292, "end_time": "2022-02-16T09:20:28.028029", "exception": false, "start_time": "2022-02-16T09:20:27.885109", "status": "completed"} tags=[]
#df_final_features.columns
X = df_final_features[['realized_vol','sum_log_return','spread','baratio','bratio']]
y = df_final_features['future_vol']
lin_model_1 = LinearRegression(fit_intercept=False).fit(X, y)

# + papermill={"duration": 0.065281, "end_time": "2022-02-16T09:20:28.149154", "exception": false, "start_time": "2022-02-16T09:20:28.083873", "status": "completed"} tags=[]
list(zip(X.columns, lin_model_1.coef_))

# + papermill={"duration": 1.854211, "end_time": "2022-02-16T09:20:30.058625", "exception": false, "start_time": "2022-02-16T09:20:28.204414", "status": "completed"} tags=[]
y_pred = lin_model_1.predict(X)
ax = sns.scatterplot(x=y, y=y_pred)
ax.plot(y, y, 'r')

# + papermill={"duration": 0.068534, "end_time": "2022-02-16T09:20:30.184424", "exception": false, "start_time": "2022-02-16T09:20:30.11589", "status": "completed"} tags=[]
mse(y, y_pred, squared=False)

# + papermill={"duration": 0.074203, "end_time": "2022-02-16T09:20:30.31579", "exception": false, "start_time": "2022-02-16T09:20:30.241587", "status": "completed"} tags=[]
R2 = round(r2_score(y_true = y, y_pred = y_pred),3)
RMSPE = round(rmspe(y = y, y_pred = y_pred),3)
print(f'Performance of the linear prediction: R2 score: {R2}, RMSPE: {RMSPE}')

# + [markdown] papermill={"duration": 0.057885, "end_time": "2022-02-16T09:20:30.432852", "exception": false, "start_time": "2022-02-16T09:20:30.374967", "status": "completed"} tags=[]
# # Decision Tree

# + papermill={"duration": 5.62758, "end_time": "2022-02-16T09:20:36.118153", "exception": false, "start_time": "2022-02-16T09:20:30.490573", "status": "completed"} tags=[]
X = df_final_features[['realized_vol','sum_log_return','spread','baratio','bratio','aratio']]
y = df_final_features['future_vol']
dt_model_1 = DecisionTreeRegressor(max_leaf_nodes=1000).fit(X, y)

# + papermill={"duration": 1.858068, "end_time": "2022-02-16T09:20:38.033765", "exception": false, "start_time": "2022-02-16T09:20:36.175697", "status": "completed"} tags=[]
y_pred = dt_model_1.predict(X)
ax = sns.scatterplot(x=y, y=y_pred)
ax.plot(y, y, 'r')

# + papermill={"duration": 0.278488, "end_time": "2022-02-16T09:20:38.372017", "exception": false, "start_time": "2022-02-16T09:20:38.093529", "status": "completed"} tags=[]
pd.Series(dt_model_1.feature_importances_,
          index=X.columns).sort_values(ascending=False).plot.barh()

# + papermill={"duration": 0.082238, "end_time": "2022-02-16T09:20:38.517178", "exception": false, "start_time": "2022-02-16T09:20:38.43494", "status": "completed"} tags=[]
msle(y, y_pred)**0.5

# + papermill={"duration": 0.088929, "end_time": "2022-02-16T09:20:38.677499", "exception": false, "start_time": "2022-02-16T09:20:38.58857", "status": "completed"} tags=[]
R2 = round(r2_score(y_true = y, y_pred = y_pred),3)
RMSPE = round(rmspe(y = y, y_pred = y_pred),3)
print(f'Performance of DT model: R2 score: {R2}, RMSPE: {RMSPE}')

# + [markdown] papermill={"duration": 0.066159, "end_time": "2022-02-16T09:20:38.823623", "exception": false, "start_time": "2022-02-16T09:20:38.757464", "status": "completed"} tags=[]
# # KNN

# + papermill={"duration": 6.763423, "end_time": "2022-02-16T09:20:45.658694", "exception": false, "start_time": "2022-02-16T09:20:38.895271", "status": "completed"} tags=[]
knn_model_1 = KNeighborsRegressor(n_neighbors=4).fit(X, y)
y_pred = knn_model_1.predict(X)

# + papermill={"duration": 1.812258, "end_time": "2022-02-16T09:20:47.532323", "exception": false, "start_time": "2022-02-16T09:20:45.720065", "status": "completed"} tags=[]
ax = sns.scatterplot(x=y, y=y_pred)
ax.plot(y, y, 'r')

# + papermill={"duration": 0.081922, "end_time": "2022-02-16T09:20:47.677349", "exception": false, "start_time": "2022-02-16T09:20:47.595427", "status": "completed"} tags=[]
R2 = round(r2_score(y_true = y, y_pred = y_pred),3)
RMSPE = round(rmspe(y = y, y_pred = y_pred),3)
print(f'Performance of KNN: R2 score: {R2}, RMSPE: {RMSPE}')

# + [markdown] papermill={"duration": 0.068148, "end_time": "2022-02-16T09:20:47.827698", "exception": false, "start_time": "2022-02-16T09:20:47.75955", "status": "completed"} tags=[]
# # Validating the Decision tree

# + [markdown] papermill={"duration": 0.063548, "end_time": "2022-02-16T09:20:47.955409", "exception": false, "start_time": "2022-02-16T09:20:47.891861", "status": "completed"} tags=[]
# > To be continued with better models

StarcoderdataJupyterScriptsDedupFiltered