# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Volume Bar Analysis

# +
# %matplotlib inline

import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import pyplot as plt
# -

# ## 1. Reading Data

# +
from utils import read_bars as read_bars_tmp

def read_bars(csv_file: str)->pd.DataFrame:
    TIME_BEGIN = pd.to_datetime('2020-05-09T00:00:00.000Z')
    TIME_END = pd.to_datetime('2020-05-15T00:00:00.000Z')
    bars_df = read_bars_tmp(csv_file)
    # bars_df = bars_df[(bars_df['timestamp'] >= TIME_BEGIN) & (bars_df['timestamp_end'] < TIME_END)]
    return bars_df


# -

BAR_SIZE = 32  # 4, 8, 16

volume_bars = read_bars(f'/data/bars/VolumeBar/{BAR_SIZE}/VolumeBar.{BAR_SIZE}.OKEx.Swap.BTC_USDT.csv')

volume_bars

volume_bars.plot.line(x='timestamp', y='vwap', figsize=(16, 8), title='Price')

# ## 2. Volume Size

volume_sizes = volume_bars['volume']

# ### 2.1 Remove the last bar

pd.Series.all(volume_sizes[:-1] >= BAR_SIZE)  # except the last bar

volume_bars = volume_bars[:-1] # remove last bar because it's less than volume size
volume_sizes = volume_bars['volume']

pd.Series.all(volume_sizes >= BAR_SIZE)

# ### 2.2 Plot histogram of volume sizes

volume_sizes.describe([0.05, 0.5, 0.95, 0.99])

volume_sizes.hist(bins=100, density=True)

volume_sizes[volume_sizes < volume_sizes.quantile(0.99)].hist(bins=100, density=True)

# ### 2.3 Filter out bars with too large volume size

volume_bars_filtered = volume_bars[volume_sizes < volume_sizes.quantile(0.99)] # Filter out outliers

volume_bars_filtered['volume'].sum()/volume_bars['volume'].sum()

# Only 1.7% trading volume are lost after filtering.

# +
get_duration = lambda df: df['timestamp_end']-df['timestamp']

get_duration(volume_bars_filtered).sum() / get_duration(volume_bars).sum()
# -

# Only 0.5% time are lost after filtering.

# From the two percentages above, we can safely filter out bars with too large volume size.

volume_bars = volume_bars_filtered # switch

# ## 3. Duration

duration = (volume_bars['timestamp_end']-volume_bars['timestamp']).apply(lambda x : x.total_seconds())

pd.Series.all(duration >= 0)

duration.describe(percentiles=[0.05, 0.9, 0.95, 0.999])

duration.hist(bins=50, density=True)

# ### 3.1 Filter out invalid bars when crawler is down

duration[duration < duration.quantile(0.999)].hist(bins=50, density=True)

volume_bars_filtered = volume_bars[duration < duration.quantile(0.999)] # Filter out invalid bars when crawler is down

volume_bars_filtered['volume'].sum()/volume_bars['volume'].sum()

duration[duration < duration.quantile(0.999)].sum() / duration.sum()

# By filtering by `duration < duration.quantile(0.999)`, we only lose 0.2% volume, so this filtering is safe.

# switch
volume_bars = volume_bars_filtered
duration = (volume_bars['timestamp_end']-volume_bars['timestamp']).apply(lambda x : x.total_seconds())

# ### 3.2 Filter out as much volume as possible while keeping as much time as possible

sns.lineplot(
    x=np.linspace(0,1,101),
    y=[duration[duration > duration.quantile(x)].sum() /  duration.sum() for x in np.linspace(0,1,101)]
)

duration[duration > duration.quantile(0.4)].sum() /  duration.sum()

# **By filtering out 40% volume bars, we can keep 95.15% time, this is good.**

duration.quantile(0.4)

TIME_BEGIN = pd.to_datetime('2020-05-09T00:00:00.000Z')
TIME_END = pd.to_datetime('2020-05-10T00:00:00.000Z')
tmp = volume_bars[['timestamp', 'timestamp_end', 'vwap']].copy()
tmp = tmp[(tmp['timestamp'] >= TIME_BEGIN) & (tmp['timestamp_end'] < TIME_END)]
tmp['color'] = 'blue'
tmp.loc[duration > duration.quantile(0.4), 'color'] = "blue"
tmp.loc[duration <= duration.quantile(0.4), 'color'] = "red"

fig, ax = plt.subplots(figsize=(16, 8))
sns.scatterplot(data=tmp, x=tmp.index, y='vwap', hue='color') # TODO: multicolored line

# ## 4. Percentage Return

returns = volume_bars['vwap'].pct_change().dropna().rename('return')

sns.distplot(returns)

OKEX_SWAP_TAKER_FEE = 0.0005
OKEX_SWAP_MAKER_FEE = 0.0002

returns[returns > OKEX_SWAP_MAKER_FEE].size / returns.size

returns[returns > OKEX_SWAP_TAKER_FEE].size / returns.size

assert returns[returns > OKEX_SWAP_TAKER_FEE].size / returns.size > 0.1

# ## 5. VPIN

vpin = (volume_bars['volume_sell'] - volume_bars['volume_buy']).abs() / volume_bars['volume']

vpin.describe([0.05, 0.5, 0.95])

assert (vpin[vpin < 1]).size / vpin.size > 0.95

vpin.quantile(0.95)

sns.distplot(vpin)

assert duration.corr(vpin) < 0

StarcoderdataJupyterScriptsDedupFiltered