"""
This notebook illustrates that for a given (stock_id, time_id, seconds_in_bucket), there may exist trade data but not book data.

This is probably a mistake in the data, that sould be accounted for. See more here: https://www.kaggle.com/c/optiver-realized-volatility-prediction/discussion/255129


"""
import pandas as pd
import numpy as np
import glob


train_book_files = glob.glob('/kaggle/input/optiver-realized-volatility-prediction/book_train.parquet/*')
train_trade_files = glob.glob('/kaggle/input/optiver-realized-volatility-prediction/trade_train.parquet/*')

wrong_stocks = []

for book_fname, train_fname in zip(train_book_files, train_trade_files):
    book_example = pd.read_parquet(book_fname)
    trade_example = pd.read_parquet(train_fname)
    if book_example.shape[0] != book_example.merge(trade_example, on=['time_id','seconds_in_bucket'], how='outer').shape[0]:
        print("Shape mismatch found!")
        print("Stock_id =", book_fname.partition('=')[-1])
        print("Row num of book =", book_example.shape[0])
        print("Row num of outer join =", book_example.merge(trade_example, on=['time_id','seconds_in_bucket'], how='outer').shape[0])
        wrong_stocks.append(book_fname.partition('=')[-1])
        

for stock_id in wrong_stocks:
    book_example = pd.read_parquet(f'/kaggle/input/optiver-realized-volatility-prediction/book_train.parquet/stock_id={stock_id}')
    trade_example =  pd.read_parquet(f'/kaggle/input/optiver-realized-volatility-prediction/trade_train.parquet/stock_id={stock_id}')
    merged = book_example.merge(trade_example, on=['time_id','seconds_in_bucket'], how='outer')
    print(f"Printing wrong trades for stock {stock_id}:")
    print(merged[merged.bid_price1.isna()])