import os
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import datetime as dt
import scipy.optimize as spo
import sys
from time import time
from sklearn.metrics import r2_score, median_absolute_error

%matplotlib inline

%pylab inline
pylab.rcParams['figure.figsize'] = (20.0, 10.0)

%load_ext autoreload
%autoreload 2

sys.path.append('../../')

from utils import preprocessing as pp

data_df = pd.read_pickle('../../data/data_train_val_df.pkl')

print(data_df.shape)
data_df.head()

data_df.columns.nlevels

"""
Explanation: Special care should be taken with missing data on this problem.
Missing data shall never be filled in the target variable, or the results evaluation would be corrupted. That is a risk on this problem, if things are done without care, because the target variable and the features are the same, only time-shifted.
First forward and then backwards fill is the best way to try to keep causality as much as possible.
Some filtering of symbols that have a lot of missing data could help, or the predictor may find itself full of constant data.
Filling missing data and dropping "bad samples" can be done in two or three levels: In the total data level, in the training time level, or in the base samples level. The differences are probably small for the filling part, but may be significant when dropping samples.
End of explanation
"""


data_df['Close'].shape

good_ratios = 1.0 - (data_df['Close'].isnull().sum()/ data_df['Close'].shape[0])
good_ratios.sort_values(ascending=False).plot()

filtered_data_df = pp.drop_irrelevant_symbols(data_df['Close'], good_data_ratio=0.99)

good_ratios = 1.0 - (filtered_data_df.isnull().sum()/ filtered_data_df.shape[0])
good_ratios.sort_values(ascending=False).plot()

filtered_data_df.shape

filtered_data_df.head()

filtered_data_df.isnull().sum().sort_values(ascending=False)

"""
Explanation: Let's first filter at the symbol level
End of explanation
"""


good_data_ratio = 0.99
FEATURE_OF_INTEREST = 'Close'

filtered_data_df = data_df[FEATURE_OF_INTEREST].dropna(thresh=math.ceil(good_data_ratio*data_df[FEATURE_OF_INTEREST].shape[0]), axis=1)

filtered_data_df.head()

filtered_data_df.columns

fdata_df = data_df.loc[:,(slice(None),filtered_data_df.columns.tolist())]

new_cols = fdata_df.columns.get_level_values(1)

np.setdiff1d(new_cols, filtered_data_df.columns)

np.setdiff1d(filtered_data_df.columns, new_cols)

np.intersect1d(filtered_data_df.columns, new_cols).shape

filtered_data_df.columns.shape

"""
Explanation: Let's try to filter the whole dataset using only the 'Close' values
End of explanation
"""


filtered_data_df = pp.drop_irrelevant_symbols(data_df, good_data_ratio=0.99)

good_ratios = 1.0 - (filtered_data_df['Close'].isnull().sum()/ filtered_data_df['Close'].shape[0])
good_ratios.sort_values(ascending=False).plot()

"""
Explanation: Looks good to me... Let's test it on the full dataset
End of explanation
"""


import predictor.feature_extraction as fe

train_time = -1 # In real time days
base_days = 7 # In market days
step_days = 30 # market days
ahead_days = 1 # market days
today = data_df.index[-1] # Real date

tic = time()
x, y = fe.generate_train_intervals(data_df, 
                                   train_time, 
                                   base_days, 
                                   step_days, 
                                   ahead_days, 
                                   today, 
                                   fe.feature_close_one_to_one)
toc = time()
print('Elapsed time: %i seconds.' % (toc-tic))

x.shape

y.shape

x_y_df = pd.concat([x, y], axis=1)

x_y_df.shape

x_y_df.head()

x_y_df.isnull().sum(axis=1).sum()

import utils.preprocessing as pp

x_y_dropped_df = pp.drop_irrelevant_samples(x_y_df, -1)

x_y_dropped_df.isnull().sum(axis=1).sum()

print(x_y_dropped_df.shape)
x_y_dropped_df.head()

"""
Explanation: Now, let's filter at the sample level
End of explanation
"""


