def generate_train_intervals(data_df, train_time, base_time, step, days_ahead, today):
    pass

"""
Explanation: The first objective of this notebook is to implement the next function (to extract sample intervals from the total period).
End of explanation
"""


# I will try to keep the convention to name with the "days" suffix, 
# to all the variables that represent "market days". The ones that 
# represent real time will be named more arbitrarily.

train_time = 365 # In real time days
base_days = 7 # In market days
step_days = 7 # market days
ahead_days = 1 # market days
today = data_df.index[-1] # Real date

today

"""
Explanation: Let's define the parameters as constants, just to do some scratch work.
End of explanation
"""


data_df.index[data_df.index <= today][-(ahead_days + 1)]

def add_market_days(base, delta, data_df):
    """
    base is in real time.
    delta is in market days.
    """
    market_days = data_df.index
    if base not in market_days:
        raise Exception('The base date is not in the market days list.')
    base_index = market_days.tolist().index(base)
    if base_index + delta >= len(market_days):
        return market_days[-1]
    if base_index + delta < 0:
        return market_days[0]
    return market_days[base_index + delta]

# Remember the last target days are not used for training, but that is a "market days" period.
end_of_training_date = add_market_days(today, -ahead_days, data_df)
start_date = end_of_training_date - dt.timedelta(train_time) 
print('Start date: %s.  End of training date: %s.' % (start_date, end_of_training_date))

TARGET_FEATURE = 'Close'

"""
Explanation: The amount of samples to be generated would be (train_time - base_time) * num_companies / step. There are days_ahead market days left, only for target values, so the total "used" period is train_time + days_ahead.
The option of training with all, one, or some companies can be done by the user when it inputs the data (just filter data_df to get the companies you want). Anyway, one interesting choice would be to allow the training with multiple companies, targeting only one. That would multiply the features by the number of available companies, but would reduce the samples a lot. By now, I want to keep the complexity low, so I won't implement that idea, yet. A many to many approach could also be implemented (the target would be the vector with all the companies data). I will start with the simple "one to one".
End of explanation
"""


def print_period(data_df):
    print('Period: %s  to  %s.' % (data_df.index[0], data_df.index[-1]))

data_train_df = data_df[start_date:end_of_training_date]

print_period(data_train_df)
data_train_df.shape

start_target_date = add_market_days(start_date, base_days + ahead_days - 1, data_df)
data_target_df = data_df.loc[start_target_date: today,TARGET_FEATURE]

print_period(data_target_df)
data_target_df.shape

"""
Explanation: One important thing to note: the base time is in "market days", that means that it doesn't represent a period of "real" time (the real time may vary with each base interval).
End of explanation
"""


data_train_df.index[:10]

"""
Explanation: Is that initial date correct?
End of explanation
"""


date_base_ini = start_date
date_base_end = add_market_days(date_base_ini, base_days - 1, data_df)
date_target = add_market_days(date_base_end, ahead_days, data_df)

sample_blob = (data_train_df[date_base_ini: date_base_end], pd.DataFrame(data_target_df.loc[date_target]))
sample_blob[0]

target = sample_blob[1].T
target

"""
Explanation: Ok, it looks so.
Let's split now!
I should allow for different feature extraction functions to be used, after the time divisions.
End of explanation
"""


feat_close = sample_blob[0][TARGET_FEATURE]
feat_close.index = np.arange(feat_close.shape[0])
feat_close

target.index = ['target']
target

x_y_samples = feat_close.append(target)
x_y_samples

x_y_samples_shuffled = x_y_samples.T.sample(frac=1).reset_index(drop=True)
x_y_samples_shuffled.head()

"""
Explanation: Let's define a function that takes a "sample blob" and produces one sample per symbol, only for the "Close" feature (looks like the easiest to do first). The dates in the base period should be substituted by an index, and the symbols shuffled later (along with their labels).
End of explanation
"""


x_y_samples_shuffled.isnull().sum()

x_y_samples_filtered = x_y_samples_shuffled.dropna(axis=0, how='any')
print(x_y_samples_filtered.shape)
x_y_samples_filtered.isnull().sum()

# At some point I will have to standarize those values... (not now, but just as a reminder...)

std_samples = x_y_samples_shuffled.apply(lambda x: x / np.mean(x), axis=1)
std_samples.head()

features = std_samples.iloc[:,:-1]
features.head()

target = pd.DataFrame(std_samples.iloc[:,-1])
target.head()

"""
Explanation: It is important to take care of the NaN values. Possibly at this sample_blob level is a good point to do so; just discard too bad samples.
End of explanation
"""


TARGET_FEATURE = 'Close'


def feature_close_one_to_one(sample_blob):
    target = sample_blob[1].T
    feat_close = sample_blob[0][TARGET_FEATURE]
    feat_close.index = np.arange(feat_close.shape[0])
    target.index = ['target']
    x_y_samples = feat_close.append(target)
    x_y_samples_shuffled = x_y_samples.T.sample(frac=1).reset_index(drop=True)
    x_y_samples_filtered = x_y_samples_shuffled.dropna(axis=0, how='any')
    
    return x_y_samples_filtered

print(feature_close_one_to_one(sample_blob).shape)
feature_close_one_to_one(sample_blob).head()

date_base_ini = start_date
date_base_end = add_market_days(date_base_ini, base_days - 1, data_df)
date_target = add_market_days(date_base_end, ahead_days, data_df)
feat_tgt_df = pd.DataFrame()

while date_base_end < end_of_training_date:
    sample_blob = (data_train_df[date_base_ini: date_base_end],
                   pd.DataFrame(data_target_df.loc[date_target]))
    feat_tgt_blob = feature_close_one_to_one(sample_blob) # TODO: Change for a generic function
    feat_tgt_df = feat_tgt_df.append(feat_tgt_blob, ignore_index=True)
    
    date_base_ini = add_market_days(date_base_ini, step_days, data_df)
    date_base_end = add_market_days(date_base_ini, base_days - 1, data_df)
    date_target = add_market_days(date_base_end, ahead_days, data_df)
    # print('Start: %s,  End:%s' % (date_base_ini, date_base_end))

feat_tgt_df = feat_tgt_df.sample(frac=1).reset_index(drop=True)

X_df = feat_tgt_df.iloc[:,:-1]
y_df = pd.DataFrame(feat_tgt_df.iloc[:,-1])

print(X_df.shape)
X_df.head()

print(y_df.shape)
y_df.head()

"""
Explanation: Let's create the samples divider function
End of explanation
"""


def generate_train_intervals(data_df, train_time, base_days, step_days, ahead_days, today, blob_fun):
    end_of_training_date = add_market_days(today, -ahead_days, data_df)
    start_date = end_of_training_date - dt.timedelta(train_time)
    start_target_date = add_market_days(start_date, base_days + ahead_days - 1, data_df)
    
    data_train_df = data_df[start_date:end_of_training_date]
    data_target_df = data_df.loc[start_target_date: today,TARGET_FEATURE]
    
    date_base_ini = start_date
    date_base_end = add_market_days(date_base_ini, base_days - 1, data_df)
    date_target = add_market_days(date_base_end, ahead_days, data_df)
    feat_tgt_df = pd.DataFrame()

    while date_base_end < end_of_training_date:
        sample_blob = (data_train_df[date_base_ini: date_base_end],
                       pd.DataFrame(data_target_df.loc[date_target]))
        feat_tgt_blob = blob_fun(sample_blob)
        feat_tgt_df = feat_tgt_df.append(feat_tgt_blob, ignore_index=True)
        
        date_base_ini = add_market_days(date_base_ini, step_days, data_df)
        date_base_end = add_market_days(date_base_ini, base_days - 1, data_df)
        date_target = add_market_days(date_base_end, ahead_days, data_df)
        # print('Start: %s,  End:%s' % (date_base_ini, date_base_end))
    
    feat_tgt_df = feat_tgt_df.sample(frac=1).reset_index(drop=True)
    
    X_df = feat_tgt_df.iloc[:,:-1]
    y_df = pd.DataFrame(feat_tgt_df.iloc[:,-1])
    
    return X_df, y_df

train_time = 365 # In real time days
base_days = 7 # In market days
step_days = 7 # market days
ahead_days = 1 # market days
today = data_df.index[-1] # Real date

X, y = generate_train_intervals(data_df, train_time, base_days, step_days, ahead_days, today, feature_close_one_to_one)

print(X.shape)
X.head()

print(y.shape)
y.head()

%pwd

sys.path.append('../../')
import predictor.feature_extraction as fe

"""
Explanation: So, I have everything to define the final function of this notebook
End of explanation
"""


X, y = fe.generate_train_intervals(data_df, 
                                   train_time, 
                                   base_days, 
                                   step_days, 
                                   ahead_days, 
                                   today, 
                                   feature_close_one_to_one)

print(X.shape)
X.head()

print(y.shape)
y.head()

"""
Explanation: Let's try the function as it was saved in the package:
End of explanation
"""


x_y_samples

target = sample_blob[1].T
feat_close = sample_blob[0][TARGET_FEATURE]
x_y_samples = feat_close.append(target)
x_y_samples

x_y_samples.index = pd.MultiIndex.from_product([[x_y_samples.index[0]], np.arange(x_y_samples.shape[0])])
x_y_samples

"""
Explanation: Looks good
Sometimes, it may be useful to keep the dates information...
End of explanation
"""


x_y_samples.unstack().stack(0).sample(frac=1).reset_index(level=1, drop=True).head()

"""
Explanation: That would be the way to go: the timestamp of the first day of the base period works as a global timestamp for the base period.
End of explanation
"""


TARGET_FEATURE = 'Close'


def feature_close_one_to_one(sample_blob):
    target = sample_blob[1].T
    feat_close = sample_blob[0][TARGET_FEATURE]
    x_y_samples = feat_close.append(target)
    x_y_samples.index = pd.MultiIndex.from_product([[x_y_samples.index[0]], 
                                                    np.arange(x_y_samples.shape[0])])
    x_y_samples_shuffled = x_y_samples.unstack().stack(0).sample(frac=1).reset_index(level=1, drop=True)
    x_y_samples_filtered = x_y_samples_shuffled.dropna(axis=0, how='any')
    
    return x_y_samples_filtered

print(feature_close_one_to_one(sample_blob).shape)
feature_close_one_to_one(sample_blob).head()

def generate_train_intervals(data_df, train_time, base_days, step_days, ahead_days, today, blob_fun):
    end_of_training_date = add_market_days(today, -ahead_days, data_df)
    start_date = end_of_training_date - dt.timedelta(train_time)
    start_target_date = add_market_days(start_date, base_days + ahead_days - 1, data_df)
    
    data_train_df = data_df[start_date:end_of_training_date]
    data_target_df = data_df.loc[start_target_date: today, TARGET_FEATURE]
    
    date_base_ini = start_date
    date_base_end = add_market_days(date_base_ini, base_days - 1, data_df)
    date_target = add_market_days(date_base_end, ahead_days, data_df)
    feat_tgt_df = pd.DataFrame()

    while date_base_end < end_of_training_date:
        sample_blob = (data_train_df[date_base_ini: date_base_end],
                       pd.DataFrame(data_target_df.loc[date_target]))
        feat_tgt_blob = blob_fun(sample_blob)
        feat_tgt_df = feat_tgt_df.append(feat_tgt_blob)
        
        date_base_ini = add_market_days(date_base_ini, step_days, data_df)
        date_base_end = add_market_days(date_base_ini, base_days - 1, data_df)
        date_target = add_market_days(date_base_end, ahead_days, data_df)
        # print('Start: %s,  End:%s' % (date_base_ini, date_base_end))
    
    feat_tgt_df = feat_tgt_df.sample(frac=1)
    
    X_df = feat_tgt_df.iloc[:,:-1]
    y_df = pd.DataFrame(feat_tgt_df.iloc[:,-1]).rename(columns={7:'target'})
    
    return X_df, y_df

from time import time

tic = time()
X, y = generate_train_intervals(data_df, 
                                train_time, 
                                base_days, 
                                step_days, 
                                ahead_days, 
                                today, 
                                feature_close_one_to_one)
toc = time()
print('Elapsed time: %i seconds.' % (toc-tic))

print(X.shape)
X.head(10)

print(y.shape)
y.head(10)

"""
Explanation: Let's try the whole function, with shuffle (it's better to do it early, so that I won't forget later and get some artificial results), but keeping the index.
End of explanation
"""


sys.path.append('../../')
import predictor.feature_extraction as fe

X, y = fe.generate_train_intervals(data_df, 
                                   train_time, 
                                   base_days, 
                                   step_days, 
                                   ahead_days, 
                                   today, 
                                   feature_close_one_to_one)

print(X.shape)
X.head(10)

print(y.shape)
y.head(10)

"""
Explanation: Let's test the "final" (you never know...) function in its module
End of explanation
"""


data_df

base = data_df.index[0]
delta = 252

market_days = np.unique(data_df.sort_index().index)
len(market_days)

def add_market_days(base, delta, data_df):
    """
    base is in real time.
    delta is in market days.
    """
    market_days = data_df.index
    if base not in market_days:
        raise Exception('The base date is not in the market days list.')
    base_index = market_days.tolist().index(base)
    if base_index + delta >= len(market_days):
        return market_days[-1]
    if base_index + delta < 0:
        return market_days[0]
    return market_days[base_index + delta]

"""
Explanation: Nice!
I will try to modify the add_market_days function to make it return a shift in real days instead of an index shift (that takes into account the possible duplicates, that are very common in some of the approaches I will follow)
End of explanation
"""


