factors(689)

max_seq_len = 682

#full_train_size = 55820
#train_size = 55800
#small_train_size = 6000 #just because of performance reasons, no statistics behind this decision
#test_size = 6200

data_path = '../../../../Dropbox/data'

phae_path = data_path + '/price_hist_autoencoder'

csv_in = '../price_history_03_seq_start_suddens_trimmed.csv'
assert path.isfile(csv_in)

npz_unprocessed = phae_path + '/price_history_full_seqs.npz'
assert path.isfile(npz_unprocessed)

npz_dates = phae_path + '/price_history_full_seqs_dates.npz'
assert path.isfile(npz_dates)

npz_train = phae_path + '/price_history_seqs_dates_normed_train.npz'
assert path.isfile(npz_train)

npz_test = phae_path + '/price_history_seqs_dates_normed_test.npz'
assert path.isfile(npz_test)

npz_path = npz_train[:-len('_train.npz')]

for key, val in np.load(npz_train).iteritems():
    print key, ",", val.shape

"""
Explanation: Step 0 - hyperparams
End of explanation
"""


dp = PriceHistoryAutoEncDataProvider(npz_path=npz_path, batch_size=53, with_EOS=False)
for data in dp.datalist:
    print data.shape

# for item in dp.next():
#     print item.shape

"""
Explanation: Step 1 - collect data
End of explanation
"""


# model = PriceHistoryAutoencoder(rng=random_state, dtype=dtype, config=config)
# graph = model.getGraph(batch_size=53,
#                        enc_num_units = 10,
#                        dec_num_units = 10,
#                        ts_len=max_seq_len)

"""
Explanation: Step 2 - Build model
End of explanation
"""


#show_graph(graph)

"""
Explanation: targets
Tensor("data/strided_slice:0", shape=(53, 682), dtype=float32)
Tensor("encoder_rnn_layer/rnn/while/Exit_2:0", shape=(?, 10), dtype=float32)
Tensor("encoder_state_out_process/Elu:0", shape=(?, 2), dtype=float32)
Tensor("decoder_state_in_process/Elu:0", shape=(?, 10), dtype=float32)
Tensor("decoder_rnn_layer/rnn/transpose:0", shape=(53, 682, 10), dtype=float32)
Tensor("decoder_outs/Reshape:0", shape=(36146, 10), dtype=float32)
Tensor("readout_affine/Identity:0", shape=(36146, 1), dtype=float32)
Tensor("readout_affine/Reshape:0", shape=(53, 682), dtype=float32)
Tensor("error/mul_1:0", shape=(53, 682), dtype=float32)
Tensor("error/Mean:0", shape=(), dtype=float32)
Tensor("error/Mean:0", shape=(), dtype=float32)
Tensor("error_diff/mul_1:0", shape=(53, 681), dtype=float32)
Tensor("error_diff/Mean:0", shape=(), dtype=float32)
Tensor("error_diff/Mean:0", shape=(), dtype=float32)
End of explanation
"""


def experiment():
    return model.run(npz_path=npz_path,
                     epochs=2,
                     batch_size = 53,
                     enc_num_units = 400,
                     dec_num_units = 400,
                     ts_len=max_seq_len,
                     learning_rate = 1e-4,
                     preds_gather_enabled = False,
                    )
dyn_stats_dic = experiment()

dyn_stats_dic['dyn_stats'].plotStats()
plt.show()
dyn_stats_dic['dyn_stats_diff'].plotStats()
plt.show()

"""
Explanation: Quick test run
End of explanation
"""


model = PriceHistoryAutoencoder(rng=random_state, dtype=dtype, config=config)

npz_test = npz_path + '_test.npz'
assert path.isfile(npz_test)
path.abspath(npz_test)

def experiment():
    return model.run(npz_path=npz_path,
                     epochs=50,
                     batch_size = 53,
                     enc_num_units = 450,
                     dec_num_units = 450,
                     ts_len=max_seq_len,
                     learning_rate = 1e-3,
                     preds_gather_enabled = True,
                    )

#%%time
# dyn_stats_dic, preds_dict, targets, twods = experiment()
dyn_stats_dic, preds_dict, targets, twods = get_or_run_nn(experiment, filename='035_autoencoder_001',
                                                      nn_runs_folder = data_path + "/nn_runs")

dyn_stats_dic['dyn_stats'].plotStats()
plt.show()
dyn_stats_dic['dyn_stats_diff'].plotStats()
plt.show()

r2_scores = [r2_score(y_true=targets[ind], y_pred=preds_dict[ind])
            for ind in range(len(targets))]

ind = np.argmin(r2_scores)
ind

reals = targets[ind]
preds = preds_dict[ind]

r2_score(y_true=reals, y_pred=preds)

#sns.tsplot(data=dp.inputs[ind].flatten())

fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

%%time
dtw_scores = [fastdtw(targets[ind], preds_dict[ind])[0]
             for ind in range(len(targets))]

np.mean(dtw_scores)

coint(preds, reals)

cur_ind = np.random.randint(len(targets))
reals = targets[cur_ind]
preds = preds_dict[cur_ind]
fig = plt.figure(figsize=(15,6))
plt.plot(reals, 'b', label='reals')
plt.plot(preds, 'g')
plt.legend(['reals','preds'])
plt.show()

"""
Explanation: Step 3 training the network
End of explanation
"""


twod_arr = np.array(twods.values())
twod_arr.shape

plt.figure(figsize=(16,7))
plt.plot(twod_arr[:, 0], twod_arr[:, 1], 'r.')
plt.title('two dimensional representation of our time series after dimensionality reduction')
plt.xlabel('first dimension')
plt.ylabel('second dimension')
plt.show()

"""
Explanation: Conclusion
The autoencoder is not able to represent in a visibly obvious way our price history time series
TS in Two Dimensions
End of explanation
"""


