# parse data from google trend
def parseWeek(w):
    return w.split(" - ")[1]

"""
Explanation: reading google trend data
Google trend data seems to be updated since the orignal google trend paper. Currently, we are unable to get the absolute search volume data. Instead, for each week, we get a normalized search interest index. This index seems to be ranging from 0 to 100. The index is normalized against the search region. Therefore, it shows more of a search "density", instead of the pure count of searches.
This update might render the replicate not possible.
End of explanation
"""


trend = pd.read_csv("./data/trend.csv", converters={0:parseWeek})
trend['Week'] = pd.to_datetime(trend['Week'])
trend.set_index(['Week'], inplace=True)
trend.columns = ['search']
trend.head()

"""
Explanation: In this replicate, we focus on one keyword, 'Bitcoin'.
End of explanation
"""


time_format = "%Y-%m-%dT%H:%M:%S"
data = pd.read_csv("./data/price.csv", names=['time', 'price'], index_col='time',
                   parse_dates=[0], date_parser=lambda x: datetime.datetime.strptime(x, time_format))
bpi = data.resample('w-sat', how='ohlc')
bpi.index.name = 'Week'
bpi = pd.DataFrame(bpi['price']['close'])
bpi.head()

trend_bpi = pd.merge(trend, bpi, how='right', left_index=True, right_index=True)
trend_bpi.columns = ['search', 'close_price']
trend_bpi = trend_bpi['2012':]
trend_bpi.head()

"""
Explanation: We use the Bitcoin price index data from coinbase. Resample it per week starting at every Saturday to match google trend data. That means every Friday will be our action day, to either buy or sell bitcoin.
End of explanation
"""


plt.figure()
ax = trend_bpi.plot(secondary_y=['close_price'])

"""
Explanation: BPI and search interest plot
End of explanation
"""


trend_bpi.corr()

"""
Explanation: Correlation given by Pearson's coefficient
End of explanation
"""


trend_bpi.pct_change().plot()

"""
Explanation: BPI v. search interest (relative change)
Similar to return index, here we calculate the change of both variables (BPI and search interest) each week compared with last week. We only show 2014 to 2015 as previous variance is too big.
End of explanation
"""


trend_bpi.pct_change().corr()

"""
Explanation: Again, correlation.
End of explanation
"""


delta_t = 3

trend_bpi['rolling_SII'] = pd.rolling_mean(trend_bpi.search, delta_t)
trend_bpi.head()

"""
Explanation: Pearson correlation coefficient has been decreased once we start examining the return index instead of the actual value of two variables. Does this matter?
Replicating google trend paper
We first take the moving average of the search interest index (SII).
End of explanation
"""


trend_bpi['rolling_SII_shifted'] = trend_bpi.rolling_SII.shift(1)
trend_bpi.head()

"""
Explanation: We shift the moving average one week ahead, as we trying to predict BPI based on previous search interest.
End of explanation
"""


trend_bpi['order']=0
trend_bpi['SII_diff'] = trend_bpi.search - trend_bpi.rolling_SII_shifted
## SII_diff >= diff => search interest rises this week => price rises next week
trend_bpi.loc[trend_bpi.SII_diff >= 0,'order'] = -1
## SII_diff < diff => search interest falls this week => price falls next week
trend_bpi.loc[trend_bpi.SII_diff < 0,'order'] = 1
trend_bpi.head()

"""
Explanation: Generate order signal
If the search interest of this week is less than the moving average of interest of the past three week (delta_t), people search less about bitcoint, therefore it is likely this is the time to buy in; otherwise, if people start searching more about bitcoin this week, it is the time to sell.
We generate order data. If it is a 1, it means at that particular week we buy BTC, and sell it next week. If it is -1, we sell it this week and buy it back next week. 
We assign the order signal based on the comparison of this week's search interest and the rolling mean of previous three weeks' search interest.
End of explanation
"""


trend_bpi['log_returns'] = trend_bpi.order * np.log(trend_bpi.close_price.shift(-1)) - \
                            trend_bpi.order * np.log(trend_bpi.close_price)

trend_bpi.log_returns.head()

"""
Explanation: Evaluation as returns
Compute log returns as proposed in the paper: the difference of prices of two consecutive weeks times the order signal.
End of explanation
"""


trend_bpi[trend_bpi.log_returns>0].close_price.count()

"""
Explanation: Positive returns indicate earning on that week.
End of explanation
"""


trend_bpi[trend_bpi.log_returns<0].close_price.count()

"""
Explanation: Negative returns indicate losing on that week.
End of explanation
"""


(np.exp(trend_bpi.log_returns.cumsum()) - 1).plot()

"""
Explanation: Plot cumulative returns over time.
End of explanation
"""


(np.exp(trend_bpi['2015'].log_returns.cumsum()) - 1).plot()

"""
Explanation: Plot cumulative return only for 2015
End of explanation
"""


(np.exp(trend_bpi['2014'].log_returns.cumsum()) - 1).plot()

"""
Explanation: Plot only 2014
End of explanation
"""


(np.exp(trend_bpi['2013'].log_returns.cumsum()) - 1).plot()

"""
Explanation: Plot only 2013
End of explanation
"""


def trend_label(cur,prev):
    if cur == prev:
        return 0
    elif cur > prev:
        return 1
    else:
        return -1
trend_bpi['truth'] = np.vectorize(trend_label)(trend_bpi.close_price, trend_bpi.close_price.shift(1))
trend_bpi.head()

trend_bpi.groupby('truth').truth.count()

"""
Explanation: It seems this strategy relies on heavily the performance of earlies times (e.g. 2013), than later times. If earlier times work well, the performance overall is really good. However, cutting off earlier times the performance is much worse.
Evaluation as prediction of trend
Label each week as up (1) and down (-1) comparing its price and that of previous week.
End of explanation
"""


trend_bpi.groupby('order').truth.count()

trend_bpi_exclude_init = trend_bpi[3:]
true_prediction = trend_bpi_exclude_init.truth==trend_bpi_exclude_init.order
correct_ratio = trend_bpi_exclude_init[true_prediction].close_price.count()/float(trend_bpi_exclude_init.close_price.count())
print "Correctly predicting trend: %f" % correct_ratio

true_positive = trend_bpi[(trend_bpi.truth==1)&(trend_bpi.order==1)].order.count()
false_negative = trend_bpi[(trend_bpi.truth==1)&(trend_bpi.order==-1)].order.count()
false_positive = trend_bpi[(trend_bpi.truth==-1)&(trend_bpi.order==1)].order.count()
true_negative = trend_bpi[(trend_bpi.truth==-1)&(trend_bpi.order==-1)].order.count()
print "TP: %d, FN: %d, FP: %d, TN: %d" % (true_positive, false_negative, false_positive, true_negative)

tp_rate = float(true_positive) /(true_positive+false_negative)
fp_rate = float(false_positive) /(true_negative+false_positive)
print "TPR: %f, FPR: %f" % (tp_rate, fp_rate)

"""
Explanation: therefore in our case "-1" is the positive case and "1" negative case.
End of explanation
"""


