from quantopian.research import run_pipeline
from quantopian.pipeline import Pipeline
from quantopian.pipeline.factors import Latest
from quantopian.pipeline.data.builtin import USEquityPricing
from quantopian.pipeline.data import morningstar
from quantopian.pipeline import CustomFactor
from quantopian.pipeline.factors import Returns
from quantopian.pipeline.classifiers.morningstar import Sector
from quantopian.pipeline.filters import Q500US, Q1500US
from quantopian.pipeline.data.zacks import EarningsSurprises

import pandas as pd
import numpy as np
from time import time
import math

import matplotlib.pyplot as plt
from sklearn import ensemble, preprocessing, metrics
from sklearn.linear_model import LogisticRegression

n_fwd_days = 21

bs = morningstar.balance_sheet
cfs = morningstar.cash_flow_statement
is_ = morningstar.income_statement
or_ = morningstar.operation_ratios
er = morningstar.earnings_report
v = morningstar.valuation
vr = morningstar.valuation_ratios

def make_pipeline():

    base_universe = Q1500US()

    class StdDev(CustomFactor):
        '''
        3 months realized volatility
        '''
        def compute(self, today, asset_ids, out, values):
            # Calculates the column-wise standard deviation, ignoring NaNs
            out[:] = np.nanstd(values, axis=0)

    class Momentum(CustomFactor):
        # Default inputs
        inputs = [USEquityPricing.close]
        """
        1-Month Price Momentum:
        1-month closing price rate of change.
        https://www.pnc.com/content/dam/pnc-com/pdf/personal/wealth-investments/WhitePapers/FactorAnalysisFeb2014.pdf # NOQA
        Notes:
        High value suggests momentum (shorter term)
        Equivalent to analysis of returns (1-month window)
        """
        # Compute momentum
        def compute(self, today, assets, out, close):
            out[:] = close[-1] / close[0]

    class Mean_Reversion_1M(CustomFactor):
        inputs = [Returns(window_length=21)]
        window_length = 252

        def compute(self, today, assets, out, monthly_rets):
            out[:] = (monthly_rets[-1] - np.nanmean(monthly_rets, axis=0)) / \
                np.nanstd(monthly_rets, axis=0)

    class Price_Oscillator(CustomFactor):
        """
        4/52-Week Price Oscillator:
        Average close prices over 4-weeks divided by average close
        prices over 52-weeks all less 1.
        https://www.math.nyu.edu/faculty/avellane/Lo13030.pdf
        Notes:
        High value suggests momentum
        """
        inputs = [USEquityPricing.close]
        window_length = 252

        def compute(self, today, assets, out, close):
            four_week_period = close[-20:]
            out[:] = (np.nanmean(four_week_period, axis=0) /
                      np.nanmean(close, axis=0)) - 1.

    def Earnings_Quality():
        return cfs.operating_cash_flow.latest / \
               EarningsSurprises.eps_act.latest
    
    Price_Momentum_1M = Momentum(window_length=21)      
    std_dev = StdDev(inputs=[USEquityPricing.close], window_length=63, mask=base_universe)
    Price_Oscillator = Price_Oscillator()
    
    return Pipeline(
        columns={'pb_ratio': vr.pb_ratio.latest,
                 'gp_ta': is_.gross_profit.latest / bs.total_assets.latest,
                 'roe': or_.roe.latest,
                 'net_margin': or_.net_margin.latest,
                 'assets_turnover': or_.assets_turnover.latest,
                 'gearing': bs.total_debt.latest / bs.total_equity.latest,
                 'forward_earning_yield': vr.forward_earning_yield.latest,
                 'cf_yield': vr.cf_yield.latest,
                 'dividend_yield': vr.dividend_yield.latest,
                 'market_cap': v.market_cap.latest,
                 'vol': std_dev,
                 'Price_Momentum_1M': Price_Momentum_1M,
                 'Earnings_Quality': cfs.operating_cash_flow.latest / EarningsSurprises.eps_act.latest,
                 'Price_Oscillator': Price_Oscillator,
                 
                },
        screen=base_universe
  )

start = pd.Timestamp("2015-01-01")
end = pd.Timestamp("2015-06-01")
end_1m = pd.Timestamp("2015-07-15")
test_start = pd.Timestamp("2015-05-01")

start_timer = time()
results = run_pipeline(make_pipeline(), start, end)
end_timer = time()
print "Time to run pipeline %.2f secs" % (end_timer - start_timer)

results.index.names = ['date', 'security']

results

start_timer = time()
assets = results.index.levels[1].unique()
pricing = get_pricing(assets, start_date=start, end_date=end_1m, fields='price')
end_timer = time()
print "Time to extract prices %.2f secs" % (end_timer - start_timer)

pricing.head()

"""
Explanation: Global Stock Selection Strategy
Subset of 1500 stocks - Q1500US and here.
Let us consider a global stock selection model that predicts 1-month stock returns across the Q1500 US universe, using the 14 risk factors shown below. 
14 Fundamental Risk Factors

Price-to-book ratio
Gross profit / Total assets
ROE
Net margin
Asset  turnover
Gearing
Forward earnings yield
Cash flow yield
Dividend yield
Market capitalisation
Volatility
Price Momentum 1 month
Earnings quality
Price Oscillator

References
130/30
End of explanation
"""


df = pricing.pct_change(n_fwd_days).shift(-n_fwd_days)
df = df[:end]
df

df1 = df.stack()
df1

df1.index.names = ['date', 'security']

df1

df1.name = 'returns'

df_combined = results.join(df1)
df_combined

"""
Explanation: Calculate the 1 month forward returns
End of explanation
"""


res = df_combined.copy()

res = res.reset_index().set_index('date')
res.pop('security')
res

"""
Explanation: Remove Ticker Name
remove all the ticker names. We do not need them to build our model
End of explanation
"""


for e in res.columns:
    res[e]  = res[e].replace(np.nan, 0)
    res[e]  = res[e].replace([np.inf, -np.inf], 0)

res

"""
Explanation: try replacing nan with 0. Alternative is to remove the data.
Prefer not to use mean as is the case for sklean imputer and it would distort the model
End of explanation
"""


X = res.copy()
Y = X.pop('returns')

X

X.describe()

Y

Y = Y > 0

Y

X_train = X[X.index < test_start]
X_test = X[X.index >= test_start]
Y_train = Y[Y.index < test_start]
Y_test = Y[Y.index >= test_start]

X_train.shape, Y_train.shape

X_test.shape, Y_test.shape

"""
Explanation: Splitting Training and Testing Datasets
In this case, we will hold back last month as our test dataset.
The test data date need to be changed manually. The number of data points vary from day to day. Difficult to set automatically.
End of explanation
"""


start = time()
scaler = preprocessing.StandardScaler()
clf = LogisticRegression(random_state=0)
X_train_trans = scaler.fit_transform(X_train)
clf.fit(X_train_trans, Y_train)
end = time()

print "Time taken to train ML: %0.2f secs" % (end - start)

"""
Explanation: Training our Logistic ML
End of explanation
"""


# Transform test data
X_test_trans = scaler.transform(X_test)

Y_pred = clf.predict(X_test_trans)

from sklearn.metrics import confusion_matrix, accuracy_score

cm = confusion_matrix(Y_test, Y_pred)
print(accuracy_score(Y_test, Y_pred))

print(cm)

"""
Explanation: Evaluating our ML Classifier
End of explanation
"""


