import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from quantopian.pipeline.classifiers.morningstar import Sector
from quantopian.pipeline import Pipeline
from quantopian.pipeline.data.builtin import USEquityPricing
from quantopian.pipeline.filters import QTradableStocksUS, AtLeastN
from quantopian.research import run_pipeline
from quantopian.pipeline.data import morningstar
from quantopian.pipeline.factors import CustomFactor, AverageDollarVolume

"""
Explanation: Exercises: Universe Selection - Answer Key
Lecture Link
This exercise notebook refers to this lecture. Please use the lecture for explanations and sample code.
https://www.quantopian.com/lectures#Universe-Selection
Part of the Quantopian Lecture Series:

www.quantopian.com/lectures
github.com/quantopian/research_public
End of explanation
"""


def calculate_daily_turnover(unstacked):
    return (unstacked
            .diff()        # Get True/False showing where values changed from previous day.
            .iloc[1:]      # Drop first row, which is meaningless after diff().
            .astype(bool)  # diff() coerces from bool -> object :(.  Undo that.
            .groupby(axis=1, level=0)  
            .sum())  

"""
Explanation: Helper Functions
End of explanation
"""


# Your code goes here

universe = QTradableStocksUS()

"""
Explanation: Exercise 1: Examining the QTradableStocksUS Universe
a. Initializing the Universe
Set the QTradableStocksUS as your universe by using the QTradableStocksUS() function.
End of explanation
"""


# Your code goes here

pipe = Pipeline(
    columns={
    },
    screen=universe
)

result = run_pipeline(pipe, '2016-07-01', '2016-07-01')

pd.DataFrame(result.index.levels[1])

"""
Explanation: b. Finding Asset Composition
Use the pipeline API with the QTradableStocksUS as a screen to find and print the list of equities included in the QTradableStocksUS on 2016-07-01.
End of explanation
"""


SECTOR_CODE_NAMES = {
    Sector.BASIC_MATERIALS: 'Basic Materials',
    Sector.CONSUMER_CYCLICAL: 'Consumer Cyclical',
    Sector.FINANCIAL_SERVICES: 'Financial Services',
    Sector.REAL_ESTATE: 'Real Estate',
    Sector.CONSUMER_DEFENSIVE: 'Consumer Defensive',
    Sector.HEALTHCARE: 'Healthcare',
    Sector.UTILITIES: 'Utilities',
    Sector.COMMUNICATION_SERVICES: 'Communication Services',
    Sector.ENERGY: 'Energy',
    Sector.INDUSTRIALS: 'Industrials',
    Sector.TECHNOLOGY: 'Technology',
    -1 : 'Misc'
}

# Your code goes here

pipe = Pipeline(
    columns={'Sector': Sector()
    },
    screen=universe
)

result = run_pipeline(pipe, '2016-07-01', '2016-07-01')

sectors = result.groupby('Sector').size()
sectors.index = sectors.index.map(lambda code: SECTOR_CODE_NAMES[code])
sectors

"""
Explanation: c. Sector Exposure
Use the pipeline API with the QTradableStocksUS as a screen to find and print the sector composition of the universe on 2016-07-01.
End of explanation
"""


# Your code goes here

pipe = Pipeline(
    columns={'QTradableStocksUS' : universe
    },
    screen=universe
)

result = run_pipeline(pipe, '2016-01-01', '2017-01-01')

result = result.unstack().fillna(False)

turnover = calculate_daily_turnover(result)

turnover.plot(figsize=(14, 8));

turnover.describe()

"""
Explanation: d. Turnover Rate
Use the pipeline API with the QTradableStocksUS as a screen and the calculate_daily_turnover helper function to find and plot the turnover of the universe during 2016.
End of explanation
"""


# Your code goes here

universe = morningstar.income_statement.net_income.latest.top(1500)

pipe = Pipeline(
    columns={
    },
    screen=universe
)

result = run_pipeline(pipe, '2016-07-01', '2016-07-01')

pd.DataFrame(result.index.levels[1])

"""
Explanation: Exercise 2: Examining Tradability
a. NetIncome 1500
Create a universe consisting of the top 1500 equities by net income then find and print the list of equities included in the universe on 2016-07-01.
End of explanation
"""


# Your code goes here

pipe = Pipeline(
    columns={'Average Dollar Volume' : AverageDollarVolume(window_length = 200)
    },
    screen=universe
)

result = run_pipeline(pipe, '2016-07-01', '2016-07-01')

print "NetIncome 1500 ADV:", np.mean(result['Average Dollar Volume'])

pipe = Pipeline(
    columns={'Average Dollar Volume' : AverageDollarVolume(window_length = 30)
    },
    screen=QTradableStocksUS()
)

result = run_pipeline(pipe, '2016-07-01', '2016-07-01')

print "QTradableStocksUS ADV:", np.mean(result['Average Dollar Volume'])

"""
Explanation: b. Measuring Tradability
Find the average 200 day average dollar volume of the NetIncome 1500 universe using the AverageDollarVolume built in factor and compare to that of the QTradableStocksUS.
End of explanation
"""


# Your code goes here

universe = morningstar.income_statement.net_income.latest.top(1500)

pipe = Pipeline(
    columns={
    },
    screen=universe
)

result = run_pipeline(pipe, '2016-07-01', '2016-07-01')

pd.DataFrame(result.index.levels[1])

"""
Explanation: Using average dollar volume as a stand-in liquidity metric, the QTradableStocksUS is more liquid than the NetIncome 1500. 
We used ADV as a liquidity metric here as it is an important indicator and its calculation is simple. However, average dollar volume is not a perfect measurement of liquidity as liquidity is determined by several factors and volume is only one of those. For more information on liquidity and its effects on algorithm performance, refer to the lecture on Volume, Slippage, and Liquidity
Exercise 3: Sector Balance
a. Dividend 1500
Create a universe consisting of the top 1500 equities by dividend yield then find and print the list of equities included in the this universe on 2016-07-01.
End of explanation
"""


SECTOR_CODE_NAMES = {
    Sector.BASIC_MATERIALS: 'Basic Materials',
    Sector.CONSUMER_CYCLICAL: 'Consumer Cyclical',
    Sector.FINANCIAL_SERVICES: 'Financial Services',
    Sector.REAL_ESTATE: 'Real Estate',
    Sector.CONSUMER_DEFENSIVE: 'Consumer Defensive',
    Sector.HEALTHCARE: 'Healthcare',
    Sector.UTILITIES: 'Utilities',
    Sector.COMMUNICATION_SERVICES: 'Communication Services',
    Sector.ENERGY: 'Energy',
    Sector.INDUSTRIALS: 'Industrials',
    Sector.TECHNOLOGY: 'Technology',
    -1 : 'Misc'
}

# Your code goes here

universe = morningstar.valuation_ratios.dividend_yield.latest.top(1500)


pipe = Pipeline(
    columns={'Sector': Sector()
    },
    screen=universe
)

result = run_pipeline(pipe, '2016-07-01', '2016-07-01')

sectors = 100*result.groupby('Sector').size()/1500
sectors.index = sectors.index.map(lambda code: SECTOR_CODE_NAMES[code])
sectors

"""
Explanation: b. Dividend 1500 Sector Composition
Find and print the sector composition of the universe on 2016-07-01.
End of explanation
"""


# Your code goes here

universe = morningstar.valuation_ratios.pe_ratio.latest.top(1500)

pipe = Pipeline(
    columns={
    },
    screen=universe
)

result = run_pipeline(pipe, '2016-07-01', '2016-07-01')

pd.DataFrame(result.index.levels[1])

"""
Explanation: The Dividend 1500 has a very unbalanced sector composition, with 34% of equities being from the financial services sector. This exposes the universe to sector risk and makes it vulnerable to volatility in the financial services sector. The QTradableStocksUS avoids this by capping the number of equities from any single sector at 30%.
Exercise 4: Turnover Smoothing
a. PE 1500
Create a universe consisting of the top 1500 equities by price to earnings ratio then find and print the list of equities included in the this universe on 2016-07-01.
End of explanation
"""


# Your code goes here

pipe = Pipeline(
    columns={'Price to Earnings Ratio 1500' : universe
    },
    screen=universe
)

result = run_pipeline(pipe, '2016-01-01', '2017-01-01')

result = result.unstack().fillna(False)

turnover = calculate_daily_turnover(result)

turnover.plot(figsize=(14, 8));

print turnover.describe().loc['mean']

"""
Explanation: b. PE 1500 Turnover
Use the calculate_daily_turnover helper function to find and plot the turnover of the PE 1500 universe during 2016. Compare the average to that of the QTradableStocksUS.
End of explanation
"""


# Your code goes here

universe_smoothed = AtLeastN(inputs=[universe],
                       window_length=21,
                       N=16,)

pipe = Pipeline(
    columns={'Smoothed PE 1500' : universe_smoothed
    },
    screen=universe_smoothed
)

result = run_pipeline(pipe, '2016-01-01', '2017-01-01')

result = result.unstack().fillna(False)

turnover = calculate_daily_turnover(result)

print turnover.describe().loc['mean']

"""
Explanation: The mean turnover was almost twice as high as in the QTradableStocksUS, which has built-in smoothing features to prevent equities near the threshold from entering and exiting frequently.
c. Smoothing the PE 1500
Using AtLeastN, apply a smoothing function to the PE 1500 to reduce turnover noise and find the new mean turnover.
End of explanation
"""


