# import the dataset
# from quantopian.interactive.data.eventvestor import _13d_filings as dataset
# or if you want to import the free dataset, use:
from quantopian.interactive.data.eventvestor import _13d_filings_free as dataset

# import data operations
from odo import odo
# import other libraries we will use
import pandas as pd

# Let's use blaze to understand the data a bit using Blaze dshape()
dataset.dshape

# And how many rows are there?
# N.B. we're using a Blaze function to do this, not len()
dataset.count()

# Let's see what the data looks like. We'll grab the first three rows.
dataset[:3]

"""
Explanation: EventVestor: 13D Filings
In this notebook, we'll take a look at EventVestor's 13D Filings dataset, available on the Quantopian Store. This dataset spans January 01, 2007 through the current day, and documents 13D filings with the SEC by activist shareholders disclosing a beneficial ownership of 5% or more.
Notebook Contents
There are two ways to access the data and you'll find both of them listed below. Just click on the section you'd like to read through.

<a href='#interactive'><strong>Interactive overview</strong></a>: This is only available on Research and uses blaze to give you access to large amounts of data. Recommended for exploration and plotting.
<a href='#pipeline'><strong>Pipeline overview</strong></a>: Data is made available through pipeline which is available on both the Research & Backtesting environment. Recommended for custom factor development and moving back & forth between research/backtesting.

Free samples and limits
One key caveat: we limit the number of results returned from any given expression to 10,000 to protect against runaway memory usage. To be clear, you have access to all the data server side. We are limiting the size of the responses back from Blaze.
There is a free version of this dataset as well as a paid one. The free sample includes data until 2 months prior to the current date.
To access the most up-to-date values for this data set for trading a live algorithm (as with other partner sets), you need to purchase acess to the full set.
With preamble in place, let's get started:
<a id='interactive'></a>
Interactive Overview
Accessing the data with Blaze and Interactive on Research
Partner datasets are available on Quantopian Research through an API service known as Blaze. Blaze provides the Quantopian user with a convenient interface to access very large datasets, in an interactive, generic manner.
Blaze provides an important function for accessing these datasets. Some of these sets are many millions of records. Bringing that data directly into Quantopian Research directly just is not viable. So Blaze allows us to provide a simple querying interface and shift the burden over to the server side.
It is common to use Blaze to reduce your dataset in size, convert it over to Pandas and then to use Pandas for further computation, manipulation and visualization.
Helpful links:
* Query building for Blaze
* Pandas-to-Blaze dictionary
* SQL-to-Blaze dictionary.
Once you've limited the size of your Blaze object, you can convert it to a Pandas DataFrames using:

from odo import odo
odo(expr, pandas.DataFrame)

To see how this data can be used in your algorithm, search for the Pipeline Overview section of this notebook or head straight to <a href='#pipeline'>Pipeline Overview</a>
End of explanation
"""


# first we find how Icahn is represented exactly in the `acquiring_entity` column
icahns = set(filter(lambda x: "Icahn" in x, dataset.acquiring_entity))
icahns

# Since Carl Icahn is represented in several ways, we use `like` to capture all those ways
names = dataset[dataset.acquiring_entity.like("*Icahn*")]  
# now let's filter down to the percentage of shares, timestamp, and sid  
names = names[['timestamp', 'percent_shares', 'sid', 'acquiring_entity']]  
# # When displaying a Blaze Data Object, the printout is automatically truncated to ten rows.  
names.sort('timestamp')

"""
Explanation: Let's go over the columns:
- event_id: the unique identifier for this event.
- asof_date: EventVestor's timestamp of event capture.
- trade_date: for event announcements made before trading ends, trade_date is the same as event_date. For announcements issued after market close, trade_date is next market open day.
- symbol: stock ticker symbol of the affected company.
- event_type: this should always be 13D Issuer.
- event_headline: a brief description of the event
- percent_shares: the percent of shares owned by the investor as per the 13D
- number_shares: the number of shares owned by the investor as per the 13D. Note: these shares are not split-adjusted in the way that the rest of our equity data is. We suggest using percent_shares with market cap (from Fundamentals) to get the total value of shares owned, and dividing by the close_price (from price data) to get the number of shares.
- acquiring_entity: the investor filing the 13D
- event_rating: this is always 1. The meaning of this is uncertain.
- timestamp: this is our timestamp on when we registered the data.
- sid: the equity's unique identifier. Use this instead of the symbol. Note: this sid represents the  company the shares of which are being purchased, not the acquiring entity.
We've done much of the data processing for you. Fields like timestamp and sid are standardized across all our Store Datasets, so the datasets are easy to combine. We have standardized the sid across all our equity databases.
We can select columns and rows with ease. Below, we'll fetch all 13D filings by Carl Icahn.
End of explanation
"""


# we can call odo on the entire dataset
filings_df = odo(dataset, pd.DataFrame)
june = filings_df[(filings_df.timestamp < '2013-07-01') & ('2013-05-31'< filings_df.timestamp)]
june

"""
Explanation: Finally, suppose we want a DataFrame of all 13D Filings in June 2013:
End of explanation
"""


# Import necessary Pipeline modules
from quantopian.pipeline import Pipeline
from quantopian.research import run_pipeline
from quantopian.pipeline.factors import AverageDollarVolume

# Import the datasets available
from quantopian.pipeline.data.eventvestor import _13DFilings

# Builtin Factors for easy access
from quantopian.pipeline.factors.eventvestor import BusinessDaysSince13DFilingsDate

"""
Explanation: <a id='pipeline'></a>
Pipeline Overview
Accessing the data in your algorithms & research
The only method for accessing partner data within algorithms running on Quantopian is via the pipeline API. Different data sets work differently but in the case of this data, you can add this data to your pipeline as follows:
Import the data set here

from quantopian.pipeline.data.eventvestor import _13DFilings

Then in intialize() you could do something simple like adding the raw value of one of the fields to your pipeline:

pipe.add(_13DFilings.disclosure_date.latest, 'disclosure_date')

Pipeline usage is very similar between the backtester and Research so let's go over how to import this data through pipeline and view its outputs.
End of explanation
"""


print "Here are the list of available fields per dataset:"
print "---------------------------------------------------\n"

def _print_fields(dataset):
    print "Dataset: %s\n" % dataset.__name__
    print "Fields:"
    for field in list(dataset.columns):
        print "%s - %s" % (field.name, field.dtype)
    print "\n"

_print_fields(_13DFilings)


print "---------------------------------------------------\n"

"""
Explanation: Now that we've imported the data, let's take a look at which fields are available for each dataset.
You'll find the dataset, the available fields, and the datatypes for each of those fields.
End of explanation
"""


pipe = Pipeline()
       
pipe.add(_13DFilings.disclosure_date.latest, "last_disclosure_date")
pipe.add(_13DFilings.percent_shares.latest, "percent_shares")
pipe.add(_13DFilings.number_shares.latest, "number_shares")
pipe.add(BusinessDaysSince13DFilingsDate(), "business_days_since_last_filing")

# Setting some basic liquidity strings (just for good habit)
dollar_volume = AverageDollarVolume(window_length=20)
top_1000_most_liquid = dollar_volume.rank(ascending=False) < 1000

pipe.set_screen(top_1000_most_liquid & BusinessDaysSince13DFilingsDate().notnan())

# The show_graph() method of pipeline objects produces a graph to show how it is being calculated.
pipe.show_graph(format='png')

# run_pipeline will show the output of your pipeline
pipe_output = run_pipeline(pipe, start_date='2013-11-01', end_date='2013-11-25')
pipe_output

"""
Explanation: Now that we know what fields we have access to, let's see what this data looks like when we run it through Pipeline.
This is constructed the same way as you would in the backtester. For more information on using Pipeline in Research view this thread:
https://www.quantopian.com/posts/pipeline-in-research-build-test-and-visualize-your-factors-and-filters
End of explanation
"""


# This section is only importable in the backtester
from quantopian.algorithm import attach_pipeline, pipeline_output

# General pipeline imports
from quantopian.pipeline import Pipeline
from quantopian.pipeline.factors import AverageDollarVolume

# Import the datasets available
from quantopian.pipeline.data.eventvestor import _13DFilings

# Builtin Factors for easy access
from quantopian.pipeline.factors.eventvestor import BusinessDaysSince13DFilingsDate

def make_pipeline():
    # Create our pipeline
    pipe = Pipeline()
    
    # Screen out penny stocks and low liquidity securities.
    dollar_volume = AverageDollarVolume(window_length=20)
    is_liquid = dollar_volume.rank(ascending=False) < 1000
    
    # Create the mask that we will use for our percentile methods.
    base_universe = (is_liquid)

    # Add pipeline factors
    pipe.add(_13DFilings.disclosure_date.latest, "last_disclosure_date")
    pipe.add(_13DFilings.percent_shares.latest, "percent_shares")
    pipe.add(_13DFilings.number_shares.latest, "number_shares")
    pipe.add(BusinessDaysSince13DFilingsDate(), "business_days_since_last_filing")

    # Set our pipeline screens
    pipe.set_screen(is_liquid)
    return pipe

def initialize(context):
    attach_pipeline(make_pipeline(), "pipeline")
    
def before_trading_start(context, data):
    results = pipeline_output('pipeline')

"""
Explanation: Taking what we've seen from above, let's see how we'd move that into the backtester.
End of explanation
"""


