# import packages 
import pandas as pd                   # data management
import matplotlib.pyplot as plt       # graphics 
import datetime as dt                 # check today's date 
import sys                            # check Python version 
import numpy as np

# IPython command, puts plots in notebook 
%matplotlib inline

print('Today is', dt.date.today())
print('Python version:\n', sys.version, sep='')

"""
Explanation: Data Bootcamp "Group Project"
Analysis of historical stock return and volatility by industries using Fama-French Data
Sung Kim / Arthur Hong / Kevin Park
Contents:
1. Background
2. About the Data
3. Key Data
    + 3.1 | Slicing the Key Data
    + 3.2 | Joining the Key Data
4. Beta Calculation
5. Creating Visual Output
End of explanation
"""


# importing 30 industry portfolio data set
import pandas_datareader.data as web

ff=web.DataReader("30_Industry_Portfolios", "famafrench")

print(ff['DESCR'])

# extracting value-weighted return only
ff[0]

ind=ff[0]

ind.shape

# importing mkt data from 3 factors model

mkt=web.DataReader("F-F_Research_Data_Factors", "famafrench")

mkt

print(mkt['DESCR'])

# Dropping annual result
mkt1=mkt[0]

mkt1

mkt1['Mkt']=mkt1['Mkt-RF']+mkt1['RF']

"""
Explanation: 1 | Background
Designed by Eugene Fama and Kenneth French, Fama-French factor model is a widely used tool in finance created by employing statistical techniques to estimate returns of stocks. Within this project, we attempted to analyze stock returns and risks by calculating betas of different industries over the past seven year period.
Source: Fama-French Website
2 | About the Data
We collected our data by using PANDAS DataReader to get a direct feed to Kenneth French's data, where numerous equity market data are available online. Among them, we used the "30 Industry Portfolio" dataset to compare stock returns and risks of different industries. 
Links to  data:
+ 30 Industry Portfolio
+ Documentation to get direct feed from DataReader
3 | Key Data
We first imported the 30 industry portfolio data set. There are different categories: value or equal weighted, monthly or annual, etc. Detailed breakdown and description are shown below. 
3.1 | Slicing the Key Data
Among many different types of data, we will extract value-weighted monthly return (dataframe: 0 in 30 Industry Portfolios) since 2010 to run our analysis and store it in a dataframe ind. 
We also imported Fama-French 3-factor data with the same time frame, which contain:

Mkt-RF (market return - risk free rate)
SMB (Small-Minus-Big, the average return on the three small portfolios minus the average return on the three big portfolios)
HML (High-Minus-Low,  the average return on the two value portfolios minus the average return on the two growth portfolios)

Because we need market return rather than equity risk premium we add "Mkt" column in the dataframe by combining "Mkt-RF" and "RF". Finally, we set this 3-factor market data (Mkt-RF, SMB, HML, RF, and Mkt) in dataframe mkt1.
End of explanation
"""


# Adding mkt data to 30 industry data set
ind['Mkt']=mkt1['Mkt']

ind.tail()

# calculating historical average return and standard deviation
ind_stat=ind.describe()
ind_stat

"""
Explanation: 3.2 | Joining the Key Data
Now we add market return column ("Mkt" in dataframe mkt1) to ind dataframe.
Then key statistics, including means and standard deviations, are calculated to further derive betas of different industries. Since such statistics will be used in our beta calculation, we store it to a new dataframe ind_stat.
End of explanation
"""


# inverse matrix
ind_stat_inv = pd.DataFrame(np.linalg.pinv(ind_stat.values), ind_stat.columns, ind_stat.index)
ind_stat_inv

# beta calculation

def calc_beta(n):
    np_array = ind.values
    m = np_array[:,30] # market returns are column zero from numpy array
    s = np_array[:,n] # stock returns are column one from numpy array
    covariance = np.cov(s,m) # Calculate covariance between stock and market
    beta = covariance[0,1]/covariance[1,1]
    beta = covariance[0,1]/covariance[1,1]
    return beta

numlist=range(0,31,1)

beta=[calc_beta(i) for i in numlist]
beta

# Adding beta result
ind_stat_inv['Beta']=beta
ind_stat_inv

sort=ind_stat_inv = ind_stat_inv.sort_values(by='Beta', ascending=False)
sort

"""
Explanation: 4 | Beta Calculation
In order to facilitate matrix calculation, we altered the form of ind_stat to a inverse matrix and stored it as ind_stat_inv.
By definition, industry betas are calculated as:
Beta = (covariance between market and an industry) / (variance of market)
Once we found industry betas, we created a new column "Beta" to our ind_stat_inv and sorted by beta in ascending order.
End of explanation
"""


#Transpose industry returns table to make heatmap
ind_heatmap = ind.T
ind_heatmap.tail()

#heatmap of monthly returns since 2010 
import seaborn as sns
sns.set()
fig, ax = plt.subplots(figsize=(20,8)) 
sns.heatmap(ind_heatmap, annot=False, linewidths=.5)
ax.set_title("Monthly Returns by Industry (10 Years)")

"""
Explanation: 5 | Creating Visual Output
We created four visual presentations to help better understand how different betas can be shown in a given timeframe.
End of explanation
"""


#Sort a beta-only table to create beta bar chart 
beta_table = sort[['Beta']]
beta_table.head()

#Bar chart of betas sorted from high to low
plt.style.use('seaborn-pastel')

ax = beta_table.plot(kind='bar', colormap = "Pastel2")
ax.set_title("Betas Across Industries")

"""
Explanation: On the heatmap of monthly returns by industry, we can see which industries have more extreme swings, and which have less variation. 'Coal' looks to be particularly volatile. 'Food' and 'Utilities' looks more stable.
End of explanation
"""


#Creating a dataframe just to see the most extreme values from the beta bar chart
industry_set = ind[['Coal ','Util ','Mkt']]
industry_set = industry_set.rename(columns={'Coal ':'Coal','Util ':'Utilities','Mkt':'Market'})
industry_set.tail()

#Line plot of the returns of Coal, Utilities, and the general market
import seaborn as sns
plt.style.use('seaborn-pastel')

ax = industry_set.plot(linestyle='-', colormap = "Accent", figsize = (16,5))
ax.set_title("Monthly Returns over 10 Years")

"""
Explanation: Sure enough, 'Coal' had the highest beta value (beta > 1 means that returns are more volatile than market returns), and 'Utilities' had the lowest beta
End of explanation
"""


#Calculating a new dataframe to look at excess returns
industry_diff = industry_set
industry_diff['Coal Excess Returns'] = industry_set['Coal'] - industry_set['Market']
industry_diff['Utilities Excess Returns'] = industry_set['Utilities'] - industry_set['Market']
industry_diff = industry_diff.drop(industry_diff.columns[[0,1,2]], 1)
industry_diff.tail()

#Line plot of the excess returns
plt.style.use('seaborn-pastel')

ax = industry_diff.plot(linestyle='-', colormap = "Accent", figsize = (16,5))
ax.set_title("Market Excess Returns")

"""
Explanation: By plotting 'Coal' and 'Utilities' returns against the market's returns, we can verify the amplitude each's returns... although the chart is a little hectic
End of explanation
"""


