# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Build Week 2 -- Modeling Project

# ## Unzipping, Reading CSV, and Creating a DF via Concatenation

# +
# import os
# import zipfile

# ## Unzipping each file in the filefolder and putting them in the 'unzipped' directory
# def gather_data(filefolder):
#     """ Produces List of Documents from a Directory
#     filefolder (str): a path of .zip files
#     returns list of strings"""
    
#     files = os.listdir(filefolder) 
#     for csv_zips in files: 
#         path = os.path.join(filefolder, csv_zips)
#         if  path[-3:] == 'zip':
#             with zipfile.ZipFile(path, 'r') as zip_ref:
#                 zip_ref.extractall('./SPX CBOE Data/unzipped')

                
# ## Calling the gather_data function (above)
# gather_data('./SPX CBOE Data')

# +
# import pandas as pd

# ## Instantiating a new, empty list
# lst = []

# ## Reading in one CSV at a time, sampling 10%, and then appending the sample to 'lst'
# def readin_csv(filefolder):
#     """Reads in a list of csv documents from a directory 
#     filefoler (str): a path of .csv files returns list of 
#     strings"""
    
#     files = os.listdir(filefolder)
#     for csvs in files:
#         path = os.path.join(filefolder, csvs)
#         if  path[-3:] == 'csv':
#             temp_frame = pd.read_csv(path)
#             lst.append(temp_frame.sample(frac = 0.1))

            
# ## Calling the readin_csv function (above)
# readin_csv('./SPX CBOE Data/unzipped')

# ## Viewing the first couple of rows 'lst'
# lst[:5]

# +
# ## Viewing all the available rows
# pd.set_option('display.max_columns', None)

# ## Creating a pandas dataframe by concatenating the list objects
# df = pd.concat(objs = lst, axis = 0)

# # CHECK:
# print(df.shape)
# df.head()

# +
# df.to_csv('df_file')

# +
# Ignore Warnings
import warnings
warnings.filterwarnings('ignore')


import pandas as pd
# Viewing all the columns
pd.set_option('display.max_columns', None)

# Reading in the csv
df = pd.read_csv('df_file', index_col = [0])

# CHECK:
df.head()

# +
# Resetting the index for the df since we concatenated random samples to a list
df = df.reset_index(drop = True)

# CHECK:
df.head()
# -

# ## Data Cleaning, Analysis, Exploration, and Manipulation

# Viewing dtypes and info
df.info()

# +
##### Creating a Wrangle Function to Clean Data and Create Features: #####
import numpy as np

# Creating a wrangling function 
def wrangle(X):
    """Wrangling a dataframe and creating features 
    in the same way across copies."""
    
    # Creating a df copy
    X = X.copy()
    
    # Inferring the DateTime format to Dates / Time:
    X['quote_datetime'] = pd.to_datetime(X['quote_datetime'], infer_datetime_format = True)
    X['expiration'] = pd.to_datetime(X['expiration'], infer_datetime_format = True)
    
    ## DateTime Features:
    # Dates and Time
    X['datetime_year_month'] = (X['quote_datetime'].dt.to_period('M')).astype('str') # String for rate substitution later
    X['datetime_fulldate'] = X['quote_datetime'].dt.date
    X['datetime_time'] = X['quote_datetime'].dt.time
    # Time to Maturity (Days)
    X['time_to_maturity'] = X['expiration'] - X['quote_datetime']
    X['time_to_maturity'] = X['time_to_maturity'].astype('string')
    X['time_to_maturity'] = X['time_to_maturity'].str.split(' days').str[0]
    X['time_to_maturity'] = X['time_to_maturity'].astype('int')
    
    ###Column Adjustments:
    # Column Names Dictionary to make changes
    col_names = {
        'best_bid':'option_bid',
        'best_ask':'option_ask',
        'bid_size':'option_bid_size',
        'ask_size':'option_ask_size'
        
    }
    
    # Column Renaming
    X.rename(columns = col_names, inplace = True)
    
    # Dropping Unnecessary Columns
    col_drop = ['sequence_number', 'root', 'exchange_id', 'trade_condition_id',
                'canceled_trade_condition_id','number_of_exchanges','{exchange',
                'ask}[number_of_exchanges]', 'bid']
    
    X.drop(columns = col_drop, inplace = True)
    
    
    ## Feature Engineering:
    # option_bid_ask_spread -- Option Liquidty Proxy
    X['option_bid_ask_spread'] = X['option_ask'] - X['option_bid']
    
    # underlying_bid_ask_spread -- Underlying Liquidity Proxy
    X['underlying_bid_ask_spread'] =  X['underlying_ask'] - X['underlying_bid']
    
    # bid_ask_size_spread
    X['option_bid_ask_size_spread'] = X['option_ask_size'] - X['option_bid_size']
    
    # underlying_distance_from_strike
    X['distance_from_strike'] = X['underlying_ask'] - X['strike']
    
    # risk_free_rate -- Risk-Free Rate (10yr Treasury) Proxy for 'r' in BSM
    X['risk_free_rate'] = .0214
    
    # annualized volatility for SPX 2019
    X['annualized_vol'] = .2888
    
    # T = Time to Expiration in decimal
    X['T'] = X['time_to_maturity'] / 365
    
    # erT = necessary for BSM model
    X['erT'] = np.exp(-X['risk_free_rate'] * X['T'])
    
        
    ## Reordering Columns:
    # Column Ordering
    col_order = ['underlying_symbol',
                 'quote_datetime',
                 'datetime_fulldate',
                 'datetime_year_month',
                 'datetime_time',
                 'expiration',
                 'time_to_maturity',
                 'T',
                 'option_type',   
                 'strike',
                 'option_bid',
                 'option_ask',
                 'option_bid_size',
                 'option_ask_size',
                 'option_bid_ask_spread',
                 'option_bid_ask_size_spread',
                 'trade_size',
                 'trade_iv',
                 'trade_delta',
                 'annualized_vol',
                 'underlying_bid',
                 'underlying_ask',
                 'underlying_bid_ask_spread',
                 'distance_from_strike',
                 'risk_free_rate',
                 'erT',
                 'trade_price']
    # Changing Order
    X = X[col_order] 
    
    
    ## Creating dataframes with subsets of information based on ordering
    # Returning only the 2019 year values
    X = X[X['datetime_year_month'] <= '2019-12']
    # Where Time to Maturity > 0
    X = X[X['time_to_maturity'] > 0]
    # Returning only rows where underlying ask != 0
    X = X[X['underlying_ask'] != 0]
    
    ## Resetting Index again for alignment purposes
    X = X.reset_index(drop = True)
        
    ## Return newly created dataframe
    return X
    

    
# applying the wrangle function from above
dfx = wrangle(df)

# CHECK:
print(dfx.shape)
dfx.head()

# -

# ## BMS Model (for SPX Options)
#
# ### Inputs
#
#
# **S:** Current Stock Price
#
# **X:** Strike
#
# **T:** Time to Expiration (days / 365)
#
# **r:** Annualized Risk-Free Rate (10-year, FFR as proxies??)
#
#
# **sigma:** Annualized Standard Deviation of Stock Returns (Annualized IV?)
#
# \begin{align}
# \sigma
# \end{align}
#
#
# **d1**:
#
# \begin{equation}
# d_{1} = \frac{\ln(\frac{{S}} {X}) + (r + (\frac{{\sigma^2}} 2))T }{\sigma{\sqrt T}}
# \end{equation}
#
# **d2**:
#
# \begin{equation}
# d_{2} = d_{1} - \sigma\sqrt(T)
# \end{equation}
#
# **N(d1):** Gaussian Distribution of d1 (Call)
#
# **(erf = error function for partial differential equation)**
#
# \begin{equation}
# N(d_{1}) = \frac{1 + erf(\frac{d_{1}} {\sqrt2})} {2}
# \end{equation}
#
#
# **N(d2):** Probability Call Option lands ITM
#
# \begin{equation}
# N(d_{2}) = \frac{1 + erf(\frac{d_{2}} {\sqrt2})} {2}
# \end{equation}
#
# **iN(d1):** Gaussian Distribution of d1 (Put)
#
# \begin{equation}
# iN(d_{1}) = \frac{1 + erf(\frac{-d_{1}} {\sqrt2})} {2}
# \end{equation}
#
# **iN(d2):** Probability Put Option lands ITM
#
# \begin{equation}
# iN(d_{2}) = \frac{1 + erf(\frac{-d_{2}} {\sqrt2})} {2}
# \end{equation}
#
#
# ### Outputs
#
#
#
#
#
#
# **Call:**
#
# \begin{equation}
# C = S_{0}N(d_{1}) - Xe^{-r{T}} N(d_{2})
# \end{equation}
#
# **Put:**
#
# \begin{equation}
# P = X{e^{−rT}}N(−d_{2}) − S_{0}N(−d_{1})
# \end{equation}

# +
from scipy.stats import norm
import numpy as np

# BSM Inputs Function
def bsm_inputs(dataf):
    
    Y = dataf.copy()
    
    S = Y['underlying_ask']
    K = Y['strike']
    r = Y['risk_free_rate']
    v = Y['annualized_vol']
    T = Y['T']
    
    D1 = ((np.log(S / K) + (r + (v ** 2 / 2))) * T) / (v * np.sqrt(T))
    Y['d1'] = D1
    
    D2 = D1 - (v * np.sqrt(T))
    Y['d2'] = D2
    
    Nd1 = norm.cdf(D1)
    Y['nd1'] = Nd1
    
    Nd2 = norm.cdf(D2)
    Y['nd2'] = Nd2
    
    return Y
 
# Calling the function
dfy = bsm_inputs(dfx)

# CHECK:
print(dfy.shape)
dfy.head()

# -

# # Modeling
#
# ### Train, Validate, Test Split

# +
##### Train, Validation and Test Set Split: #####
from sklearn.model_selection import train_test_split

# Splitting original df into training and testing sets
train, test = train_test_split(dfy, train_size = .9, test_size = .1, random_state = 42)

# Splitting training df into training and validation sets
train, val = train_test_split(train, train_size = .8, test_size = .2, random_state = 42)

# CHECK:
print(train.shape, val.shape, test.shape)
train.head()

# -

# ### Assigning Features and Target

# +
import category_encoders as ce
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error
from sklearn.metrics import r2_score

# Target and Features
target = 'trade_price'
baseline_features = ['underlying_ask', 'option_type']
bsm_features = ['underlying_ask', 'nd1', 'strike', 'erT', 'nd2', 'option_type']

other_features = ['trade_iv', 'trade_delta', 'option_bid_ask_spread',
                  'underlying_bid_ask_spread', 'risk_free_rate', 'annualized_vol']

all_features = bsm_features + other_features


# y target vector
y_train = train[target]
y_val = val[target]
y_test = test[target]
# -

# ### Linear Regression

# +
from sklearn.pipeline import make_pipeline

##### Linear Model Baseline #####

# Features Assignment
X_train_baseline = train[baseline_features]
X_val_baseline = val[baseline_features]

# Linear Regression Pipeline
lr_pipeline = make_pipeline(
    ce.OrdinalEncoder(),
    StandardScaler(),
    LinearRegression()
)

# Fitting
lr_pipeline.fit(X_train_baseline, y_train)

# Prediction
y_pred_baseline = lr_pipeline.predict(X_val_baseline)


# Scoring
def scoring_func(pipelinemodel, features, Xval, yval, ypred_val):
    # Coefficients
    coefs = pipelinemodel.named_steps['linearregression'].coef_
    train_coefs = pd.Series(coefs, features)
    print('Model Coefficients:')
    print(train_coefs)
    print('\n')

    # Intercept
    intercept = pipelinemodel.named_steps['linearregression'].intercept_
    print('Intercept:', intercept)
    print('\n')

    # Mean Square Error (MSE) and Root Mean Square Error (RMSE)
    mse = mean_squared_error(yval, ypred_val)
    rmse = np.sqrt(mse)
    r2 = r2_score(yval, ypred_val)
    print('Mean Absolute Error (MSE):', mse)
    print('Root Mean Square Error (RMSE):', rmse)
    print('R^2:', r2)
    
    return 

# Calling the function on the pipeline, feature matrices and target vectors
scoring_func(lr_pipeline, baseline_features, X_train_baseline, y_val, y_pred_baseline)

# +
from sklearn.pipeline import make_pipeline

##### Linear Model 1 #####

# Features Assignment
X_train_bsm = train[bsm_features]
X_val_bsm = val[bsm_features]

# Linear Regression Pipeline
lr_pipeline = make_pipeline(
    ce.OrdinalEncoder(),
    StandardScaler(),
    LinearRegression()
)

# Fitting
lr_pipeline.fit(X_train_bsm, y_train)

# Prediction
y_pred_bsm = lr_pipeline.predict(X_val_bsm)

# Calling Scoring Function
scoring_func(lr_pipeline, bsm_features, X_train_bsm, y_val, y_pred_bsm)

# +
##### Linear Model 2 #####

# Features 
X_train_other = train[other_features]
X_val_other = val[other_features]

# Linear Regression Pipeline
lr_pipeline2 = make_pipeline(
    ce.OrdinalEncoder(),
    StandardScaler(),
    LinearRegression()
)

# Fitting
lr_pipeline2.fit(X_train_other, y_train)

# Prediction
y_pred_other = lr_pipeline2.predict(X_val_other)

# Calling Scoring Function
scoring_func(lr_pipeline2, other_features, X_train_other, y_val, y_pred_other)

# +
##### Linear Model 3 #####

# Features 
X_train_all = train[all_features]
X_val_all = val[all_features]

# Linear Regression Pipeline
lr_pipeline3 = make_pipeline(
    ce.OrdinalEncoder(),
    StandardScaler(),
    LinearRegression()
)

# Fitting
lr_pipeline3.fit(X_train_all, y_train)

# Prediction
y_pred_all = lr_pipeline3.predict(X_val_all)

# Calling Scoring Function
scoring_func(lr_pipeline3, all_features, X_train_all, y_val, y_pred_all)
# -

# ### Permutation Importances on Features

# +
import eli5
from eli5.sklearn import PermutationImportance
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error
from sklearn.metrics import r2_score


### Linear Model 1 ###


# X Features Matrix
X_train = train[bsm_features]
X_val = val[bsm_features]
X_test = test[bsm_features]

# X Features Matrix (Encoded and Scaled)
encoder = ce.OrdinalEncoder()
scaler = StandardScaler()

X_train = encoder.fit_transform(X_train)
X_val = encoder.transform(X_val)
X_test = encoder.transform(X_test)

X_train_sc = scaler.fit_transform(X_train)
X_val_sc = scaler.transform(X_val)
X_test_sc = scaler.transform(X_test)


# Creating a df for lm features and data ONLY
X_train_lm = pd.DataFrame(X_train_sc, columns = bsm_features)
X_val_lm = pd.DataFrame(X_val_sc, columns = bsm_features)
X_test_lm = pd.DataFrame(X_test_sc, columns = bsm_features)


# Instantiate Model
lr = LinearRegression()


# Fitted Model
lin_reg1 = lr.fit(X_train_lm, y_train)


# Prediction of X_val
y_pred_val = lin_reg1.predict(X_val_lm)

    
# Creating the permuter importance 
permutation = PermutationImportance(
    lin_reg1,
    scoring = 'r2',
    n_iter = 10,
    random_state = 42
)


# Fitting 
permutation.fit(X_val, y_val)
    
## eli5 Weightings Chart
# Adding feature names to a list
feature_names = X_val.columns.tolist()
    
# eli5 Chart
eli5_weights = eli5.show_weights(
    permutation,
    top = None,
    feature_names = feature_names
)
 
# Return    
eli5_weights

# -

# #### Other Features

# +
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_error
from sklearn.metrics import r2_score


### Linear Model 2 ###


# X Features Matrix
X_train = train[other_features]
X_val = val[other_features]
X_test = test[other_features]

# X Features Matrix (Encoded and Scaled)
encoder = ce.OrdinalEncoder()
scaler = StandardScaler()

# Encoding
X_train = encoder.fit_transform(X_train)
X_val = encoder.transform(X_val)
X_test = encoder.transform(X_test)

# Scaling
X_train_sc = scaler.fit_transform(X_train)
X_val_sc = scaler.transform(X_val)
X_test_sc = scaler.transform(X_test)


# Creating a df for lm features and data ONLY
X_train_lm = pd.DataFrame(X_train_sc, columns = bsm_features)
X_val_lm = pd.DataFrame(X_val_sc, columns = bsm_features)
X_test_lm = pd.DataFrame(X_test_sc, columns = bsm_features)


# Instantiate Model
lr = LinearRegression()


# Fitted Model
lin_reg2 = lr.fit(X_train_lm, y_train)


# Prediction of X_val
y_pred_val = lin_reg2.predict(X_val_lm)


# Creating the permuter importance 
permutation = PermutationImportance(
    lin_reg2,
    scoring = 'r2',
    n_iter = 10,
    random_state = 42
)
    
# Fitting 
permutation.fit(X_val, y_val)
    
## eli5 Weightings Chart
# Adding feature names to a list
feature_names = X_val.columns.tolist()
    
# eli5 Chart
eli5_weights = eli5.show_weights(
    permutation,
    top = None,
    feature_names = feature_names
)
 
# Return    
eli5_weights

# +
### Linear Model 3 ###


# X Features Matrix
X_train = train[all_features]
X_val = val[all_features]
X_test = test[all_features]

# X Features Matrix (Encoded)
encoder = ce.OrdinalEncoder()

X_train = encoder.fit_transform(X_train)
X_val = encoder.transform(X_val)
X_test = encoder.transform(X_test)

# Standard Scaler
scaler = StandardScaler()

X_train_sc = scaler.fit_transform(X_train)
X_val_sc = scaler.transform(X_val)
X_test_sc = scaler.transform(X_test)


# Creating a df for lm features and data ONLY
X_train_lm = pd.DataFrame(X_train_sc, columns = all_features)
X_val_lm = pd.DataFrame(X_val_sc, columns = all_features)
X_test_lm = pd.DataFrame(X_test_sc, columns = all_features)


# Instantiate Model
lr = LinearRegression()


# Fitted Model
lin_reg3 = lr.fit(X_train_lm, y_train)


# Prediction of X_val
y_pred_val = lin_reg3.predict(X_val_lm)


# Creating the permuter importance 
permutation = PermutationImportance(
    lin_reg3,
    scoring = 'r2',
    n_iter = 10,
    random_state = 42
)
    
# Fitting 
permutation.fit(X_val, y_val)
    
## eli5 Weightings Chart
# Adding feature names to a list
feature_names = X_val.columns.tolist()
    
# eli5 Chart
eli5_weights = eli5.show_weights(
    permutation,
    top = None,
    feature_names = feature_names
)
 
# Return    
eli5_weights
# -

# ### Random Forest Regressor
#
# #### Random Forest Model 1

# +
# %matplotlib inline
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestRegressor

# X Features Matrix
X_train = train[bsm_features]
X_val = val[bsm_features]
X_test = test[bsm_features]

# X Features Matrix (Encoded)
encoder = ce.OrdinalEncoder()

X_train = encoder.fit_transform(X_train)
X_val = encoder.transform(X_val)
X_test = encoder.transform(X_test)

# Instantiate Model
rf = RandomForestRegressor(n_jobs = -1)

# Fitted Model
rand_forest1 = rf.fit(X_train, y_train)
y_pred_test = rf.predict(X_test)
r2 = r2_score(y_test, y_pred_test)
print('R^2 (Validation):', rand_forest1.score(X_val, y_val))
print('R^2 (Testing):', r2)


# +
### Graphing:

# Getting Features Importances
importances = pd.Series(rand_forest1.feature_importances_, X_train.columns)
        
# Plotting Importances
plt.figure(figsize = (10, 10))
plt.title('Feature Importances')
importances.sort_values(ascending = True).plot.barh(color = 'blue');

# +
import eli5
from eli5.sklearn import PermutationImportance
    
# Creating the permuter importance 
permutation = PermutationImportance(
    rand_forest1,
    scoring = 'r2',
    n_iter = 10,
    random_state = 42
)
    
# Fitting 
permutation.fit(X_val, y_val)
    
## eli5 Weightings Chart
# Adding feature names to a list
feature_names = X_val.columns.tolist()
    
# eli5 Chart
eli5_weights = eli5.show_weights(
    permutation,
    top = None,
    feature_names = feature_names
)
 
# Return    
eli5_weights

# -

# #### Random Foest Model 2

# +
# %matplotlib inline
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestRegressor

# X Features Matrix
X_train = train[other_features]
X_val = val[other_features]
X_test = test[other_features]

# X Features Matrix (Encoded)
encoder = ce.OrdinalEncoder()

X_train = encoder.fit_transform(X_train)
X_val = encoder.transform(X_val)
X_test = encoder.transform(X_test)

# Instantiate Model
rf = RandomForestRegressor(n_jobs = -1)

# Fitted Model
rand_forest2 = rf.fit(X_train, y_train)
print('R^2:', rand_forest2.score(X_val, y_val))

# +
### Graphing:

# Getting Features Importances
importances = pd.Series(rand_forest2.feature_importances_, X_train.columns)
        
# Plotting Importances
plt.figure(figsize = (10, 10))
plt.title('Feature Importances')
importances.sort_values(ascending = True).plot.barh(color = 'blue');

# +
import eli5
from eli5.sklearn import PermutationImportance
    
# Creating the permuter importance 
permutation = PermutationImportance(
    rand_forest2,
    scoring = 'r2',
    n_iter = 10,
    random_state = 42
)
    
# Fitting 
permutation.fit(X_val, y_val)
    
## eli5 Weightings Chart
# Adding feature names to a list
feature_names = X_val.columns.tolist()
    
# eli5 Chart
eli5_weights = eli5.show_weights(
    permutation,
    top = None,
    feature_names = feature_names
)
 
# Return    
eli5_weights


StarcoderdataJupyterScriptsDedupFiltered