# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # CSE 163 Final Project
# ## Stock Price Prediction using Reddit News Headlines

import pandas as pd
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
from pprint import pprint
from sklearn.model_selection import RandomizedSearchCV
from sklearn.metrics import accuracy_score
from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import classification_report
from sklearn.metrics import accuracy_score,recall_score,precision_score,f1_score
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix
from sklearn.linear_model import SGDClassifier
from sklearn.ensemble import RandomForestClassifier
from subprocess import check_output
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Conv1D, GlobalMaxPooling1D

# ## Data preparation

# +
data = pd.read_csv('Combined_News_DJIA.csv')
train = data[data['Date'] <= '2015-01-01']
test = data[data['Date'] >= '2014-12-31']
# combine all the headlines into one string for each dates in lowercase
trainheadlines = []
testheadlines = []
for row in range(0,len(train.index)):
    trainheadlines.append(' '.join(str(x).lower() for x in train.iloc[row,2:27]))
for row in range(0, len(test.index)):
    testheadlines.append(' '.join(str(x).lower() for x in test.iloc[row, 2:27]))

data.describe()
# -

# Since we wish to predict up and down of stock price based on news headlines, we have assumed that a single word (i.e. unigram ) will not contain enough information of sentiment of word towards up and down of stock price.
#
# First we will naively verify the claim by calculating accuracy score of each n-grams on LogisticRegression and use the best n-gram.

# ## Vectorizing using Tf-Idf and n-gram
# ### Purpose: To find the best n-gram for future use

# +
# Use logistic regression to choose naively choose best n-gram
# We will also use Tf-idf in addition to n-gram to give a score to the words from headlines to identify which word or 
# words best represents a set of headlines, and use that score of each word or words as input to the

unigramVectorizer =  TfidfVectorizer( min_df=0.03, max_df=0.97, max_features = 200000, ngram_range = (1, 1))
bigramVectorizer =  TfidfVectorizer( min_df=0.03, max_df=0.97, max_features = 200000, ngram_range = (2, 2))

uniTrain = unigramVectorizer.fit_transform(trainheadlines)

biTrain = bigramVectorizer.fit_transform(trainheadlines)

uniTest = unigramVectorizer.transform(testheadlines)

biTest = bigramVectorizer.transform(testheadlines)
print("unigram train set shape: ", uniTrain.shape)
print("bigram train set shape: ", biTrain.shape)
print("unigram test set shape: ", uniTest.shape)
print("bigram test set shape: ", biTest.shape)
# -

biLogisticR = LogisticRegression()
uniLogisticR = LogisticRegression()
biLogisticR.fit(biTrain, train['Label'])
uniLogisticR.fit(uniTrain, train['Label'])
# Accuracy score of two
uni_accuracy = accuracy_score(test['Label'], uniLogisticR.predict(uniTest))
bi_accuracy = accuracy_score(test['Label'], biLogisticR.predict(biTest))
print("Unigram accuracy score: ", uni_accuracy)
print("Bigram accuracy score: ", bi_accuracy)


# show highest coefficient and corresponding feature
def head_tail(vectorizer, model):
    features = vectorizer.get_feature_names()
    coeff = list(model.coef_)[0]
    df = pd.DataFrame({"features": features,
                      "coefficient": coeff})
    df = df.sort_values('coefficient', ascending=False)
    result_df = pd.concat([df.head(), df.tail()])
    return result_df


head_tail(bigramVectorizer,biLogisticR)

# Although, bigram had accuracy greater than random, the words(features) are not yet intuitive.
#
# It looks like two word is better than one word, but still not enough to hold meaningful intuition.

trigramVectorizer = TfidfVectorizer(min_df=0.03, max_df=0.97, max_features = 200000, ngram_range = (3, 3))
triTrain = trigramVectorizer.fit_transform(trainheadlines)
triTest = trigramVectorizer.transform(testheadlines)
triLogisticR = LogisticRegression()
triLogisticR.fit(triTrain, train['Label'])
tri_accuracy = accuracy_score(test['Label'], triLogisticR.predict(triTest))
tri_accuracy

head_tail(trigramVectorizer,triLogisticR)

# combine all uni, bi, and tri
allVectorizer = TfidfVectorizer(min_df=0.03, max_df=0.97, max_features = 200000, ngram_range = (1, 3))
allTrain = allVectorizer.fit_transform(trainheadlines)
allTest = allVectorizer.transform(testheadlines)
allLogisticR = LogisticRegression()
allLogisticR.fit(allTrain, train['Label'])
all_accuracy = accuracy_score(test['Label'], allLogisticR.predict(allTest))
all_accuracy

head_tail(allVectorizer,allLogisticR)

# With the results above, we procede our stock price up and down prediction with **bi-gram** tf_idf

# ### Logisitic regression with bigram
#

# Previously, We already used Logistic Regression with bi-gram without any hyperparameter tuning. <br\>
# The follwoing is confusion matrix with accuracy score.

print("Bigram accuracy score: ", bi_accuracy)
confusion_matrix(test['Label'],biLogisticR.predict(biTest))

plt.matshow(confusion_matrix(test['Label'],biLogisticR.predict(biTest)))

# ### Hyperparameter tuning on Test data(No validation set)
#

sns.set()
best_c = 0
best_acc = 0
best_model = None
data_acc = []
for i in np.arange(0.1, 1.0, 0.001):
  model = LogisticRegression(C = i)
  model.fit(biTrain, train['Label'])
  
  y_train_pred = model.predict(biTrain)
  train_acc = accuracy_score(train['Label'], y_train_pred)
  
  y_test_pred = model.predict(biTest)
  test_acc = accuracy_score(test['Label'], y_test_pred)
  
  data_acc.append({'C': i, 'train accuracy': train_acc, 
               'test accuracy': test_acc})
  if(test_acc > best_acc):
    best_acc = test_acc
    best_c = i
    best_model = model
data_acc = pd.DataFrame(data_acc)
sns.relplot(kind='line', x='C', y='train accuracy', data=data_acc)
plt.savefig('train_acc.png')
sns.relplot(kind='line', x='C', y='test accuracy', data=data_acc)
plt.savefig('test_acc.png')
print("best_c: ", best_c)
print("best_acc: ", best_acc)
print("best_model: ", best_model)
confusion_matrix(test['Label'],best_model.predict(biTest))

# The result above is not quite regularized since it tested it's hyperparameters on test dataset, which make the model overfit to the test data. We will improve this by using cross validation and using validation set to test hyper parameters.

# With the visualization best hyperparameter C, which is inverse of regularization strength. Now with this idea, let's peform GRID SEARCH of range from 0.4 to 0.6 for each 0.001 steps

# +
grid_values = {'penalty': ['l2'],'C':np.arange(0.45, 0.55, 0.0001)}
grid_clf_acc = GridSearchCV(LogisticRegression(), param_grid = grid_values, verbose=1, cv=3)
grid_clf_acc.fit(biTrain, train['Label'])

#Predict values based on new parameters
y_pred_acc = grid_clf_acc.predict(biTest)

# New Model Evaluation metrics 
print('Accuracy Score : ' + str(accuracy_score(test['Label'],y_pred_acc)))
print('Precision Score : ' + str(precision_score(test['Label'],y_pred_acc)))
print('Recall Score : ' + str(recall_score(test['Label'],y_pred_acc)))
print('F1 Score : ' + str(f1_score(test['Label'],y_pred_acc)))
print('best model: ' + str(grid_clf_acc.best_estimator_))
#Logistic Regression (Grid Search) Confusion matrix
confusion_matrix(test['Label'],y_pred_acc)
# -

# We can see decrease in accuray by 0.01 but this is acceptable because this model is no longer overfit to test data. Now, let this logistic regression accuracy be our baseline for the accuracy of futher prediction by other classifiers. <br/>
# Let's try Random Forest

# ### Base Random Forest Model

#Create a Gaussian Classifier
base_randomForest=RandomForestClassifier(n_estimators=100, random_state=1)
#Train the model using the training sets y_pred=base_randomForest.predict(X_test)
base_randomForest.fit(biTrain,train['Label'])
y_pred = base_randomForest.predict(biTest)
base_randomForest_acc = accuracy_score(test['Label'],y_pred)
print("Base RandomForest Accuracy: ", base_randomForest_acc)

# Simple randomfrest classifier whoed 0.52 accuracy, which is still worse than our baseline.<br/> 
# Let's do radom search to narrow down the hyperparameter space to improve the accuracy by hyperparameter tuning later in Grid Search.

# ### RandomSearch Grid

# Number of trees in random forest
n_estimators = np.arange(100, 1000, 100)
# Number of features to consider at /3every split
max_features = ['sqrt'] # auto is using all features, sqrt is usually good for classification
# Maximum number of levels in tree
max_depth = np.arange(10, 110, 10)
max_depth = np.append(max_depth,None)
# Minimum number of samples required to split a node
min_samples_split = [2, 5, 10]
# Minimum number of samples required at each leaf node
min_samples_leaf = [1, 2, 4]
# Method of selecting samples for training each tree
bootstrap = [True, False]
# Create the random grid
random_grid = {'n_estimators': n_estimators,
               'max_features': max_features,
               'max_depth': max_depth,
               'min_samples_split': min_samples_split,
               'min_samples_leaf': min_samples_leaf,
               'bootstrap': bootstrap,
               'random_state': [1]}
pprint(random_grid)

# ### RandomSearch

randomForest = RandomForestClassifier()
randomForest_random = RandomizedSearchCV(estimator = randomForest, param_distributions = random_grid, n_iter = 100, cv = 3, verbose=1, n_jobs = -1)
# Fit the random search model
randomForest_random.fit(biTrain, train['Label'])


# ### Let's checkout best estimator of RandomSearch

def evaluate(model, test_features, test_labels):
    predictions = model.predict(test_features)
    errors = abs(predictions - test_labels)
    mape = 100 * np.mean(errors / test_labels)
    accuracy = 100 - mape
    print('Model Performance')
    print('Average Error: {:0.4f} degrees.'.format(np.mean(errors)))
    print('Accuracy = {:0.2f}%.'.format(accuracy))
    
    return accuracy



# Best parameters from random search
print(randomForest_random.best_estimator_)
print(accuracy_score(test['Label'], randomForest_random.predict(biTest)))

# ### Now let's use this values to tune hyperparameters more precisesly

# ### Grid Search

n_features = randomForest_random.best_estimator_.n_features_
max_feature_by3 = n_features // 3
param_grid = {
    'bootstrap': [True],
    'max_depth': np.arange(20, 51, 10),
    'max_features': ['sqrt', max_feature_by3],
    'min_samples_leaf': [4, 5],
    'min_samples_split': [8, 10, 12],
    'n_estimators': np.arange(300, 1000, 100),
    'random_state': [1]
}
# Instantiate the grid search model
grid_search = GridSearchCV(estimator = RandomForestClassifier(), param_grid = param_grid, 
                          cv = 3, n_jobs = -1, verbose = 1)
grid_search.fit(biTrain,train['Label'])

n_features = randomForest_random.best_estimator_.n_features_
max_feature_by3 = n_features // 3
param_grid = {
    'bootstrap': [True],
    'max_depth': np.arange(20, 51, 10),
    'max_features': ['sqrt', max_feature_by3],
    'min_samples_leaf': [4, 5],
    'min_samples_split': [8, 10, 12],
    'n_estimators': np.arange(400, 600, 50),
    'random_state': [1]
}
# Instantiate the grid search model
grid_search2 = GridSearchCV(estimator = RandomForestClassifier(), param_grid = param_grid, 
                          cv = 3, n_jobs = -1, verbose = 1)
grid_search2.fit(biTrain,train['Label'])

# We will definitely compare accuracy of models

grid_search.best_estimator_

accuracy_score(test['Label'],grid_search.predict(biTest))

# +
from sklearn.tree import export_graphviz
import pydot
best_grid_search = grid_search.best_estimator_
# Pull out one tree from the forest
tree = best_grid_search.estimators_[5]

# Export the image to a dot file
export_graphviz(tree, out_file = 'tree.dot', feature_names = feature_list, rounded = True, precision = 1)
# Use dot file to create a graph
(graph, ) = pydot.graph_from_dot_file('tree.dot')
# Write graph to a png file
graph.write_png('tree.png')
# -

# model 1: Hyperparameter tuning on Test data(No validation set)
# model 2: Hyperparameter tuning using validation set
# model 3: Base RandomForest
# model 4: RandomForest using RandomSearch
# model 5: RandomForest using GridSearch
models = ['model 1', 'model 2', 'model 3', 'model 4', 'model 5']
accs = [0.5883905013192612, 0.575197889182058, 0.5224274406332454, 0.5224274406332454, 0.5277044854881267]
sns.set(style="whitegrid")
sns.barplot(x=models, y=accs, palette="rocket")
plt.title('Compare Models\' Accuracy Score')
plt.ylabel('Accuracy')
plt.savefig('compare_acc.png')

epochs = np.arange(1, 10, 1)
dense = np.arange(8, 65, 8)
models = []
acc_result = []
for i in epochs:
    for j in dense:
        model = Sequential()
        model.add(Dense(j,input_shape=(biTrain.shape[1],)))
        model.add(Dropout(0.2))
        model.add(Activation('relu'))
        model.add(Dense(j))
        model.add(Dropout(0.2))
        model.add(Activation('relu'))
        model.add(Dense(1))
        model.add(Activation('sigmoid'))
        model.summary()
        model.compile(loss='binary_crossentropy',
                      optimizer='adam',
                      metrics=['acc'])
        model.fit(biTrain,train['Label'],batch_size=32,epochs=i,verbose=1,validation_split=0.2)
        test_loss, test_acc = model.evaluate(biTest, test['Label'])
        models.append(model)
        acc_result.append({'epochs': i, 'dense': j, 'test accuracy': test_acc})


acc_result[acc_result['test accuracy'] == max(acc_result['test accuracy'])]

acc_result

acc_result = pd.DataFrame(acc_result)
sns.set(rc={'figure.figsize':(16, 8)})
sns.scatterplot(x='epochs', y='dense', hue="test accuracy", size="test accuracy",data=acc_result, sizes=(10, 500))
plt.savefig('nn.png')

StarcoderdataJupyterScriptsDedupFiltered