import numpy as np 
import pandas as pd 
from pandas import datetime
pd.set_option('display.max_columns', None)  
pd.set_option('display.expand_frame_repr', False)
pd.set_option('max_colwidth', None)

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))


import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px

from copy import deepcopy
from sklearn.preprocessing import MinMaxScaler

from tensorflow.keras.preprocessing.sequence import TimeseriesGenerator # Generates batches for sequence data
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense,LSTM
from tensorflow.keras.callbacks import EarlyStopping

#import warnings
#warnings.filterwarnings("ignore")
"""
It was written that prices-split-adjusted was done on the basis of prices, so we will use only 'adjusted' one. Let's load the other data and see if it is useful to predict close price.
"""
prices = pd.read_csv('/kaggle/input/nyse/prices-split-adjusted.csv')
fundamentals = pd.read_csv('/kaggle/input/nyse/fundamentals.csv')
securities = pd.read_csv('/kaggle/input/nyse/securities.csv')
prices.head(10)
securities.head(10)
fundamentals.head(10)
"""
So, we have seen that 'prices' contains date, symbol, volume and prices variables like open, close, high and low. 

"Fundamentals' doesn't look very promicing in predicting price.

'Securities' contains additional info about the companies, like their sectors, full titles, headquarters, etc. Let's explore what companies are in Information Technology sector.
"""
securities[securities['GICS Sector'] == 'Information Technology']['Security'].unique()
#let's have a look at how things went for 'Visa Inc.'
securities[securities['Security']=='Visa Inc.'] #to get ticker symbol, it's "V"
visa = prices[prices['symbol']=="V"] #making a subset of prices as a new set for Visa Inc
visa = visa.drop('symbol', 1)
visa
"""
We got 1762 rows and 7 columns, let's explore them.
"""
import pandas_profiling
profile = pandas_profiling.ProfileReport(visa)
profile
"""
From profile report we can conclude that there are no missing values, prices have range from 16 to 84 and there is a strong correlation between prices variables. Another important note is that date column is categorical, not a datetime object. We need to fix this.
"""
visa['date'] = pd.to_datetime(visa['date'])
visa.info()
px.line(visa, x = 'date', y = ['open', 'close'])
px.line(visa, x = 'date', y = ['low', 'high'])
px.line(visa, x = 'date', y = ['volume'])
"""
General trend shows increasing of prices year by year while the volume is decreasing (with some outliers).
"""
df1 = deepcopy(visa) #for Neural Prophet, contains 'date'
df = deepcopy(visa) #for LSTM, dropping 'date' as a column, using as index
df.set_index("date", inplace=True)
#scaling values in order to look at all of them on one graph
for feature in ['open', 'close', 'high', 'low', 'volume']:
    sc = MinMaxScaler()
    visa[feature] = sc.fit_transform(visa[feature].values.reshape(-1,1))
    
#looking at all numerical features at once    
px.line(visa, x = 'date', y = ['open', 'close', 'high', 'low', 'volume'])
len(df) # 1762
test_point = 50 #We want to predict for the last 50 values
test_index = int(len(df) - test_point) #1712
train = df.iloc[:test_index,1:2] #to get the first 1712 values of the index and vwap
test = df.iloc[test_index:,1:2] #to get the last 50 values of the index and vwap
train #to look at output (under cut)
scaler = MinMaxScaler() #scaler for df dataset
scaler.fit(train) #fit the scaler on train set
MinMaxScaler(copy=True, feature_range=(0, 1))
scaled_train = scaler.transform(train) #transform on train and test sets separately
scaled_test = scaler.transform(test)
length = 49 #for input shape
batch_size = 1
n_features=1 #target variable

#creating generators for fitting a model
generator = TimeseriesGenerator(scaled_train,scaled_train, length=length,batch_size=batch_size)
validation_generator = TimeseriesGenerator(scaled_test,scaled_test,length=length,batch_size=batch_size)

#very simple Sequantial model
model = Sequential()
model.add(LSTM(50,input_shape=(length,n_features)))
model.add(Dense(1))

#adam optimizer, loss = mean squared error
model.compile(optimizer = 'adam',loss='mse',metrics=['accuracy'])

model.fit_generator(generator,epochs=20,validation_data=validation_generator)
test_predictions = [] #variable for predictions
first_eval_batch = scaled_train[-length:] #(49,1)
current_batch = first_eval_batch.reshape(1,length,n_features) #reshape to (1,49,1)

#using a loop to predict: predict the first value,add the resut to test predictions 
#and replace the value in current batch. To make next predictions on the previous predictions
for i in range(len(test)):
    current_pred = model.predict(current_batch)[0]
    test_predictions.append(current_pred)
    current_batch = np.append(current_batch[:,1:,:],
                              [[current_pred]],axis = 1)
#inversing predictions from scaled to normal
true_predictions = scaler.inverse_transform(test_predictions)
test['LSTM Predictions'] = true_predictions #add predictions to test set
test.plot(figsize=(12,8)) #plotting
df1 = df1[["date", "close"]]
df1.rename(columns={"date": "ds", "close": "y"}, inplace=True)
df_train = df1.iloc[:test_index] #splitting data for not scaled train set
#we already have test set as test
!pip install git+https://github.com/ourownstory/neural_prophet.git  
!pip install livelossplot    #it will allow us to use plot_live_live parameter 
#in the train function to get live training and validation plots.
from neuralprophet import NeuralProphet
model = NeuralProphet(n_changepoints=10,
                      trend_reg=0.1,
                      yearly_seasonality=False,
                      weekly_seasonality=False,
                      daily_seasonality=True)
metrics = model.fit(df_train, validate_each_epoch=True, 
                    valid_p=0.2, freq='D', 
                    plot_live_loss=True, 
                    epochs=100)
future = model.make_future_dataframe(df_train, 
                periods=50) #predict for 50 days
forecast = model.predict(future)
test["Forecast_Prophet"] = forecast.yhat1.values
test.plot(figsize=(14, 7)) #plotting
from sklearn.metrics import mean_absolute_error
print("Results for the test set:")
print("MAE of LSTM:", mean_absolute_error(test.close, 
                            test['LSTM Predictions']))
print("MAE of Neural Prophet:", 
      mean_absolute_error(test.close, test.Forecast_Prophet))
"""
We can look at MAE scores and admit that LSTM does a better job, but here it is even more obvious if we look on the graph. LSTM shows a nice generalization for 50 days. However, if we needed prediction for 20 days, Neural Prophet could be a better choice.
"""