"""
# STOCK PREDICTION USING TWITTER SENTIMENT ANALYSIS
"""
"""
#### importing machine learning libraries
"""
import numpy as np
import pandas as pd
from nltk.classify import NaiveBayesClassifier
from nltk.corpus import subjectivity
from nltk.sentiment import SentimentAnalyzer
from nltk.sentiment.util import *
import matplotlib.pyplot as mlpt
import math
import keras
from keras.models import Sequential
from keras.layers import Dense
from keras.metrics import Accuracy
from keras.layers import LSTM
from keras.layers import Dropout
from keras.layers import *
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import train_test_split
from keras.callbacks import EarlyStopping
"""
#### importing library to fetch data from twitter
"""
!pip3 install tweepy
import tweepy
import csv
import pandas as pd
import random
import numpy as np
import pandas as pd
"""
#### setting up consumer key and access token
"""
consumer_key    = '3jmA1BqasLHfItBXj3KnAIGFB'
consumer_secret = 'imyEeVTctFZuK62QHmL1I0AUAMudg5HKJDfkx0oR7oFbFinbvA'

access_token  = '265857263-pF1DRxgIcxUbxEEFtLwLODPzD3aMl6d4zOKlMnme'
access_token_secret = 'uUFoOOGeNJfOYD3atlcmPtaxxniXxQzAU4ESJLopA1lbC'

auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tweepy.API(auth,wait_on_rate_limit=True)
"""
#### Fetching tweets for Google from Sundar Pichai's timeline in extended mode (means entire tweet will come and not just few words + link)
"""
fetch_tweets=tweepy.Cursor(api.user_timeline, screen_name='@sundarpichai', tweet_mode="extended").items()
# fetch_tweets=tweepy.Cursor(api.search, q="#YESBANK",count=100000, lang ="en",since="2020-01-01", tweet_mode="extended").items()
data=pd.DataFrame(data=[[tweet_info.created_at.date(),tweet_info.full_text]for tweet_info in fetch_tweets],columns=['Date','Tweets'])
"""
#### Removing special character from each tweets
"""
data.to_csv("Tweets.csv")
cdata=pd.DataFrame(columns=['Date','Tweets'])
total=100
index=0
for index,row in data.iterrows():
    stre=row["Tweets"]
    my_new_string = re.sub('[^ a-zA-Z0-9]', '', stre)
    cdata.sort_index()
    cdata.at[index,'Date'] = row["Date"]
    cdata.at[index,'Tweets'] = my_new_string
    index=index+1
#print(cdata.dtypes)
"""
#### Displaying the data with date and tweets, you can notice there are multiple tweets for each day. So we will club them together later.
"""
print(cdata, min(cdata['Date']))
"""
#### Creating a dataframe where we will combine the tweets date wise and store into
"""
ccdata=pd.DataFrame(columns=['Date','Tweets'])
indx=0
get_tweet=""
for i in range(0,len(cdata)-1):
    get_date=cdata.Date.iloc[i]
    next_date=cdata.Date.iloc[i+1]
    if(str(get_date)==str(next_date)):
        get_tweet=get_tweet+cdata.Tweets.iloc[i]+" "
    if(str(get_date)!=str(next_date)):
        ccdata.at[indx,'Date'] = get_date
        ccdata.at[indx,'Tweets'] = get_tweet
        indx=indx+1
        get_tweet=" "
"""
#### All the tweets has been clubbed as per their date.
"""
ccdata
"""
#### Now to know the "closing price" of each day we will import STOCK PRICE DATA for GOOGLE from "yahoo.finance". We will consider "Close" price only.
"""

read_stock_p=pd.read_csv('../input/nsefinancedata/stocks.csv')
read_stock_p
"""
#### Adding a "Price" column in our dataframe and fetching the stock price as per the date in our dataframe.
"""
ccdata['Prices']=""
indx=0
for i in range (0,len(ccdata)):
    for j in range (0,len(read_stock_p)):
        get_tweet_date=ccdata.Date.iloc[i]
        get_stock_date=read_stock_p.Date.iloc[j]
        if(str(get_stock_date)==str(get_tweet_date)):
            #print(get_stock_date," ",get_tweet_date)
            ccdata.at[i,'Prices'] = int(read_stock_p.Close[j])
            break
"""
#### Prices are fetched but some entires are blank as close price might not be available for that day due to some reason (like holiday, etc.)
"""
print(ccdata)
"""
#### So we take the mean for the close price and put it in the blank value
"""
mean=0
summ=0
count=0
for i in range(0,len(ccdata)):
    if(ccdata.Prices.iloc[i]!=""):
        summ=summ+int(ccdata.Prices.iloc[i])
        count=count+1
mean=summ/count
for i in range(0,len(ccdata)):
    if(ccdata.Prices.iloc[i]==""):
        ccdata.Prices.iloc[i]=int(mean)
"""
#### Now all the entries have some value
"""
ccdata
"""
#### Making "prices" column as integer so mathematical operations could be performed easily.
"""
ccdata['Prices'] = ccdata['Prices'].apply(np.int64)
"""
#### Adding 4 new columns in our dataframe so that sentiment analysis could be performed.. Comp is "Compound" it will tell whether the statement is overall negative or positive. If it has negative value then it is negative, if it has positive value then it is positive. If it has value 0, then it is neutral.
"""
ccdata["Comp"] = ''
ccdata["Negative"] = ''
ccdata["Neutral"] = ''
ccdata["Positive"] = ''
ccdata
"""
#### Downloading this package was essential to perform sentiment analysis.
"""
import nltk
nltk.download('vader_lexicon')
"""
#### This part of the code is responsible for assigning the polarity for each statement. That is how much positive, negative, neutral you statement is. And also assign the compound value that is overall sentiment of the statement.
"""
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import unicodedata
sentiment_i_a = SentimentIntensityAnalyzer()
for indexx, row in ccdata.T.iteritems():
    try:
        sentence_i = unicodedata.normalize('NFKD', ccdata.loc[indexx, 'Tweets'])
        sentence_sentiment = sentiment_i_a.polarity_scores(sentence_i)
        ccdata.at[indexx, 'Comp'] = sentence_sentiment['compound']
        ccdata.at[indexx, 'Negative'] = sentence_sentiment['neg']
        ccdata.at[indexx, 'Neutral'] = sentence_sentiment['neu']
        ccdata.at[indexx, 'Positive'] = sentence_sentiment['pos']
    except TypeError:
        print (stocks_dataf.loc[indexx, 'Tweets'])
        print (indexx)
ccdata
"""
#### Calculating the percentage of postive and negative tweets, and plotting the PIE chart for the same.
"""
posi=0
nega=0
for i in range (0,len(ccdata)):
    get_val=ccdata.Comp[i]
    if(float(get_val)<(0)):
        nega=nega+1
    if(float(get_val>(0))):
        posi=posi+1
posper=(posi/(len(ccdata)))*100
negper=(nega/(len(ccdata)))*100
print("% of positive tweets= ",posper)
print("% of negative tweets= ",negper)
arr=np.asarray([posper,negper], dtype=int)
mlpt.pie(arr,labels=['positive','negative'])
mlpt.plot()
"""
#### Making a new dataframe with necessary columns for providing machine learning.
"""
df_=ccdata[['Date','Prices','Comp','Negative','Neutral','Positive']].copy()
df_
"""
#### Dividing the dataset into train and test.
"""
train_start_index = '0'
train_end_index = '6'
test_start_index = '7'
test_end_index = '9'
train = df_.loc[train_start_index : train_end_index]
test = df_.loc[test_start_index:test_end_index]
"""
#### Making a 2D array that will store the Negative and Positive sentiment for Training dataset.
"""
sentiment_score_list = []
for date, row in train.T.iteritems():
    sentiment_score = np.asarray([df_.loc[date, 'Negative'],df_.loc[date, 'Positive']])
    sentiment_score_list.append(sentiment_score)
numpy_df_train = np.asarray(sentiment_score_list)
print(numpy_df_train)
"""
#### Making a 2D array that will store the Negative and Positive sentiment for Testing dataset.
"""
sentiment_score_list = []
for date, row in test.T.iteritems():
    sentiment_score = np.asarray([df_.loc[date, 'Negative'],df_.loc[date, 'Positive']])
    sentiment_score_list.append(sentiment_score)
numpy_df_test = np.asarray(sentiment_score_list)
print(numpy_df_test)
"""
#### Making 2 dataframe for Training and Testing "Prices". You can also make 1-D array for the same.
"""
y_train = pd.DataFrame(train['Prices'])
#y_train=[91,91,91,92,91,92,91]
y_test = pd.DataFrame(test['Prices'])
print(y_train)
"""
#### Fitting the sentiments(this acts as in independent value) and prices(this acts as a dependent value (like class-lables in iris dataset))
"""
!pip3 install treeinterpreter
from treeinterpreter import treeinterpreter as ti
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import classification_report,confusion_matrix

rf = RandomForestRegressor()
rf.fit(numpy_df_train, y_train)
"""
#### Making Predictions
"""
prediction, bias, contributions = ti.predict(rf, numpy_df_test)
print(prediction)
"""
#### Importing matplotlib library for plotting graph
"""
import matplotlib.pyplot as plt
"""
#### Defining index position for the test data. Making dataframe for the predicted value.
"""
idx=np.arange(int(test_start_index),int(test_end_index)+198)
predictions_df_ = pd.DataFrame(data=prediction[0:], index = idx, columns=['Prices'])
predictions_df_

"""
#### Plotting the graph for the Predicted_price VS Actual Price
"""
ax = predictions_df_.rename(columns={"Prices": "predicted_price"}).plot(title='Random Forest predicted prices')#predicted value
ax.set_xlabel("Indexes")
ax.set_ylabel("Stock Prices")
fig = y_test.rename(columns={"Prices": "actual_price"}).plot(ax = ax).get_figure()#actual value
fig.savefig("random forest.png")
from treeinterpreter import treeinterpreter as ti
from sklearn.tree import DecisionTreeRegressor
from sklearn.linear_model import LinearRegression
from sklearn.metrics import classification_report,confusion_matrix

reg = LinearRegression()
reg.fit(numpy_df_train, y_train)
reg.predict(numpy_df_test)

# reg.score(reg.predict(numpy_df_test), y_test)

"""
Gather The STock DATA from Yahoo FInance
"""
df = pd.read_csv('../input/googlestockdata/stocks.csv')
df.head(5)
"""
Divide the data into Training and Testing
"""
training_set = df.iloc[:800, 1:2].values
test_set = df.iloc[800:, 1:2].values
# Feature Scaling
sc = MinMaxScaler(feature_range = (0, 1))
training_set_scaled = sc.fit_transform(training_set)
# Creating a data structure with 60 time-steps and 1 output
X_train = []
y_train = []
for i in range(60, 800):
    X_train.append(training_set_scaled[i-60:i, 0])
    y_train.append(training_set_scaled[i, 0])
X_train, y_train = np.array(X_train), np.array(y_train)
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
#(740, 60, 1)
model = Sequential()
#Adding the first LSTM layer and some Dropout regularisation
model.add(LSTM(units = 50, return_sequences = True, input_shape = (X_train.shape[1], 1)))
model.add(Dropout(0.2))
# Adding a second LSTM layer and some Dropout regularisation
model.add(LSTM(units = 50, return_sequences = True))
model.add(Dropout(0.2))
# Adding a third LSTM layer and some Dropout regularisation
model.add(LSTM(units = 50, return_sequences = True))
model.add(Dropout(0.2))
# Adding a fourth LSTM layer and some Dropout regularisation
model.add(LSTM(units = 50))
model.add(Dropout(0.2))
# Adding the output layer
model.add(Dense(units = 1))
# Compiling the RNN
model.compile(optimizer = 'adam', loss = 'mean_squared_error')
# Fitting the RNN to the Training set
history = model.fit(X_train, y_train, epochs = 50, batch_size = 32)
# Getting the predicted stock price of 2017
dataset_train = df.iloc[:800, 1:2]
dataset_test = df.iloc[800:, 1:2]
dataset_total = pd.concat((dataset_train, dataset_test), axis = 0)
inputs = dataset_total[len(dataset_total) - len(dataset_test) - 60:].values
inputs = inputs.reshape(-1,1)
inputs = sc.transform(inputs)
X_test = []

for i in range(60, 519):
    X_test.append(inputs[i-60:i, 0])
    
X_test = np.array(X_test)
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
print(X_test.shape)
# (459, 60, 1)
"""
**Store the predicted Stock Price**
"""
predicted_stock_price = model.predict(X_test)
predicted_stock_price = sc.inverse_transform(predicted_stock_price)
df.columns
# Visualising the results
plt.plot(df.loc[800:,'Date'],dataset_test.values, color = 'red', label = 'Real GOOGLE Stock Price')
plt.plot(df.loc[800:,'Date'][:459],predicted_stock_price, color = 'blue', label = 'Predicted GOOGLE Stock Price')
plt.xticks(np.arange(0,459,50))
plt.title('GOOGLE Stock Price Prediction')
plt.xlabel('Time')
plt.ylabel('GOOGLE Stock Price')
plt.legend()
plt.show()