# this notebook makes it very easy for you to create
# a file with sentiment from the subreddit of your
# choosing about a stock ticker which you select.
#
# it also pulls down daily stock market data for the
# same stock ticker via yahoo finace
# you can change the number of submissions you
# wish you analyze as well & for what time period
#
# this notebook is ready to run, and will produce two
# .csv files automatically
# the main parameters to change are all highlighted in the code below
# they are:-
#
# selectedsubreddit = the name of the subreddit you want to carry out the analysis on
# howmanysumbissions = how many submissions do you want to analyse. takes about 5 seconds for each one.
# selectedTickerSymbol = the ticker symbol you want to explore. in this case 'GME' of course
#
# i was pleased that i was able to pull real live data from the web
# and performing sentiment analysis i found most of this code on the internet
# and then customised it to get the data I wanted from the sub reddit
# r/wallstreetsbets. My idea was to carry out sentiment analysis
# against the top x number of comments
# for a month, for a specific stock. I only record the sentiment from that comment, if there is at least one mention
# of that stock. This wasn't a serious project, more just to prove the process works.

# I decided to leave this as a dataset creation tool for sentiment analysis on reddit posts vs the chosen stock data
# and have two files created. I will then add another notebook for EDA and possibly train a predictive model,
# in the unlikely event of correlation ! I need to add the following as features on the extract as well:-

# text blob: add polarity and subjectivity

# i left in these commands below. I comment them out on Pycharm, then activated for Kaggle. They install the required
# libraries

!pip install --upgrade pip
!pip install yfinance --no-dependencies
!pip install multitasking --no-dependencies
!pip install praw

#import libraries. yfinance is yahoo finance and praw handles the reddit requests
import yfinance as yf
import pandas as pd
from datetime import datetime
import praw
import nltk

# some global variables. sorry !
from numpy.core.defchararray import rstrip

global selectedTickerSymbolCount
global selectedTickerSymbol
selectedTickerSymbolCount = 0

# kaggle secrets functionality below. Such a great Kaggle function and only added because it was commented on
# so thank you
from kaggle_secrets import UserSecretsClient
user_secrets = UserSecretsClient()
secret_value_0 = user_secrets.get_secret("secret_client_id")
secret_value_1 = user_secrets.get_secret("secret_client_secret")
secret_value_2 = user_secrets.get_secret("secret_user_agent")

# <<<<< Change the string below to the ticker you want to analyze, the program only records comments with a mention of
# and it records how many mentions per post, which is added to the data as a feature. You might screen out all postings
# with less than 5 mentions of the chosen ticker for example >>>>>
selectedTickerSymbol = 'GME'

# <<<<< select your subreddit below >>>>
selectedsubreddit = 'wallstreetbets'

# <<<<< how many submissions to analyze? >>>>>
# 100000 should be the majority for the month, if not all, that's average 3,333 a day
# 10000 should suffice.  I did the math and I think 100K would be 6 days, so 10K half a day?
howmanysumbissions = 25000

#create a reddit object with PRAW
reddit = praw.Reddit(client_id=secret_value_0,
                     client_secret=secret_value_1,
                     user_agent=secret_value_2)

# get x number of top posts from the subreddit
top_posts = reddit.subreddit(selectedsubreddit).top('month', limit=howmanysumbissions)

# install textblob if not already installed using "pip install -U textblob"
from textblob import TextBlob

# Download VADER, if not downloaded
# nltk.download('vader_lexicon')
from nltk.sentiment.vader import SentimentIntensityAnalyzer

# create object for VADER sentiment function interaction
sia = SentimentIntensityAnalyzer()

# Sentiment analysis function for TextBlob tool
def text_blob_sentiment(review, sub_entries_textblob):
    analysis = TextBlob(review)
    if analysis.sentiment.polarity >= 0.0001:
        if analysis.sentiment.polarity > 0:
            sub_entries_textblob['positive'] = sub_entries_textblob['positive'] + 1
            return 'Positive'

    elif analysis.sentiment.polarity <= -0.0001:
        if analysis.sentiment.polarity <= 0:
            sub_entries_textblob['negative'] = sub_entries_textblob['negative'] + 1
            return 'Negative'
    else:
        sub_entries_textblob['neutral'] = sub_entries_textblob['neutral'] + 1
        return 'Neutral'


# sentiment analysis function for VADER tool
def nltk_sentiment(review, sub_entries_nltk):
    vs = sia.polarity_scores(review)
    if not vs['neg'] > 0.05:
        if vs['pos'] - vs['neg'] > 0:
            sub_entries_nltk['positive'] = sub_entries_nltk['positive'] + 1
            return 'Positive'
        else:
            sub_entries_nltk['neutral'] = sub_entries_nltk['neutral'] + 1
            return 'Neutral'

    elif not vs['pos'] > 0.05:
        if vs['pos'] - vs['neg'] <= 0:
            sub_entries_nltk['negative'] = sub_entries_nltk['negative'] + 1
            return 'Negative'
        else:
            sub_entries_nltk['neutral'] = sub_entries_nltk['neutral'] + 1
            return 'Neutral'
    else:
        sub_entries_nltk['neutral'] = sub_entries_nltk['neutral'] + 1
        return 'Neutral'

# replication of comment section of reddit post
def replies_of(top_level_comment, count_comment, sub_entries_textblob, sub_entries_nltk):
    global selectedTickerSymbolCount
    if len(top_level_comment.replies) == 0:
        count_comment = 0
        return
    else:
        for num, comment in enumerate(top_level_comment.replies):
            try:
                count_comment += 1
                # print('-' * count_comment, comment.body)
                text_blob_sentiment(comment.body, sub_entries_textblob)
                nltk_sentiment(comment.body, sub_entries_nltk)

                # Search for stock ticker symbol in comment body and increment counter if found
                if comment.body.find(selectedTickerSymbol) != -1:
                    selectedTickerSymbolCount += 1
            except:
                continue
            replies_of(comment, count_comment, sub_entries_textblob, sub_entries_nltk)


def main():
    # global int for recording how many times the program finds the stock ticker in the reddit comments
    # i only wanted to extract records with a mention of the selected stock. I also will pull that number into my model
    # as a feature.

    # creates empty pandas dataframe, which will be filled with data from the reddit comments as the program runs
    global selectedTickerSymbolCount
    commentDF = pd.DataFrame()

    # this creates a data frame called 'hist' and fills it with stock data for the selected stock. I am taking
    # all the data as there are less than 10,000 records. It then prints the data set.
    selectedTicker = yf.Ticker(selectedTickerSymbol)
    hist = selectedTicker.history(period="max")
    print('\n')
    print('\n')
    print('\n')
    print(hist)

    # loop through the submissions
    submissioncounter = 1
    for submission in top_posts:
        sub_entries_textblob = {'negative': 0, 'positive': 0, 'neutral': 0}
        sub_entries_nltk = {'negative': 0, 'positive': 0, 'neutral': 0}
        print('Submission:', submissioncounter,'   |  Title of the post :', submission.title)
        submissioncounter = submissioncounter + 1
        text_blob_sentiment(submission.title, sub_entries_textblob)
        nltk_sentiment(submission.title, sub_entries_nltk)
        # print("\n")
        submission_comm = reddit.submission(id=submission.id)

        for count, top_level_comment in enumerate(submission_comm.comments):
            # print(f"-------------{count} top level comment start--------------")
            count_comm = 0
            try:
                # print(top_level_comment.body)
                text_blob_sentiment(top_level_comment.body, sub_entries_textblob)
                nltk_sentiment(top_level_comment.body, sub_entries_nltk)
                replies_of(top_level_comment,
                           count_comm,
                           sub_entries_textblob,
                           sub_entries_nltk)
            except:
                continue

        # fetching the Unix timestamp from the comment
        unix_time = submission_comm.created_utc
        commentPostedUnixTime = str(datetime.fromtimestamp(unix_time))
        commentYMD = commentPostedUnixTime[0:11:1]

        # get the senitment values. These are from a dictionary so have to be access via the key
        # and then array location. The code uses both textblob and vader.
        textblobnegative = sub_entries_textblob.get('negative',0)
        textblobpositive = sub_entries_textblob.get('positive',0)
        textblobneutral = sub_entries_textblob.get('neutral',0)

        vadernegative = sub_entries_nltk.get('negative',0)
        vaderpositive = sub_entries_nltk.get('positive',0)
        vaderneutral = sub_entries_nltk.get('neutral',0)

        # if the program finds 1 or more mentions of the stock ticker in the comments, then it will record that record
        # and append it to the data set
        if selectedTickerSymbolCount > 0:

            commentYMD = commentYMD.rstrip()
            commentYMD = datetime.strptime(commentYMD, '%Y-%m-%d')

            rowsToAppend = {'Title': submission.title, 'Ticker': selectedTickerSymbol, 'Date': commentYMD,
                            'NumberOfTickerMentions': selectedTickerSymbolCount, 'Vader Neg': vadernegative,
                            'Vader Pos': vaderpositive, 'Vader Neut': vaderneutral, 'textblob Negative': textblobnegative
                            , 'textblob Positive': textblobpositive, 'textblob Neut': textblobneutral}
            commentDF = commentDF.append(rowsToAppend, ignore_index = True)
            selectedTickerSymbolCount = 0
            # reset the counter that records how many times the stock symbol is found in the comments


    # export the files as a .csv
    # comment_analysis: this has the results of the sentiment analysis
    # stockticker_history: this has the extract of the stock price information for the selected stock

    commentDF.to_csv(r'comment_analysis.csv', index=False, header=True)
    hist.to_csv(r'stockticker_history.csv', index=True,header=True)
    print('\n')
    print('Sentiment analysis is below:')
    print('\n')
    print(commentDF)
    print('\n')
    print('Process complete. Files have been saved.')
if __name__ == '__main__':
    main()
"""
I have written another notebook, which takes the output from the sentiment analysis, merges the stockdata and sentiment, then creates an EDA report using the wonderful Pandas Profiling functionality ! It also saves that as a .html report !!!

Link to that notebook is below. It is already setup with ~ 800 submissions ready to go as the input data (Wall Street Bets - GME submissions for the month Jan-21).

https://www.kaggle.com/iainmcintosh/fork-of-reddit-sentiment-vs-stockprice-eda
"""