# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: sentiment_analysis
#     language: python
#     name: sentiment_analysis
# ---

# # Sentiment Analysis 

# In the world of computational linguistics, sentiment analysis refers to the process of determining the general views, opinions, and emotions held within a text by analysing how positive or negative the language held within is. . It is a popular and useful tool for getting an idea of how opinions change over time or comparing the general mood between texts. For this reason, it is common to see sentiment analysis used within the world of cultural data science for exploring questions such as “How do opinions on the corona vaccine differ between countries?” or “Are news articles today more ‘doom and gloom’ than they were 20 years ago?”. 
#
# Here, we compare 2 dictionary based sentiment analysis approaches and compare the differing results given by each, due to their indivdiual ways of calculating sentiment. These approaches are SpaCyTextBlob and VADER. 

# # SpacyTextBlob

# SpaCyTextBlob (TextBlob) is built on top of the NLTK package and returns two properties for a given sentence, paragraph, or document. These are: 
#
#
# i)	__Polarity__ (runs between -1 and +1 whereby anything below 0 is considered as having negative sentiment and anything above 0 is considered to have positive sentiment). 
#
#
# ii)	__Subjectivity__ (a float number between 0 and 1 which indicates how subjective or objective a sentence is. Sentences with a high subjectivity score refer to a personal opinion, emotion, or judgement while sentences with low subjectivity are objective or refer to factual information)
#
#
# It also has an additional parameter called ‘Intensity’ which is used to modify the value of a following word. For example, if there is ‘really’ in front of a word, the intensity function will ‘intensify’ and/or ‘maximise’ the polarity value of the word which proceeds the ‘really’. 
#
#
# TextBlob uses a method of ‘averaging’ whereby it ignores the syntax and structure of the words and simply computes the average value for the entire sentence.

#Dependencies 
import os
import spacy #The main sentiment analysis sentiment analysis package to be used 
import pandas as pd #For working with dataframes and reading the data 
from spacytextblob.spacytextblob import SpacyTextBlob
import matplotlib.pyplot as plt 
import seaborn as sns

#Add spacytextblob to the nlp pipeline
nlp = spacy.load("en_core_web_sm") #We're using the English small library 
spacy_text_blob = SpacyTextBlob() #This is taken from the spacy website 
nlp.add_pipe(spacy_text_blob) 

data_path = "data/abcnews-date-text.csv"

#Load in the data as a dataframe
data = pd.read_csv(data_path)

data #we have a publishing date and headline text 

# +
#We'll also just create a subset of the data to run the script on for now 
#We'll take 10,000 headlines from a chunk in the middle of the data 
subset_data = data[15000:25000]

#And save it to our output folder 
subset_csv_path = os.path.join("data", "subset_data.csv")
subset_data.to_csv(subset_csv_path, index = False)

# +
#Beware - this takes some time to run! 
sentiment_tracker = []

# for every headline in data frame (we're looking at docs, not sentences)
for doc in nlp.pipe(data["headline_text"], batch_size = 10000): 
    # calculate the sentiment of the doc (headline)
    sentiment = doc._.sentiment.polarity
    # append this to sentiment_tracker list
    sentiment_tracker.append(sentiment)
# -

# append the sentiment_tracker list to the dataframe and save as output csv file in sentiment_plots
data.insert(len(data.columns), "sentiment", sentiment_tracker)
output_csv_path = os.path.join("Output", "sentiment_tracker.csv")
data.to_csv(output_csv_path, index = False)

# Next, create a sentiment dataframe with date as the index and sentiment as a column to calculate means based on date
df_sentiment = pd.DataFrame(
    {"sentiment": sentiment_tracker}, # create a column to hold sentiment scores 
    index = pd.to_datetime(data["publish_date"], format='%Y%m%d', errors='ignore')) # index the date using to_datetime

#Inspect the dataframe 
df_sentiment

#Create a column which calculates the 7-day and 30-day rolling averages 
df_sentiment['weekly'] = df_sentiment.sentiment.rolling(7).mean()
df_sentiment['monthly'] = df_sentiment.sentiment.rolling(30).mean()


#Build a function which creates a seaborn rolling average map 
def smoothed_sentiment_textblob(text_date_window):
    if text_date_window == "1-week":
        sns.set_style('darkgrid')
        plt.figure(figsize=(10,6), tight_layout=True)
        ax = sns.lineplot(data=df_sentiment,
                   x='publish_date', y='weekly',
                   label = "7 Day Average",
                   palette='Set2', 
                   linewidth=2.5) 
        ax.set(xlabel="Publish Date", ylabel = "Sentiment Score")
        ax.set(title = "Sentiment over time with a 7 day rolling average")
        ax.grid(color='grey', linestyle='-', linewidth=0.5)
        #save the plot 
        ax.figure.savefig(os.path.join("Output", "TextBlob_weekly_sentiment.png"))
    
    else:
        sns.set_style('darkgrid')
        plt.figure(figsize=(10,6), tight_layout=True)
        ax = sns.lineplot(data=df_sentiment,
                   x='publish_date', y='monthly',
                   label = "7 Day Average",
                   palette='Set2', 
                   linewidth=2.5) 
        ax.set(xlabel="Publish Date", ylabel = "Sentiment Score")
        ax.set(title = "Sentiment over time with a 30 day rolling average")
        ax.grid(color='grey', linestyle='-', linewidth=0.5)
        #save the plot 
        ax.figure.savefig(os.path.join("Output", "TextBlob_monthly_sentiment.png"))
        


# Plot the monthly rolling average 
smoothed_sentiment_textblob("1-month")

#Plot the weekly rolling average 
smoothed_sentiment_textblob("1-week")

# __Setting up for comparison against Vadar__
#
# A key part of our analysis is interested in comparing the SpacyTextBlob approach to NLTK's Vadar approach. To do this, we'll categorise our sentiment scores into 'positive', 'neutral', and 'negative' in a new column of our dataset. 
#
# This way, we can count how many fall into each category

# The next step is to categorise our sentiment so that we can compare it against the vadar sentiment analysis 
df_sentiment['Sentiment_Type']=''
df_sentiment.loc[df_sentiment.sentiment>0,'Sentiment_Type']='positive'
df_sentiment.loc[df_sentiment.sentiment==0,'Sentiment_Type']='neutral'
df_sentiment.loc[df_sentiment.sentiment<0,'Sentiment_Type']='negative'

#Inspect the column - we'll take a slice from the middle
df_sentiment[5000:5010]

# Our column is looking good, so the next thing we'd want to do is create a barchart. We'll employ seaborn again to make the chart more asthetically appealing. 

textblob_barplot = df_sentiment.Sentiment_Type.value_counts().plot(kind='bar',title="Sentiment Analysis")
textblob_barplot.figure.savefig(os.path.join("Output", "TextBlob_barplot.png"))

# # VADAR Sentiment Analysis 

# Vadar is an alternative sentiment analysis library which computes a different kind of sentiment. Unlike SpacyTextBlob which gives an overall 'sentiment' score, Vadar computes a 'positivity', 'negativity', and 'neutral' score as well as a 'compound' - which is a weighted average of the 3. In this way, the compound score given from vadar can be used to compare against SpacyTextBlob. This is always important because we have to know what's going on in our models to be able to trust them! 

# +
#Dependencies
import nltk
nltk.download('vader_lexicon')

from nltk.sentiment.vader import SentimentIntensityAnalyzer
# create a sentiment intensity analyzer
sid = SentimentIntensityAnalyzer()
# -

#Create a new dataframe of the same data 
vadar_data = pd.read_csv(data_path)
vadar_data

# Create a new column with the sentiment scores from vadar 
vadar_data['sentiment'] = vadar_data['headline_text'].apply(lambda headline_text: sid.polarity_scores(headline_text))

#Create a columm with just the compound value of the sentiment scores & create categories
vadar_data['compound'] = vadar_data['sentiment'].apply(lambda score_dict: score_dict['compound'])
vadar_data['sentiment_type']=''
vadar_data.loc[vadar_data.compound>0,'sentiment_type']='positive'
vadar_data.loc[vadar_data.compound==0,'sentiment_type']='neutral'
vadar_data.loc[vadar_data.compound<0,'sentiment_type']='negative'

vadar_data

#create a list from the compound column 
vadar_timeseries = vadar_data["compound"].tolist()

# Then create a dataframe with this list and the publish_date data converted into date form 
vadar_sentiment = pd.DataFrame(
    {"sentiment": vadar_timeseries}, # create a column to hold sentiment scores 
    index = pd.to_datetime(vadar_data["publish_date"], format='%Y%m%d', errors='ignore'))

#Inspect the dataframe 
vadar_sentiment

#Create a column which calculates the 7-day and 30-day rolling averages 
vadar_sentiment['weekly'] = vadar_sentiment.sentiment.rolling(7).mean()
vadar_sentiment['monthly'] = vadar_sentiment.sentiment.rolling(30).mean()

#Inspect the new columns (we'll just view 5)
vadar_sentiment[5000:5005]


# +
#Creating the rolling plots 

#Build a function which creates a seaborn rolling average map 
def smoothed_sentiment_vadar(text_date_window):
    if text_date_window == "1-week":
        sns.set_style('darkgrid')
        plt.figure(figsize=(10,6), tight_layout=True)
        ax = sns.lineplot(data=vadar_sentiment,
                   x='publish_date', y='weekly',
                   label = "7 Day Average",
                   palette='Set2', 
                   linewidth=2.5) 
        ax.set(xlabel="Publish_date", ylabel = "Sentiment Score")
        ax.set(title = "Sentiment over time with a 7 day rolling average")
        ax.grid(color='grey', linestyle='-', linewidth=0.5)
        #save the plot 
        ax.figure.savefig(os.path.join("Output", "VADAR_weekly_sentiment.png"))
    
    else:
        sns.set_style('darkgrid')
        plt.figure(figsize=(10,6), tight_layout=True)
        ax = sns.lineplot(data=vadar_sentiment,
                   x='publish_date', y='monthly',
                   label = "7 Day Average",
                   palette='Set2', 
                   linewidth=2.5) 
        ax.set(xlabel="Publish_date", ylabel = "Sentiment Score")
        ax.set(title = "Sentiment over time with a 30 day rolling average")
        ax.grid(color='grey', linestyle='-', linewidth=0.5)
        #save the plot 
        ax.figure.savefig(os.path.join("Output", "VADAR_monthly_sentiment.png"))
        


# -

smoothed_sentiment_vadar("1-week")

smoothed_sentiment_vadar("1-month")

# __Boxplot of Positive, Neutral, and Negative scores__

VADER_boxplot = vadar_data.sentiment_type.value_counts().plot(kind='bar',title="Vadar Sentiment Analysis")
VADER_boxplot.figure.savefig(os.path.join("Output", "VADER_barplot.png"))



StarcoderdataJupyterScriptsDedupFiltered