# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: 'Python 3.7.4 64-bit (''py3b'': conda)'
#     language: python
#     name: python37464bitpy3bconda9e63d4e9c0474755b9b5cc46d4cddbaf
# ---

import json
import spacy
import nltk
import requests
import sys
import datetime
import pandas as pd
import matplotlib.pyplot as plt

sys.path.append("..")
from crowdrank import interpreter
from crowdrank import postprocessing

#

df_ranking = pd.read_csv("../data/results/Headphones_360.csv", index_col=False)

df_ranking.columns = ['Brands', 'Sentiment', 'Popularity', 'Variance']
df_ranking

# For each comment..., if it contains one of these words, grab the next word:
#interpreter.get_and_interpret(['headphoneadvice'], 'Headphones')
comments_upvotes = interpreter.get_local('headphoneadvice', 360)

comments_upvotes[:10]

# ## Testing if a wider context results in "more sentiment"
# There is a significant fraction of interpreted entity data where the sentiment from VADER was 0.0...
# The short EDA below shows this is probably because people say things like "I use the BOSE SH500s. I
# love them because...". If we sentencize and only look at the sentiment from the sentence, we get 0.0.

kb = interpreter.Knowledgebase(comments_upvotes)

prod_sentiments_narrow = kb.interpret_with_sentiment()

prod_sentiments_wide = kb.interpret_with_sentiment('wide')

# Compare histograms of prod_sentiments_wide and prod_sentiments_narrow
# 2D plot of one vs the other
print([s[1] for s in prod_sentiments_narrow].count(0))
plt.hist([s[1] for s in prod_sentiments_narrow])

# +
print([s[1] for s in prod_sentiments_wide].count(0))

plt.hist([s[1] for s in prod_sentiments_wide])
# -

plt.scatter([s[2] for s in prod_sentiments_narrow], [s[1] for s in prod_sentiments_narrow])

plt.hist2d([s[2] for s in prod_sentiments_narrow if s[2] <= 4], [s[1] for s in prod_sentiments_narrow if s[2] <= 4], cmap=plt.cm.jet)
plt.colorbar()

plt.scatter([s[2] for s in prod_sentiments_wide], [s[1] for s in prod_sentiments_wide])

#bins=(50, 50),
plt.hist2d([s[2] for s in prod_sentiments_wide if s[2] <= 4], [s[1] for s in prod_sentiments_wide if s[2] <= 4], cmap=plt.cm.jet)
plt.colorbar()

# ### Size of the context matter immensely for sentiments
# Should we use the sentence in which the entity appears, or the whole post?

from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
s4p = ' I am using **xFyro xS2** waterproof Bluetooth earbud. I have enjoyed these earbuds immensely. You can use them for swimming, but swimming cap is must, otherwise, it will drop from your ear.\n\nAll in all I enjoy these very much and would recommend them.'

analyzer = SentimentIntensityAnalyzer()
sentiment = analyzer.polarity_scores(s)
sentiment

s4 = ' I am using **xFyro xS2** waterproof Bluetooth earbud.'
sentiment = analyzer.polarity_scores(s4)
sentiment

sentiment = analyzer.polarity_scores(s4p)
sentiment

nlp = spacy.load("en_core_web_md")
nlp.add_pipe(nlp.create_pipe('sentencizer'))
interpreter.interpret_paragraph([s4p, 1], nlp)

doc = nlp(s4p)
for ent in doc.ents:
    print(ent.text, ent.label)

s1 = "Sennheiser HD 6XX."
s2 = "Sennheiser HD 6XX. They are the best in my opinion. You'll need a decent amp to drive them to their full potential though; check out the JDS Labs Atom for that."

analyzer = SentimentIntensityAnalyzer()
sentiment1 = analyzer.polarity_scores(s1)
analyzer = SentimentIntensityAnalyzer()
sentiment2 = analyzer.polarity_scores(s2)
print(sentiment1)
print(sentiment2)

import spacy
nlp = spacy.load("en_core_web_md")
nlp.add_pipe(nlp.create_pipe('sentencizer'))
interpreter.interpret_paragraph(comments_upvotes[7], nlp)



StarcoderdataJupyterScriptsDedupFiltered