# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ## Set-up

# %run functions.ipynb
# %matplotlib inline

# +
import tweepy
import configparser
import os
import json
import GetOldTweets3 as got
import datetime
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from datetime import datetime
import nltk
from nltk.corpus import stopwords 
from nltk.tokenize import word_tokenize 
import string
import random
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import re
import csv
import math

from collections import Counter
# -

jan_tweets = load_tweets('data/1/tweets_2020-01-01_to_2020-02-01.json')
feb_tweets = load_tweets('data/2/tweets_2020-02-01_to_2020-03-01.json')
mar_tweets = load_tweets('data/3/tweets_2020-03-01_to_2020-04-01.json')
apr_tweets = load_tweets('data/4/tweets_2020-04-01_to_2020-05-01.json')
all_time = load_tweets('data/all_time/tweets_2020-01-01_to_2020-05-01.json')
trump_tweets = load_tweets('data/all_time/realdonaldtrump_2020-01-01_to_2020-05-01.json')
pompeo_tweets = load_tweets('data/all_time/secpompeo_2020-01-01_to_2020-05-01.json')
racist_tweets = load_tweets('data/all_time/racist_tweets_2020-01-01_to_2020-05-01.json')

corpus1 = json.load(open('data/corpus_index1.json'))
corpus2 = json.load(open('data/corpus_index2.json'))
corpus3 = json.load(open('data/corpus_index3.json'))
corpus4 = json.load(open('data/corpus_index4.json'))
corp_all = json.load(open('data/corpus_index_all.json'))

# ## News articles: bigram and trigrams over time

# #### January

# +
jan_word_dist=Counter()
jan_bigram_dist=Counter()
jan_trigram_dist=Counter()

for article in corpus1:
    filename = article['Filename']
    text = open('data/text1/{}'.format(filename)).read()
    article['text'] = text
    
    jan_tokens = tokenize(article['text'], lowercase=True, strip_chars=string.punctuation)
    
    article['tokens'] = jan_tokens 
    article['token_cnt'] = len(jan_tokens)
    article['type_cnt'] = len(set(jan_tokens))
    
    jan_tokens = article['tokens']
    jan_bigrams=get_ngram_tokens(jan_tokens,2)
    jan_trigrams=get_ngram_tokens(jan_tokens,3)
    
    jan_word_dist.update(jan_tokens)
    jan_bigram_dist.update(jan_bigrams)
    jan_trigram_dist.update(jan_trigrams)
# -

top_20_bigrams = jan_bigram_dist.most_common(20)
top_20_trigrams = jan_trigram_dist.most_common(20)
bigram_list = list(pd.DataFrame(top_20_bigrams, columns = ['Bigram','Freq'])['Bigram'])
trigram_list = list(pd.DataFrame(top_20_trigrams, columns = ['Trigram','Freq'])['Trigram'])

rank = list(range(1, 21))
jan_article_bitrigram = pd.DataFrame(rank, columns = ['Rank'])
jan_article_bitrigram['Bigram']=bigram_list
jan_article_bitrigram['Trigram']=trigram_list
jan_article_bitrigram.set_index('Rank', inplace=True)

jan_article_bitrigram

# #### February

feb_word_dist=Counter()
feb_bigram_dist=Counter()
feb_trigram_dist=Counter()
feb_all_tokens = []
for article in corpus2:
    filename = article['Filename']
    text = open('data/text2/{}'.format(filename)).read()
    article['text'] = text
    
    feb_tokens = tokenize(article['text'], lowercase=True, strip_chars=string.punctuation)
    feb_all_tokens.extend(feb_tokens)
    
    article['tokens'] = feb_tokens 
    article['token_cnt'] = len(feb_tokens)
    article['type_cnt'] = len(set(feb_tokens))
    
    feb_tokens = article['tokens']
    feb_bigrams=get_ngram_tokens(feb_tokens,2)
    feb_trigrams=get_ngram_tokens(feb_tokens,3)
    
    feb_word_dist.update(feb_tokens)
    feb_bigram_dist.update(feb_bigrams)
    feb_trigram_dist.update(feb_trigrams)

article_feb_top_20_bigrams = feb_bigram_dist.most_common(20)
article_feb_top_20_trigrams = feb_trigram_dist.most_common(20)
article_bigram_list = list(pd.DataFrame(article_feb_top_20_bigrams, columns = ['Bigram','Freq'])['Bigram'])
article_trigram_list = list(pd.DataFrame(article_feb_top_20_trigrams, columns = ['Trigram','Freq'])['Trigram'])

rank = list(range(1, 21))
feb_article_bitrigram = pd.DataFrame(rank, columns = ['Rank'])
feb_article_bitrigram['Bigram']=article_bigram_list
feb_article_bitrigram['Trigram']=article_trigram_list
feb_article_bitrigram.set_index('Rank', inplace=True)

feb_article_bitrigram 

# #### March

# +
mar_word_dist=Counter()
mar_bigram_dist=Counter()
mar_trigram_dist=Counter()

for article in corpus3:
    filename = article['Filename']
    text = open('data/text3/{}'.format(filename)).read()
    article['text'] = text
    
    mar_tokens = tokenize(article['text'], lowercase=True, strip_chars=string.punctuation)
    
    article['tokens'] = mar_tokens 
    article['token_cnt'] = len(mar_tokens)
    article['type_cnt'] = len(set(mar_tokens))
    
    mar_tokens = article['tokens']
    mar_bigrams=get_ngram_tokens(mar_tokens,2)
    mar_trigrams=get_ngram_tokens(mar_tokens,3)
    
    mar_word_dist.update(mar_tokens)
    mar_bigram_dist.update(mar_bigrams)
    mar_trigram_dist.update(mar_trigrams)
# -

top_20_bigrams = mar_bigram_dist.most_common(20)
top_20_trigrams = mar_trigram_dist.most_common(20)
bigram_list = list(pd.DataFrame(top_20_bigrams, columns = ['Bigram','Freq'])['Bigram'])
trigram_list = list(pd.DataFrame(top_20_trigrams, columns = ['Trigram','Freq'])['Trigram'])

rank = list(range(1, 21))
mar_article_bitrigram = pd.DataFrame(rank, columns = ['Rank'])
mar_article_bitrigram['Bigram']=bigram_list
mar_article_bitrigram['Trigram']=trigram_list
mar_article_bitrigram.set_index('Rank', inplace=True)

mar_article_bitrigram

# #### April

# +
apr_word_dist=Counter()
apr_bigram_dist=Counter()
apr_trigram_dist=Counter()

for article in corpus4:
    filename = article['Filename']
    text = open('data/text4/{}'.format(filename)).read()
    article['text'] = text
    
    apr_tokens = tokenize(article['text'], lowercase=True, strip_chars=string.punctuation)
    
    article['tokens'] = apr_tokens 
    article['token_cnt'] = len(apr_tokens)
    article['type_cnt'] = len(set(apr_tokens))
    
    apr_tokens = article['tokens']
    apr_bigrams=get_ngram_tokens(apr_tokens,2)
    apr_trigrams=get_ngram_tokens(apr_tokens,3)
    
    apr_word_dist.update(apr_tokens)
    apr_bigram_dist.update(apr_bigrams)
    apr_trigram_dist.update(apr_trigrams)
# -

top_20_bigrams = apr_bigram_dist.most_common(20)
top_20_trigrams = apr_trigram_dist.most_common(20)
bigram_list = list(pd.DataFrame(top_20_bigrams, columns = ['Bigram','Freq'])['Bigram'])
trigram_list = list(pd.DataFrame(top_20_trigrams, columns = ['Trigram','Freq'])['Trigram'])

rank = list(range(1, 21))
apr_article_bitrigram = pd.DataFrame(rank, columns = ['Rank'])
apr_article_bitrigram['Bigram']=bigram_list
apr_article_bitrigram['Trigram']=trigram_list
apr_article_bitrigram.set_index('Rank', inplace=True)

apr_article_bitrigram

# #### Overall

# +
all_word_dist=Counter()
all_bigram_dist=Counter()
all_trigram_dist=Counter()

all_tokens = []
for article in corp_all:
    filename = article['Filename']
    text = open('data/text_all/{}'.format(filename)).read()
    tokens = tokenize(text, lowercase=True, strip_chars=string.punctuation)
    all_tokens.extend(tokens)
    
    article['token_cnt'] = len(tokens)
    article['type_cnt'] = len(set(tokens))
    
    bigrams=get_ngram_tokens(tokens,2)
    trigrams=get_ngram_tokens(tokens,3)
    
    all_word_dist.update(tokens)
    all_bigram_dist.update(bigrams)
    all_trigram_dist.update(trigrams)
# -

all_trigram_dist.most_common(20)

# ## Data cleaning

len(all_tokens)

# Because the number of tokens in the news articles is so large, I'll clean the data by splitting it into 10 parts. Like the data cleaning that took place for tweets, I'll be stripping the news article text from words that include stopwords and the query search.

s_all_tokens1 = all_tokens[:145449]

s_all_tokens2 = all_tokens[145449:290898]

s_all_tokens3 = all_tokens[290898:436347]

s_all_tokens4 = all_tokens[436347:581796]

s_all_tokens5 = all_tokens[581796:727245]

s_all_tokens6 = all_tokens[727245:872694]

s_all_tokens7 = all_tokens[872694:1018143]

s_all_tokens8 = all_tokens[1018143:1163592]

s_all_tokens9 = all_tokens[1163592:1309041]

s_all_tokens10 = all_tokens[1309041:]

queries = ['asianamerican', 'asian', 'american', \
            'racism', 'racist', 'xenophobia', 'racism', 'racist', 'xenophobia', \
            'coronavirus', 'corona virus', 'covid19', 'covid 19', 'pandemic', 'virus', "chinese virus", "china virus", \
            'coronavirus', 'covid19', 'pandemic', 'chinavirus', 'chinesevirus']

words_to_remove= stopwords.words('english')+queries

for article_word in s_all_tokens1:
    if article_word in words_to_remove:
        s_all_tokens1.remove(article_word)

for article_word in s_all_tokens2:
    if article_word in words_to_remove:
        s_all_tokens2.remove(article_word)

for article_word in s_all_tokens3:
    if article_word in words_to_remove:
        s_all_tokens3.remove(article_word)

for article_word in s_all_tokens4:
    if article_word in words_to_remove:
        s_all_tokens4.remove(article_word)

for article_word in s_all_tokens5:
    if article_word in words_to_remove:
        s_all_tokens5.remove(article_word)

for article_word in s_all_tokens6:
    if article_word in words_to_remove:
        s_all_tokens6.remove(article_word)

for article_word in s_all_tokens7:
    if article_word in words_to_remove:
        s_all_tokens7.remove(article_word)

for article_word in s_all_tokens8:
    if article_word in words_to_remove:
        s_all_tokens8.remove(article_word)

for article_word in s_all_tokens9:
    if article_word in words_to_remove:
        s_all_tokens9.remove(article_word)

for article_word in s_all_tokens10:
    if article_word in words_to_remove:
        s_all_tokens10.remove(article_word)

s_all_tokens = s_all_tokens1+s_all_tokens2+s_all_tokens3+s_all_tokens4+s_all_tokens5+s_all_tokens6+s_all_tokens7+s_all_tokens8+s_all_tokens9+s_all_tokens10

# ## Cleaned data word, bigram, and trigram distribution

s_all_tokens_dist = Counter(s_all_tokens)

s_all_tokens_dist.most_common(50)

# +
all_t_word_dist=Counter()
all_t_bigram_dist=Counter()
all_t_trigram_dist=Counter()

all_t_tokens = []
for tweet in all_time:
    text = tweet['text'].replace('&amp;', '&').replace('”', '').replace('\'', '').replace('’', '').replace('“', '')
    toks = tokenize(text, lowercase=True, strip_chars=string.punctuation)
    all_t_tokens.extend(toks)

all_t_bigrams=get_ngram_tokens(all_t_tokens,2)
all_t_trigrams=get_ngram_tokens(all_t_tokens,3)

all_t_word_dist.update(all_t_tokens)
all_t_bigram_dist.update(all_t_bigrams)
all_t_trigram_dist.update(all_t_trigrams)

# +
s_all_t_tokens = all_t_tokens
for tweet in list(s_all_t_tokens):
    if tweet in words_to_remove:
        s_all_t_tokens.remove(tweet)

s_all_t_tokens = [x for x in s_all_t_tokens if not x.startswith('https')]
s_all_t_tokens_dist = Counter(s_all_t_tokens)
# -

# ## Creating a data visualization of the most commonly used shared words

# +
queries = ['asianamerican', 'asian', 'american', \
            'racism', 'racist', 'xenophobia', 'racism', 'racist', 'xenophobia', \
            'coronavirus', 'corona virus', 'covid19', 'covid 19', 'pandemic', 'virus', "chinese virus", "china virus", \
            'coronavirus', 'covid19', 'pandemic', 'chinavirus', 'chinesevirus']
stripped_tweets_tokens = all_t_tokens
words_to_remove= stopwords.words('english')+queries
for tweet in list(stripped_tweets_tokens):
    if tweet in words_to_remove:
        stripped_tweets_tokens.remove(tweet)

stripped_tweets_tokens = [x for x in stripped_tweets_tokens if not x.startswith('https')]
stripped_tweets_wfreq = Counter(stripped_tweets_tokens)
# -

all_shared_items = [(item, value, s_all_tokens_dist.get(item)) 
                for item,value in stripped_tweets_wfreq.items() if s_all_tokens_dist.get(item)]

all_tweet_num_tokens = sum(stripped_tweets_wfreq.values())
all_article_num_tokens = sum(s_all_tokens_dist.values())

all_shared_items.sort(key=lambda i: i[1]+i[2], reverse=True)
cdf=pd.DataFrame(all_shared_items, columns=['word','Number of occurrences in Tweets','Number of occurrences in articles'])
cdf['Tweets_percent']=cdf['Number of occurrences in Tweets'] / all_tweet_num_tokens * 100
cdf['Articles_percent']=cdf['Number of occurrences in articles'] / all_article_num_tokens * 100

all_shared_items[:10]


def plot_shared_vocab(cdf, start=5, end=100):
    
    def scaler(values):
        vmin=min(values)
        vmax=max(values)

        scaled_values = [(v-vmin)/(vmax-vmin)for v in values]
        return scaled_values
    
    with plt.style.context('seaborn-paper'):

        fig = plt.figure(figsize=(18,9))

        ax_max = cdf.iloc[start:end][['Tweets_percent','Articles_percent']].max().max()
        ax_min = cdf.iloc[start:end][['Tweets_percent','Articles_percent']].min().min()
        
        cdf['text_size'] = scaler(cdf.Tweets_percent.add(cdf.Articles_percent))

        for row in cdf[start:end].itertuples():
            plt.text(row.Tweets_percent*.8, row.Articles_percent*1.2, row.word, 
                     color='#0000FF', 
                     size=140*row.text_size,
                     ha='center', va='center', alpha=0.25)

        plt.axis([0, ax_max, 0, ax_max])
        plt.plot((0,ax_max),(0,ax_max)) #, color='#A0A0A0')

        plt.xticks([])
        plt.yticks([])
        plt.xlabel('Use in Tweets (% of tokens)', fontsize=20)
        plt.ylabel('Use in Articles (% of tokens)', fontsize=20)

        plt.show()


plot_shared_vocab(cdf[np.logical_and(cdf['Tweets_percent']>0.05, cdf['Articles_percent']>0.05)])

cdf[np.logical_and(cdf['Tweets_percent']>0.1, cdf['Articles_percent']>0.1)]

# - "trump" and "president" both appear on the list
# - "hate" and "discrimination" both appear as well

# ## Sentiment Analysis

sid = SentimentIntensityAnalyzer()

# #### Tweets

# +
tweet_sid_scores=[]

for tweet in all_time:
    scores = sid.polarity_scores(tweet['text'])
    scores['text']=tweet['text']
    scores['date']=tweet['date']
    tweet_sid_scores.append(scores)
# -

by_monthdate = {}
for tweet in tweet_sid_scores:
    ymd = tweet['date'][:10]
    try:
        by_monthdate[ymd].append(tweet['compound'])
    except:
        by_monthdate[ymd] = [tweet['compound']]

data_tweets = [{ 'date': y, 'avg_sent': sum(d)/len(d)}
        for y, d in by_monthdate.items() ]

# +
fig,ax = plt.subplots(figsize=(16,4))
plt.title('Tweet Sentiment Over Time')
p=pd.DataFrame(data_tweets)
p.set_index('date',inplace=True)
ax.xaxis.tick_top()
ax.axhline(y=0, xmin=0.0, xmax=1.0, color='black', linewidth=0.8)

p.sort_index().plot(ax=ax)
# -

for item in ['ching chong','chink']:
    print(item,sid.lexicon.get(item))

# *interesting how there's no score for these derogatory terms*

# #### Articles

# +
article_sid_scores=[]

for article in corp_all:
    filename = article['Filename']
    text = open('data/text_all/{}'.format(filename)).read()
    article['text']=text
    scores = sid.polarity_scores(article['text'])
    scores['text']=article['text']
    scores['date']=article['Date']
    article_sid_scores.append(scores)
# -

a_by_monthdate = {}
for article in article_sid_scores:
    ymd = article['date']
    try:
        a_by_monthdate[ymd].append(article['compound'])
    except:
        a_by_monthdate[ymd] = [article['compound']]

data_articles = [{ 'date': y, 'avg_sent': sum(d)/len(d)}
        for y, d in a_by_monthdate.items() ]

# +
fig,ax = plt.subplots(figsize=(16,4))
plt.title('Article Sentiment Over Time')
p=pd.DataFrame(data_articles)
p.set_index('date',inplace=True)
ax.axhline(y=0, xmin=0.0, xmax=1.0, color='black', linewidth=0.8)

p.sort_index().plot(ax=ax)
# -

t = pd.DataFrame(data_tweets)
a = pd.DataFrame(data_articles)
a.head()

plt.rcParams["figure.figsize"] = (15,8)
ax = t.plot(label="tweets")
a.plot(ax=ax, label = 'articles')
plt.title('Tweet Sentiment vs. Article Sentiment')
plt.gca().legend(('tweet sentiment','article sentiment'))
ax.axhline(y=0, xmin=0.0, xmax=1.0, color='black', linewidth=0.8)

stripped_tweets_wfreq.most_common(50)

# ## Comparing key word frequency over time

s_all_tokens_dist.most_common(50)

# Compare on these keywords: 'chinese', 'community', 'health', 'public', 'trump', 'chicago', 'deaths', 'illinois', 'students', 'hate', 'crisis'

comparison_data = compare_items(jan_word_dist, feb_word_dist,mar_word_dist,apr_word_dist,['chinese', 'community', 'health', 'public', 'trump', 'chicago', 'deaths', 'illinois', 'students', 'hate', 'crisis'])
print('Article Keyword Frequency by Month')
comparison_plot(comparison_data, label1= "January Article", label2= "February Article", label3= "March Article", label4= "April Article")

# ## Collocation

# **Articles**

article_colls = Counter()
article_colls.update(collocates(all_tokens, 'health',win=[5,5]))

plot_collocates('health', article_colls, num=15, threshold=2,
                title='Article collocates of health (win=5)')

# **Tweets**

tweet_colls = Counter()
tweet_colls.update(collocates(stripped_tweets_tokens, 'health',win=[5,5]))

plot_collocates('health', tweet_colls, num=15, threshold=2,
                title='Tweet collocates of health (win=5)')

tweet_colls = Counter()
tweet_colls.update(collocates(all_tokens, 'chicago',win=[5,5]))

plot_collocates('chicago', tweet_colls, num=15, threshold=2,
                title='Tweet collocates of funny (win=5)')

StarcoderdataJupyterScriptsDedupFiltered