# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Project: Predicting financial data 
# ## Part 2 - Mining media content
# <b>Author:</b> <NAME><br>
# <b>Date:</b> 17.03.2019<br>
# <b>Source:</b> http://joerivanwijngaarden.com/project-financial-predictions-p1.html<br>
# <b>Github:</b> https://github.com/tingidev/jvw-projects/tree/master/project-2-predicting-financial-data<p>

# ## News items variables
#
# <ol>
#     <li><strong>keywords</strong> - NLTK extracted keywords of text body</li>
#     <li><strong>char_n</strong> - Number of characters in text body</li>
#     <li><strong>title</strong> - Article's title</li>
#     <li><strong>url</strong> - Article's url</li>
#     <li><strong>text</strong> - Full article's text</li>
#     <li><strong>brand</strong> - Newspaper brand where article was published</li>
#     <li><strong>date</strong> - Date</li>
#     <li><strong>date_str</strong> - Date as string</li>
#     <li><strong>summary</strong> - NLTK extracted summary of text body</li>
# </ol>
#

# +
# Import friends
import os, sys
import numpy as np
import pandas as pd
import pickle
import matplotlib.pyplot as plt
import bson
from matplotlib.ticker import FormatStrFormatter
from matplotlib.font_manager import FontProperties
from matplotlib.cm import ScalarMappable
# %matplotlib inline

# Plotting
import ast
from collections import Counter
from wordcloud import WordCloud

# Text processing
import re
import nltk
from nltk.corpus import stopwords
from nltk.corpus import wordnet
from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.stem import PorterStemmer
from nltk.stem import WordNetLemmatizer
import string


# -

# ## Functions

# +
# Wordcloud
def generate_wordcloud(tup):
    wordcloud = WordCloud(background_color='white',
                         max_words=30, max_font_size=40,
                         random_state=42
                         ).generate(str(tup))
    return wordcloud

# News categories
def get_categories(text, brand):
    splt = text.split('/')
    if brand == 'nytimes':
        # nytimes has date before category section
        if len(splt) > 6:
            cat = splt[6]
        else:
            cat = ''
    else:
        cat = splt[3]
    return cat

# Tokenize
stop_words = set(stopwords.words('english'))

def get_wordnet_pos(treebank_tag):
    if treebank_tag.startswith('J'):
        return wordnet.ADJ
    elif treebank_tag.startswith('S'):
        return wordnet.ADJ_SAT
    elif treebank_tag.startswith('V'):
        return wordnet.VERB
    elif treebank_tag.startswith('N'):
        return wordnet.NOUN
    elif treebank_tag.startswith('R'):
        return wordnet.ADV     
    else:
        return ''

def tokenize(text):
    '''
    Tokenize article text into a list of useful words.
    Excludes any stopwords, symbols and punctuation.
    Lemmatize each word to its stem.
    
    '''
    ps = PorterStemmer()
    lm = WordNetLemmatizer()
    try:
        regex = re.compile('[' + re.escape(string.punctuation) + '0-9\\r\\t\\n]')
        text = regex.sub(" ",text)
        tokens_ = word_tokenize(text)
        pos_tag = nltk.pos_tag(tokens_)
        tokens = []
        for w in pos_tag:
            if w[0].lower() not in stop_words and re.search('[a-zA-Z]', w[0]) and len(w[0]) >= 3:
                pos_val = get_wordnet_pos(w[1])
                if pos_val:
                    lemm = lm.lemmatize(w[0].lower(),pos_val)
                    tokens.append(lemm)
        return tokens    
    except TypeError as e: 
        print(text, e)


# -

# ## Load data

# Loading news BSON files
base_folder = 'C:/Users/jvw/Dropbox/data_dumps/tingi-sandbox/news_items.bson'
dump_folder = ['dump_19_01_09', 'dump_19_02_03']
newsname = base_folder + dump_folder[0] + '/news_items.bson'
with open(newsname,'rb') as f:
    b = bson.decode_all(f.read())
news = pd.DataFrame(b)
for i in range(len(dump_folder)-1):
    newsname = base_folder + dump_folder[i+1] + '/news_items.bson'
    with open(newsname,'rb') as f:
        b = bson.decode_all(f.read())
    tmp = pd.DataFrame(b)
    news = news.append(tmp)

print('Size:', news.shape, '\n')
print(news.dtypes)
news.head(3)

# Clean data by removing cbsnews (only 3 entries from testing)
tmp = news.loc[news['brand']=='cbsnews']
news = news.drop(tmp.index)

# Extract categories from URL
news['category'] = news[['url', 'brand']].apply(lambda x: get_categories(x['url'], x['brand']), axis=1)

# +
# News outlets included in dataset
brands = np.sort(news['brand'].unique())
places = ['Japan', 'USA', 'Hong Kong', 'UK']
i = 0
tot_n = 0

for brand in brands:
    grouped = news.loc[news['brand']==brand].groupby('date_str')
    mean_n = int(np.mean(grouped['date'].size().values))
    tot_n += mean_n
    print('Newspaper:', brand.upper(), '(' + places[i] + ') -', mean_n, 'articles')
    i += 1
print('Total:', tot_n, 'articles per day')
# -

# Days of gathered media
days = news['date_str'].unique()
print('Number of days:', len(days))
print('First day:', days[0])
print('Last day:', days[-1], '\n')
print(days)

# ## General overview of newspapers (unused & Fig. 6 & 7)

# +
col = plt.get_cmap('Set1')
fig, ax = plt.subplots(figsize=(10, 6))
leg = ['Japan Times (JAP)', 'NY Times (USA)', 'South China MP (HNK)', 'The Guardian (UK)']
ticks = [0, 21, 44]
i = 0

for brand in brands:
    grouped = news.loc[news['brand']==brand].groupby('date_str')
    ax.plot(grouped['date'].size().values, linewidth=3, color=col.colors[i])
    ax.set_xlim(-2, len(days) + 1)
    ax.set_ylim(0, 950)
    ax.set_yticks(np.linspace(0, 900, 10))
    ax.set_xticks(ticks)
    ax.set_xticklabels(['Dec \'18', 'Jan \'19', 'Feb \'19'])
    ax.set_title('Media collected over time', fontsize=18)
    ax.set_ylabel('Number of articles', fontsize=16)
    ax.tick_params(axis='x', which='major', labelsize=14)
    ax.tick_params(axis='y', which='major', labelsize=12)
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)   
    i += 1
    
fig.legend(leg, bbox_to_anchor=(.36, .72), fontsize=12)
ax.grid(True)

fig.tight_layout()
# plt.savefig('./images/unused.png')

# +
# Distribution of categories across newspapers
cmap = plt.get_cmap('plasma')
fig, axs = plt.subplots(ncols=2, nrows=2, figsize=(20, 10))
fig.subplots_adjust(hspace=0.7)
ttl = ['Japan Times', 'NY Times', 'South China MP', 'The Guardian']
size = 8
i = 0
for brand in brands:
    grouped = news.loc[news['brand']==brand]
    categories = grouped['category'].values.tolist()
    print("There are %d unique categories in " % len(set(categories)) + brand )
    ticks = [x[0] for x in Counter(categories).most_common(size)]
    vals = [x[1] / len(categories) * 100 for x in Counter(categories).most_common(size)]
    col = [cmap(x / 25) for x in vals]
    im = fig.axes[i].bar(range(size), vals, color=col)
    fig.axes[i].set_xticks(range(size))
    fig.axes[i].set_xticklabels(ticks, rotation=45, ha="right")
    fig.axes[i].set_ylim([0, 25])
    fig.axes[i].set_title(ttl[i], fontsize=20)
    fig.axes[i].set_ylabel('Occurence (%)', fontsize=16)
    fig.axes[i].spines['right'].set_visible(False)
    fig.axes[i].spines['top'].set_visible(False)  
    fig.axes[i].tick_params(axis='y', which='major', labelsize=12)
    fig.axes[i].tick_params(axis='x', which='major', labelsize=16)
    i += 1
    
# Colorbar
sm = ScalarMappable(cmap=cmap)
sm.set_array([0, 25])
fig.subplots_adjust(right=0.88)
cbar_ax = fig.add_axes([0.9, 0.15, 0.025, 0.7])
cb = fig.colorbar(sm, cax=cbar_ax, ticks=np.linspace(5, 25, 5))
cb.ax.tick_params(labelsize=12)
# plt.savefig('./images/fig6.png', bbox_inches='tight')
# -

# Unique keywords
list_of_keywords = news['keywords'].values.flatten()
all_keywords = [item for sublist in list_of_keywords for item in sublist]
print("There are %d unique keywords across all articles." % len(set(all_keywords)))
print("There are %d total keywords." % len(all_keywords))
most_common=Counter(all_keywords).most_common(50)
most_common[:10]

# Generate wordcloud for each newspaper
fig, axs = plt.subplots(ncols=2, nrows=2, figsize=(20, 10))
i = 0
for brand in brands:
    list_of_keywords = news.loc[news['brand']==brand]['keywords'].values.flatten()
    all_keywords = [item for sublist in list_of_keywords for item in sublist]
    img = generate_wordcloud(all_keywords)
    fig.axes[i].imshow(img)
    fig.axes[i].axis('off')
    fig.axes[i].set_title(leg[i], fontsize=24)
    i += 1
fig.tight_layout()
# plt.savefig('./images/fig7.png')

# ## Tokenize & example

# Tokenize summary (takes 10 mintutes or so on a laptop)
news['tokens'] = news['summary'].map(tokenize)

# Print examples
for text, summary, tokens, keywords in zip(news['text'].head(8), news['summary'].head(8), 
                                           news['tokens'].head(8), news['keywords'].head(8)):
    print(text)
    print(summary)
    print(tokens)
    print(keywords)
    print(' ')

# ## Term frequency-inversed document frequency (TF-IDF) & example (Fig. 8)

# +
# Apply tf-idf function to summary data
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer

n_features = 100000

tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=2,
                                   tokenizer=tokenize,
                                   max_features=n_features,
                                   ngram_range=(1, 1))

tf_vectorizer = CountVectorizer(max_df=0.95, min_df=2,
                                tokenizer=tokenize,
                                max_features=n_features,
                                ngram_range=(1, 1))

# Run it for theguardian as an example
brand = brands[1]
grouped = news.loc[news['brand']==brand]['summary'].values

tf = tf_vectorizer.fit_transform(list(grouped))
tfidf = tfidf_vectorizer.fit_transform(list(grouped))
# -

# TF counts for corpus
count = dict(zip(tf_vectorizer.get_feature_names(), tf.toarray().sum(axis=0)))
tfmat = pd.DataFrame().from_dict(count, orient='index')
tfmat.columns = ['Total count']
tfmat.sort_values(by='Total count', ascending=False).head(10).transpose()

# TF counts for example article
example = grouped[10]
print(example)
count_example = Counter(tokenize(example))
xxx = pd.DataFrame().from_dict(count_example, orient='index')
xxx.columns = ['Example count']
xxx.sort_values(by='Example count', ascending=False).head(10).transpose()

# TF-IDF for example article
score = dict(zip(tfidf_vectorizer.get_feature_names(), np.around(tfidf.toarray()[10], 2)))
tfidfmat = pd.DataFrame().from_dict(score, orient='index')
tfidfmat.columns = ['Example tf-idf']
tfidfmat.sort_values(by='Example tf-idf', ascending=False).head(10).transpose()

StarcoderdataJupyterScriptsDedupFiltered