"""
Do hacker news users favor particular topics? Can you predict weather a story will be popular from the title alone? 

In this notebook we will see that it there are definately some keywords in post titles that correlate with higher post scores. It is also possible to create a model that predicts whether a post will be popular given the title alone, albeit with only marginal accuracy improvements over random guessing.
"""
#We will work with the 'stories' table in the hacker news dataset
#preview the first few lines of the table
from google.cloud import bigquery

# Create a "Client" object
client = bigquery.Client()
# Construct a reference to the "openaq" dataset
dataset_ref = client.dataset("hacker_news", project="bigquery-public-data")
# API request - fetch the dataset
dataset = client.get_dataset(dataset_ref)
# Construct a reference to the "stories" table
table_ref = dataset_ref.table("stories")
# API request - fetch the table
table = client.get_table(table_ref)
# # Preview the first five lines of the table
client.list_rows(table, max_results=5).to_dataframe()
"""
First we will try and find a set of words that tend to be associated with high scoring posts. To do that we will first get a big set of 'candidate' popular words (from the most highly scored posts), then we get the median score for posts with these words (over the entire dataset). 
"""
#query the dataset for all titles that ended up with very high scores (over 250)
query = """
        SELECT score, title    
        FROM `bigquery-public-data.hacker_news.stories`
        WHERE score>250
        """

# Create a QueryJobConfig object to estimate size of query without running it
dry_run_config = bigquery.QueryJobConfig(dry_run=True)

# API request - dry run query to estimate costs
dry_run_query_job = client.query(query, job_config=dry_run_config)

print("This query will process {:,} bytes.".format(dry_run_query_job.total_bytes_processed))
#get the high scoring dataset and convert titles to a spacy object
import spacy
high_scores = client.query(query).to_dataframe()
nlp = spacy.load('en_core_web_lg')

docs = [nlp(doc) for doc in high_scores['title']]
from collections import defaultdict
import pandas as pd

word_lib = defaultdict(list)#this dictionary will hold a list of occurances for each word
num_words = {}

for doc in docs:
    #get the lemmastized words in the title and ignore top words
    doc_words = [token.lemma_ for token in doc if not token.is_stop and not token.is_punct]
    for word in doc_words:
        #Add an occurance to the list for each word in the title
        word_lib[word].append(1)

for key, value in word_lib.items():
    num_words[key] = sum(value)#add up the occurances for each word

#these are the 100 most popular words in the highest scoring titles. These will be our candidate words.
candidate_words = list(pd.Series(num_words).sort_values(ascending=False)[:100].index)
candidate_words
"""
Next we will query the full stories dataset and get the median score for each candidate word based on the score of titles that contain it.
"""
from spacy.matcher import PhraseMatcher

query = """
        SELECT score, title    
        FROM `bigquery-public-data.hacker_news.stories`
        """
#get all of the titles and scores in the dataset
stories = client.query(query).to_dataframe()

#drop entries with missing values, then select a random fraction of the data (due to speed and memory limitations)
data = stories.dropna().sample(frac=0.05)

#convert titles to spacy
docs = [nlp(doc) for doc in data['title']]

# Create a PhraseMatcher object. The tokenizer is the first argument. Use attr = 'LOWER' to make consistent capitalization
matcher = PhraseMatcher(nlp.vocab, attr='LOWER')

# Create a list of tokens for each candidate word
tokens_list = [nlp(item) for item in candidate_words]

# Add the candidate words to the matcher. 
matcher.add("pop_words",            # Just a name for the set of rules we're matching to
           tokens_list)

word_scores = defaultdict(list)#this will hold a list of scores for each word
avg_word_scores = {}

for idx, story in data.iterrows():
    doc = nlp(story['title'])

    matches = matcher(doc)
    
    # Create a set of the items found in the title
    found_words = set([doc[match[1]:match[2]].lower_ for match in matches])
    
    # Update item_ratings with rating for each item in found_items
    # Transform the item strings to lowercase to make it case insensitive
    for word in found_words:
        word_scores[word].append(story['score'])

from statistics import median
def standard_dev(lst):#function to compute the standard deviation
    mean = sum(lst)/len(lst)
    return (sum([(ele-mean)**2 for ele in lst])/len(lst))**0.5

for key, value in word_scores.items():
    avg_word_scores[key] = [median(value), standard_dev(value), len(value)]

popular_words = pd.DataFrame.from_dict(avg_word_scores, orient='index', columns=['median_score', 'standard_dev', 'count'])
popular_words.sort_values(by=['median_score', 'count'], ascending=False, inplace=True)
#Take a look at the 20 most popular words by mdeian score and number of occurances
popular_words[:20]
"""
It does look like certain topics tend to be more popular than others. Can we make a model that predicts whether a post will be popular fromthe title alone? We will create a classfication model that tries to predict whether a post gets a score above 5.
"""
from sklearn.svm import LinearSVC
from sklearn.metrics import mean_squared_error, accuracy_score
from sklearn.model_selection import train_test_split
from lightgbm import LGBMRegressor
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.pipeline import make_pipeline
import numpy as np

#vectorize each title (ie. convert the title to a vector in 300 dimensional space)
with nlp.disable_pipes():
    doc_vectors = np.array([nlp_doc.vector for nlp_doc in docs])
    
#our target variable will be whether the post got more than 5 score. 
data['popular'] = data['score'].map(lambda x: 1 if x>5 else 0)

X_train, X_test, y_train, y_test = train_test_split(doc_vectors, data['popular'],
                                                    test_size=0.2, random_state=20)

#we use a support vector classifier as it works well in high dimensional space
svc_model = LinearSVC(random_state=0, tol=1e-5, max_iter=1e3, dual=False, C=100)

#scale the vectors going into the model with a pipeline
svc = make_pipeline(MinMaxScaler(), 
                     svc_model
                    )

svc.fit(X_train, y_train)
"""
The suppor vector machine classifier can produce a proabbaily prediction for each class as the diatnce from the support vector to the datapoint. We will use that to produce plots of precision, recall, and ROC
"""
from sklearn.metrics import precision_recall_curve, roc_curve, auc

probas_pred = svc.decision_function(X_test)
precision, recall, thresholds = precision_recall_curve(y_test, probas_pred)
fpr, tpr, thresholds_roc = roc_curve(y_test, probas_pred)

pos_ratio = y_test.sum()/len(y_test)
AUC = auc(fpr, tpr)

import matplotlib.pyplot as plt
fig, axs = plt.subplots(1,2, figsize=(16,6))
#Plot precision and recall vs threshold
axs[0].set_title('Model Performance vs Classification Threshold')
axs[0].plot([thresholds[0],thresholds[-1]], [pos_ratio, pos_ratio], linestyle='--', color='k')
axs[0].plot(thresholds, precision[1:])
axs[0].plot(thresholds, recall[1:])
axs[0].set_xlabel('Threshold')
axs[0].set_ylabel('Score')
# axs[0].set_ylim([0, 0.4])
axs[0].legend(['Precision - Random Classifier', 'Trained Model Precision', 'Trained Model Recall'], loc='best')
axs[0].grid()
#Plot ROC
props = dict(boxstyle='square', facecolor='white')
axs[1].set_title('ROC')
axs[1].text(1.25, 0.95, 'AUC={:.3}'.format(AUC), transform=axs[0].transAxes, fontsize=14,
        verticalalignment='top', bbox=props)
axs[1].plot(fpr, tpr)
axs[1].plot([0,1], [0,1], linestyle='--', color='k')
axs[1].set_xlabel('False Positive Rate')
axs[1].set_ylabel('True Positive Rate')
axs[1].legend(['Trained Model', 'Random Classifier'], loc='lower right')
axs[1].grid()
plt.show()
"""
* The model AUC is about 0.6, so it is slightly better than random guessing (0.5) 
* The lower we set the threshold the more the model guesses that every title will be popular, so it finds all the popular titles, but also gets many false positives
* The higher we set the threshold, the more the model guesses that every title will not be popular. It may reach a higher precision this way (more true positives), but it misses most of the popular titles
* A good compormise if found around -0.5
* Comparing to the dashed lines, this model is a bit better than random guessing
"""
#plot confusion matrix for chosen threshold. 1=popular, 0=not popular
from sklearn.metrics import confusion_matrix
import seaborn as sns

thresh = -0.48
y_pred = [1 if ele>thresh else 0 for ele in probas_pred]

cm = confusion_matrix(y_test, y_pred)
ax = sns.heatmap(cm, annot=True, fmt="d")
ax.set(ylabel='True Label', xlabel='Predicted Label')
plt.show()
#function to predict whether a set of titles will be popular
def pred(model, thresh, titles, nlp):
    docs = [nlp(doc) for doc in titles]
    with nlp.disable_pipes():
        doc_vectors = np.array([nlp_doc.vector for nlp_doc in docs])
    probas_pred = model.decision_function(doc_vectors)
    return [1 if ele>thresh else 0 for ele in probas_pred]

titles = ['steve jobs hacker python', 
         'hn', 
         'the meaning of life']

pred(svc, thresh, titles, nlp)
#look at which titles the model predicted as popular and got right

[doc for doc, pred in zip(docs,y_pred) if pred==1]