"""
## HTF does Topic Modelling Work?

[WIKIPEDIA](https://en.wikipedia.org/wiki/Topic_model) In machine learning and natural language processing, a topic model is a type of statistical model for discovering the abstract "topics" that occur in a collection of documents. Topic modeling is a frequently used text-mining tool for discovery of hidden semantic structures in a text body. Intuitively, given that a document is about a particular topic, one would expect particular words to appear in the document more or less frequently: "dog" and "bone" will appear more often in documents about dogs, "cat" and "meow" will appear in documents about cats, and "the" and "is" will appear approximately equally in both. A document typically concerns multiple topics in different proportions; thus, in a document that is 10% about cats and 90% about dogs, there would probably be about 9 times more dog words than cat words. The "topics" produced by topic modeling techniques are clusters of similar words. A topic model captures this intuition in a mathematical framework, which allows examining a set of documents and discovering, based on the statistics of the words in each, what the topics might be and what each document's balance of topics is.
"""
!pip install mglearn fulltext

import os
import re
import nltk
import IPython
import sklearn
import mglearn
import fulltext
import pyLDAvis
import threading

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from PIL import Image
from pandas import Series
from tabulate import tabulate
from bs4 import BeautifulSoup
from string import punctuation
from nltk.corpus import stopwords
from IPython.display import display, clear_output
from sklearn.decomposition import LatentDirichletAllocation
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

print('complete')
threading.activeCount()

stop_words = set(stopwords.words('english'))

for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))
# TRAINING DATA
# Three articles about 3 related (auto), but different topics; Road Policing, The Auto Industry & Winter Driving
auto_article_path_1 = "../input/toronto-star-sample-articles/toronto-police-try-a-newold-trick.txt"
auto_article_path_2 = "../input/toronto-star-sample-articles/canadian-sales-down-two-per-cent-year-over-year-in-october-while-multiple-automakers-set-records.html.txt"
auto_article_path_3 = "../input/toronto-star-sample-articles/winter-car-prep-done-right.txt"

# TESTING DATA (stuff the model hasn't seen before but can hopefully recognize because of training)
# One related article about Winter Driving
auto_article_path_4 = "../input/toronto-star-sample-articles/winter-of-2020-is-coming-are-you-ready.txt"
# URLS:
# Article 1: https://www.thestar.com/autos/opinion/2020/11/18/toronto-police-try-a-newold-trick.html
# Article 2: https://www.thestar.com/autos/2020/11/16/canadian-sales-down-two-per-cent-year-over-year-in-october-while-multiple-automakers-set-records.html
# Article 3: https://www.thestar.com/autos/advice/2020/11/20/what-ive-learned-after-15-winters-of-testing-tires.html
# Article 4: https://www.thestar.com/autos/advice/2020/11/27/winter-of-2020-is-coming-are-you-ready.html

with open(auto_article_path_1, 'rb') as this_file:
    auto_article_text_1 = this_file.read()
print("---- ARTICLE 1 RAW TEXT ----")
print(auto_article_text_1[0:300])

with open(auto_article_path_2, 'rb') as this_file:
    auto_article_text_2 = this_file.read()
print("\n---- ARTICLE 2 RAW TEXT ----")
print(auto_article_text_2[0:300])

with open(auto_article_path_3, 'rb') as this_file:
    auto_article_text_3 = this_file.read()
print("\n---- ARTICLE 3 RAW TEXT ----")
print(auto_article_text_3[0:300])

with open(auto_article_path_4, 'rb') as this_file:
    auto_article_text_4 = this_file.read()
print("\n---- ARTICLE 4 RAW TEXT ----")
print(auto_article_text_4[0:300])
def clean_up(text):
    
    result = re.sub(r'\\x?..', ' ', str(text))
    result = str(result).replace(',', '')
    result = str(result).replace('|', '')
    result = str(result).replace('"', '')
    result = str(result).replace('*', '')
    result = re.sub(r'\s.\s', ' ', str(result))
    result = re.sub(r'\s.{1}\s', ' ', str(result))
    result = str(result).replace('-', ' ')
    result = str(result).replace('_', '')
    result = str(result).replace('\n', ' ')
    result = str(result).replace('\t', ' ')
    result = str(result).replace('\r', ' ')
    result = str(result).replace('&nbsp;', ' ')
        
    result = str(result.lower())
    
    result = ''.join([c for c in result if c not in punctuation])
    
    result = ' '.join([w for w in result.split() if w not in stop_words])
    
    # Remove Multiple Spaces
    result = re.sub(' +', ' ', str(result))
        
    # Remove duplicate consecutive words
    result = re.sub(r'\b(\w+)\s+\1\b', '', str(result)) 
    
    result = result[1:len(result)]
    
    return result

article_1_clean = clean_up(auto_article_text_1)
article_2_clean = clean_up(auto_article_text_2)
article_3_clean = clean_up(auto_article_text_3)
article_4_clean = clean_up(auto_article_text_4)

print("---- ARTICLE 1 CLEAN TEXT ----")
print(article_1_clean[0:300])
print("\n---- ARTICLE 2 CLEAN TEXT ----")
print(article_2_clean[0:300])
print("\n---- ARTICLE 3 CLEAN TEXT ----")
print(article_3_clean[0:300])
print("\n---- ARTICLE 4 CLEAN TEXT ----")
print(article_4_clean[0:300])
"""
### The Keyword Overlap or 'Bag of Words' Method

This method requires someone to label all articles.
"""
# Unique Words Present In Both Articles
overlap_4_1 = len(set(article_4_clean.split()) & set(article_1_clean.split()))
overlap_4_2 = len(set(article_4_clean.split()) & set(article_2_clean.split()))
overlap_4_3 = len(set(article_4_clean.split()) & set(article_3_clean.split()))

print("Article 4 (about winter driving) Contains " + str(overlap_4_1) + " Words That Are Also In Article 1 (about road policing)")
print("Article 4 (about winter driving) Contains " + str(overlap_4_2) + " Words That Are Also In Article 2 (about the auto industry)")
print("Article 4 (about winter driving) Contains " + str(overlap_4_3) + " Words That Are Also In Article 3 (about winter driving)")

all_duplicates = overlap_4_1 + overlap_4_2 + overlap_4_3

print("Weight for Artile 1: ", round(overlap_4_1 / all_duplicates, 2), "%")
print("Weight for Artile 2: ", round(overlap_4_2 / all_duplicates, 2), "%")
print("Weight for Artile 3: ", round(overlap_4_3 / all_duplicates, 2), "%")
"""
^ Boom, that's what you'd expect. Article 3 is about Winter Driving and so it our 4th test article.
"""
"""
### The Topic Modelling Way

1) Mash all articles together into one 'corpus'.

2) Put corpus through the 'LDA' model to determine how many topics are present.

3) Label each topic logically.

4) Model can then be used to output topic weights on new articles it hasn't seen.

"""
data_corpus = [ article_1_clean, article_2_clean, article_3_clean ]

vect = CountVectorizer(input='content', binary=False, min_df=.0005, max_df=0.35, ngram_range=(1,4))

dtm = vect.fit_transform(data_corpus)

matrix = pd.DataFrame(dtm.toarray(), columns=vect.get_feature_names())

print(matrix.shape)
feature_names = vect.get_feature_names()

def runLDA(dtm, vect, n_components, alpha=None, beta=None):

    print('=== LDA MODEL : ' + str(n_components) + ' TOPICS ===')
    
    lda_model = LatentDirichletAllocation(n_components=n_components,
                                          doc_topic_prior=alpha,
                                          topic_word_prior=beta,
                                          max_iter=10, 
                                          learning_method='batch', 
                                          random_state=123,
                                          n_jobs=-1,
                                          verbose=1)
    
    lda_output = lda_model.fit(dtm)

    ll = lda_model.score(dtm)                            # Log Likelyhood: Higher the better
    perp = lda_model.perplexity(dtm)                     # Perplexity: Lower the better.
    sorting = np.argsort(lda_model.components_)[:, ::-1] # sorted terms
    theta = pd.DataFrame(lda_model.transform(dtm))       # document-topic matrix
    beta = pd.DataFrame(lda_model.components_)           # components_ = topic-term matrix
    
    # Build Custom Topic Summary
    no_top_words = 1000
    weight = theta.sum(axis=0)
    support50 = (theta > 0.5).sum(axis=0)
    support10 = (theta > 0.1).sum(axis=0)
    termss = list()
    for topic_id, topic in enumerate(lda_model.components_):
        terms = " ".join([feature_names[i] for i in topic.argsort()[:-no_top_words - 1:-1]])
        termss.append(terms)
    topic_summary = pd.DataFrame({'TopicID': range(0, len(termss)), 
                                  'Support50': support50, 
                                  'Support10': support10, 
                                  'Weight': weight, 
                                  'Terms': termss})
    return {'model': lda_model, 
            'theta': theta, 
            'beta': beta, 
            'topic_summary': topic_summary, 
            'll': ll, 
            'perp': perp,
            'sorting': sorting,
            'n_components': n_components}


lda_2 = runLDA(dtm, vect, 2)
lda_3 = runLDA(dtm, vect, 3)
lda_4 = runLDA(dtm, vect, 4)
lda_5 = runLDA(dtm, vect, 5)
lda_10 = runLDA(dtm, vect, 10)
print('done')
# Perplexity (lower the better?)
pp = [lda_2['perp'], lda_3['perp'], lda_4['perp'], lda_5['perp'], lda_10['perp']]
plt.title('PERPLEXITY')
plt.ylabel('Perplexity')
plt.xlabel('Number Of Topics')
plt.plot([2, 3, 4, 5, 10], pp)
plt.show()

# Log Likelyhood (higher the better?)
ll = [lda_2['ll'], lda_3['ll'], lda_4['ll'], lda_5['ll'], lda_10['ll']]
plt.title('LOG LIKELIHOOD')
plt.ylabel('Log Likelihood')
plt.xlabel('Number Of Topics')
plt.plot([2, 3, 4, 5, 10], ll)
plt.show()

# when in doubt, go for the elbow
lda_choice = lda_3
n_topics = lda_choice['n_components']
# Word cloud for each topic
for t in range(n_topics):

    # Extract terms from this LDA topic
    topic_data = lda_3['topic_summary'].loc[t, 'Terms']
    print(topic_data)
    # Create and generate a word cloud
    wordcloud = WordCloud(max_font_size = 75,
                          max_words = 250,
                          background_color = 'white',
                          width = 600,
                          height = 400).generate(str(topic_data))

    # Display the generated image:
    plt.figure(figsize=(10, 15))
    plt.title('TOPIC ' + str(t) + ' WORDCLOUD')
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis('off')
    plt.show()
"""
^ so above it looks like:
* the first topic is winter driving
* the second topic is about policing
* the thrid topic is about the auto industry
"""
unseen_document = article_4_clean

print(lda_output.transform(vect.transform([unseen_document])))
"""
^ If we put test article 4 through this model it gives topic 1 (index 0), which we have labelled as 'winter driving', a weight of 0.78.
"""