import re

import bs4
import numpy as np
import pandas as pd

fname_posts = '../data/processed/posts_processed.json'

df_posts = pd.read_json(fname_posts, lines=True)

df_posts.index = df_posts.id
df_posts.head(3)

"""
Explanation: SE Economics - Data Exploration & Preprocessing Prototyping
Author: Matt Fortier &#109;&#97;&#116;&#116;&#46;&#102;&#111;&#114;&#116;&#105;&#101;&#114;&#64;&#111;&#112;&#101;&#110;&#109;&#97;&#105;&#108;&#98;&#111;&#120;&#46;&#111;&#114;&#103;

Creation Date: 2017-02-21 | Last Updated: 2107-08-01
Python 3.6

The goal here is to get a good feel for what kind of data is available to us, and in what general form is the text at our disposition. We should experiment with different preprocessing steps and find one that gives you satisfactory results. Objective is always to extract as many of the useful/valuable information, while ignoring noise.
End of explanation
"""


df_body = df_posts.body
df_body.iloc[0]

"""
Explanation: Body - NLP Analysis
End of explanation
"""


def cleanup_text(text):
    soup = bs4.BeautifulSoup(text, "html5lib")
    return [p.get_text().translate({ord('\\'): None}) for p in soup.find_all('p')]

cleanup_text(df_body.iloc[0])

"""
Explanation: Okay, so at a minimum we need to filter out the HTML tags and remove the backslashes. Let's prepare to do that by writing a simple function that levegerages the excellent BeutifulSoup4 library.
End of explanation
"""


import spacy

nlp = spacy.load('en_core_web_md')

test_output = nlp(' '.join(cleanup_text(df_body.iloc[0])))
test_output

"""
Explanation: We got workable text here - Let's fire up Spacy and shoot this across an entire NLP .
Spacy is a NLP library that regroups a whole set of models and algorithms to perform the most popular NLP preprocessing tasks. Its very robust, and most of the results are very close to current state of the art.
End of explanation
"""


for num, sent in enumerate(test_output.sents):
    print('{}> {}\n'.format(num, sent))

"""
Explanation: Let's print out the result of sentance segmentation...
End of explanation
"""


for num, ent in enumerate(test_output.ents):
    print('{}) {} - {}\n'.format(num+1, ent, ent.label_))

"""
Explanation: What about Named Entities?
End of explanation
"""


for token, lemma, pos in ((t.orth_, t.lemma_, t.pos_) for t in test_output):
    print('{}|{} - {}\n'.format(token, lemma, pos))

"""
Explanation: Part-of-Speech tagging identifies the syntactic function of each token/word.
Stemming and lemmatization are "normalizing" functions: They try to identify a common root for words of a same family.
End of explanation
"""


from gensim.models import Phrases
from gensim.models.word2vec import LineSentence

def is_invalid_token(token):
    return token.is_punct or token.is_space or token.lemma_ == '-PRON-'

def cleanup_html(corpus):
    for doc in corpus:
        soup = bs4.BeautifulSoup(doc, "html5lib")
        yield ' '.join([p.get_text().translate({ord('\\'): None}) for p in soup.find_all('p')])

unigram_fname = '../data/processed/unigram_processed_posts.txt'

%%time

with open(unigram_fname, 'w') as fout:
    for parsed_corpus in nlp.pipe(cleanup_html((i[1] for i in df_body.iteritems())), n_threads=8):
        for sent in parsed_corpus.sents:
            fout.write('{}\n'.format(' '.join([t.lemma_ for t in sent if not is_invalid_token(t)])))

unigram_sentences = LineSentence(unigram_fname)

bigram_model_fname = '../data/processed/bigram.model'

%%time
bigram_model = Phrases(unigram_sentences)
bigram_model.save(bigram_model_fname)

bigram_fname = '../data/processed/bigram_processed_posts.txt'

with open(bigram_fname, 'w') as fout:
    for sent in unigram_sentences:
        bigram_sent = ' '.join(bigram_model[sent])
        fout.write(bigram_sent + '\n')

bigram_sentences = LineSentence(bigram_fname)

trigram_model_fname = '../data/processed/trigram.model'

%%time
trigram_model = Phrases(bigram_sentences)
trigram_model.save(trigram_model_fname)

trigram_fname = '../data/processed/trigram_processed_posts.txt'

with open(trigram_fname, 'w') as fout:
    for sent in bigram_sentences:
        trigram_sent = ' '.join(trigram_model[sent])
        fout.write(trigram_sent + '\n')

final_fname = '../data/processed/final_processed_trigram_posts.txt'

with open(final_fname, 'w') as fout:
    for parsed_post in nlp.pipe(cleanup_html((i[1] for i in df_body.iteritems())), n_threads=10):
        unigram_post = [t.lemma_ for t in parsed_post if not is_invalid_token(t)]
        bigram_post = bigram_model[unigram_post]
        trigram_post = trigram_model[bigram_post]
        
        trigram_post = [term for term in trigram_post if term not in spacy.en.STOP_WORDS]
        
        fout.write('{}\n'.format(' '.join(trigram_post)))

"""
Explanation: Phrase Modelling - Bigrams and Trigrams
End of explanation
"""


