# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3.5
#     language: python
#     name: python3
# ---

#
# # @hidden_cell
# # The following code contains the credentials for a file in your IBM Cloud Object Storage.
# # You might want to remove those credentials before you share your notebook.
# credentials_2 = {
#     'IBM_API_KEY_ID': '<KEY>',
#     'IAM_SERVICE_ID': 'iam-ServiceId-69636595-c7b4-451a-b26e-b9a67a5b1181',
#     'ENDPOINT': 'https://s3-api.us-geo.objectstorage.service.networklayer.com',
#     'IBM_AUTH_ENDPOINT': 'https://iam.bluemix.net/oidc/token',
#     'BUCKET': 'textsummarizer-donotdelete-pr-jw3aso7wwj713f',
#     'FILE': 'Mission_Impossible_6_Review.txt'
# }
# # Text Summarization & Visualization

# ## Setup
# To prepare your environment, you need to install some packages.
#
# ### Install the necessary packages
#
# You need the latest versions of these packages:<br>

# !pip install gensim

# !pip install watson-developer-cloud==1.5

# !pip install pyldavis

# !pip install wordcloud

from gensim.summarization.summarizer import summarize
from gensim.summarization import keywords
import watson_developer_cloud
import ibm_boto3
from botocore.client import Config
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
import string
import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess
from gensim.models import CoherenceModel
import pyLDAvis
import pyLDAvis.gensim  
import matplotlib.pyplot as plt
# %matplotlib inline
import warnings
warnings.filterwarnings("ignore")
import urllib
from bs4 import BeautifulSoup
import requests
import nltk
nltk.download('all')

# ## 1. Summarization & keywords extraction

# ### 1a. Read the Data
# Click on Insert to code and then select Insert Credentials as credentials_1

# +

# @hidden_cell
# The following code contains the credentials for a file in your IBM Cloud Object Storage.
# You might want to remove those credentials before you share your notebook.
credentials_1 = {
    'IBM_API_KEY_ID': '<KEY>',
    'IAM_SERVICE_ID': 'iam-ServiceId-69636595-c7b4-451a-b26e-b9a67a5b1181',
    'ENDPOINT': 'https://s3-api.us-geo.objectstorage.service.networklayer.com',
    'IBM_AUTH_ENDPOINT': 'https://iam.bluemix.net/oidc/token',
    'BUCKET': 'textsummarizer-donotdelete-pr-jw3aso7wwj713f',
    'FILE': 'Mission_Impossible_6_Review.txt'
}

credentials_2 = {
    'IBM_API_KEY_ID': '<KEY>',
    'IAM_SERVICE_ID': 'iam-ServiceId-69636595-c7b4-451a-b26e-b9a67a5b1181',
    'ENDPOINT': 'https://s3-api.us-geo.objectstorage.service.networklayer.com',
    'IBM_AUTH_ENDPOINT': 'https://iam.bluemix.net/oidc/token',
    'BUCKET': 'textsummarizer-donotdelete-pr-jw3aso7wwj713f',
    'FILE': 'Richwine.txt'
}

credentials_3_2 = {
    'IBM_API_KEY_ID': '<KEY>',
    'IAM_SERVICE_ID': 'iam-ServiceId-69636595-c7b4-451a-b26e-b9a67a5b1181',
    'ENDPOINT': 'https://s3-api.us-geo.objectstorage.service.networklayer.com',
    'IBM_AUTH_ENDPOINT': 'https://iam.bluemix.net/oidc/token',
    'BUCKET': 'textsummarizer-donotdelete-pr-jw3aso7wwj713f',
    'FILE': 'Chavez.txt'
}

credentials_3 = {
    'IBM_API_KEY_ID': '<KEY>',
    'IAM_SERVICE_ID': 'iam-ServiceId-69636595-c7b4-451a-b26e-b9a67a5b1181',
    'ENDPOINT': 'https://s3-api.us-geo.objectstorage.service.networklayer.com',
    'IBM_AUTH_ENDPOINT': 'https://iam.bluemix.net/oidc/token',
    'BUCKET': 'textsummarizer-donotdelete-pr-jw3aso7wwj713f',
    'FILE': 'Chavez.txt'}
# @hidden_cell
# The following code contains the credentials for a file in your IBM Cloud Object Storage.
# You might want to remove those credentials before you share your notebook.
credentials_4 = {
    'IBM_API_KEY_ID': 'qMQ0q2CXDiOWd0LXP8IwNUwrLUfX0hjaOpOK6r7ioGn8',
    'IAM_SERVICE_ID': 'iam-ServiceId-69636595-c7b4-451a-b26e-b9a67a5b1181',
    'ENDPOINT': 'https://s3-api.us-geo.objectstorage.service.networklayer.com',
    'IBM_AUTH_ENDPOINT': 'https://iam.bluemix.net/oidc/token',
    'BUCKET': 'textsummarizer-donotdelete-pr-jw3aso7wwj713f',
    'FILE': 'RichCha.txt'
}
# @hidden_cell
# The following code contains the credentials for a file in your IBM Cloud Object Storage.
# You might want to remove those credentials before you share your notebook.
credentials_5 = {
    'IBM_API_KEY_ID': '<KEY>',
    'IAM_SERVICE_ID': 'iam-ServiceId-69636595-c7b4-451a-b26e-b9a67a5b1181',
    'ENDPOINT': 'https://s3-api.us-geo.objectstorage.service.networklayer.com',
    'IBM_AUTH_ENDPOINT': 'https://iam.bluemix.net/oidc/token',
    'BUCKET': 'textsummarizer-donotdelete-pr-jw3aso7wwj713f',
    'FILE': 'Frank.txt'
}
# @hidden_cell
# The following code contains the credentials for a file in your IBM Cloud Object Storage.
# You might want to remove those credentials before you share your notebook.
credentials_6 = {
    'IBM_API_KEY_ID': '<KEY>',
    'IAM_SERVICE_ID': 'iam-ServiceId-69636595-c7b4-451a-b26e-b9a67a5b1181',
    'ENDPOINT': 'https://s3-api.us-geo.objectstorage.service.networklayer.com',
    'IBM_AUTH_ENDPOINT': 'https://iam.bluemix.net/oidc/token',
    'BUCKET': 'textsummarizer-donotdelete-pr-jw3aso7wwj713f',
    'FILE': 'Lodge.txt'
}
# @hidden_cell
# The following code contains the credentials for a file in your IBM Cloud Object Storage.
# You might want to remove those credentials before you share your notebook.
credentials_7 = {
    'IBM_API_KEY_ID': '<KEY>',
    'IAM_SERVICE_ID': 'iam-ServiceId-69636595-c7b4-451a-b26e-b9a67a5b1181',
    'ENDPOINT': 'https://s3-api.us-geo.objectstorage.service.networklayer.com',
    'IBM_AUTH_ENDPOINT': 'https://iam.bluemix.net/oidc/token',
    'BUCKET': 'textsummarizer-donotdelete-pr-jw3aso7wwj713f',
    'FILE': 'Glazer-m.txt'
}
# @hidden_cell
# The following code contains the credentials for a file in your IBM Cloud Object Storage.
# You might want to remove those credentials before you share your notebook.
# @hidden_cell
# The following code contains the credentials for a file in your IBM Cloud Object Storage.
# You might want to remove those credentials before you share your notebook.
credentials_9 = {
    'IBM_API_KEY_ID': '<KEY>',
    'IAM_SERVICE_ID': 'iam-ServiceId-69636595-c7b4-451a-b26e-b9a67a5b1181',
    'ENDPOINT': 'https://s3-api.us-geo.objectstorage.service.networklayer.com',
    'IBM_AUTH_ENDPOINT': 'https://iam.bluemix.net/oidc/token',
    'BUCKET': 'textsummarizer-donotdelete-pr-jw3aso7wwj713f',
    'FILE': 'all.txt'
}
# @hidden_cell
# The following code contains the credentials for a file in your IBM Cloud Object Storage.
# You might want to remove those credentials before you share your notebook.
credentials_10 = {
    'IBM_API_KEY_ID': '<KEY>',
    'IAM_SERVICE_ID': 'iam-ServiceId-69636595-c7b4-451a-b26e-b9a67a5b1181',
    'ENDPOINT': 'https://s3-api.us-geo.objectstorage.service.networklayer.com',
    'IBM_AUTH_ENDPOINT': 'https://iam.bluemix.net/oidc/token',
    'BUCKET': 'textsummarizer-donotdelete-pr-jw3aso7wwj713f',
    'FILE': 'all.txt'
}

# -

# ### 1b. Functions to extract files from Cloud Object Storage

# +
cos = ibm_boto3.client('s3',
                    ibm_api_key_id=credentials_10['IBM_API_KEY_ID'],
                    ibm_service_instance_id=credentials_10['IAM_SERVICE_ID'],
                    ibm_auth_endpoint=credentials_10['IBM_AUTH_ENDPOINT'],
                    config=Config(signature_version='oauth'),
                    endpoint_url=credentials_10['ENDPOINT'])

def get_file(filename):
    '''Retrieve file from Cloud Object Storage'''
    fileobject = cos.get_object(Bucket=credentials_10['BUCKET'], Key=filename)['Body']
    return fileobject

def load_string(fileobject):
    '''Load the file contents into a Python string'''
    text = fileobject.read()
    return text


# -

# ### 1c. Get File Contents

text=str(load_string(get_file("Chavez.txt")))
text_2 = str(load_string(get_file("Richwine.txt")))
text3 = str(load_string(get_file("RichCha.txt")))
text4 = str(load_string(get_file("Frank.txt")))
text5 = str(load_string(get_file("Lodge.txt")))
text6 = str(load_string(get_file("Glazer-m.txt")))
text7 = str(load_string(get_file("all.txt")))

# ### 1d. Helper functions to extract summary and keywords

# +
'''Get the summary of the text'''

def get_summary(text, pct):
    summary = summarize(text,ratio=pct,split=True)
    return summary

'''Get the keywords of the text'''

def get_keywords(text):
    res = keywords(text, ratio=0.1, words=None, split=False, scores=False, pos_filter=('NN', 'JJ'), lemmatize=False, deacc=False)
    res = res.split('\n')
    return res

'''Tokenize the sentence into words & remove punctuation'''

def sent_to_words(sentences):
    for sentence in sentences:
        yield(gensim.utils.simple_preprocess(str(sentence), deacc=True))
        
def split_sentences(text):
    """ Split text into sentences.
    """
    sentence_delimiters = re.compile(u'[\\[\\]\n.!?]')
    sentences = sentence_delimiters.split(text)
    return sentences

def split_into_tokens(text):
    """ Split text into tokens.
    """
    tokens = nltk.word_tokenize(text)
    return tokens
    
def POS_tagging(text):
    """ Generate Part of speech tagging of the text.
    """
    POSofText = nltk.tag.pos_tag(text)
    return POSofText

def extract_title_text(url):
    page = urllib.request.urlopen(url).read().decode('utf8')
    soup = BeautifulSoup(page,'lxml')
    text = ' '.join(map(lambda p: p.text, soup.find_all('p')))
    return soup.title.text, text


# -

print('Printing Summary')
print('--------------------------')
print(get_summary(text7, 0.2))
print ('-------------------------')
print('Printing Keywords')
print('--------------------------')
print(get_keywords(text7))

# ### 2a. Remove punctuation & special characters

import re
my_new_text = re.sub('[^ a-zA-Z0-9]', '', text)

# ### 2b. Preprocess the text for next steps

stop_words = set(stopwords.words('english'))
lemma = WordNetLemmatizer()
word_tokens = word_tokenize(str(my_new_text)) 
filtered_sentence = [w for w in word_tokens if not w in stop_words]
normalized = " ".join(lemma.lemmatize(word) for word in filtered_sentence)

# ### 2c. Create n grams where n is the number of words

from nltk import ngrams
n = 7
total_grams = []
number_of_grams = ngrams(normalized.split(), n)
for grams in number_of_grams:
    total_grams.append(grams)

print(total_grams[:10])

# ### 2d. Create the wordcloud visualization on the processed data
# To highlight important textual data points & convey crucial information. The more a specific word appears in a source of textual data, the bigger and bolder it appears in the word cloud.

# +
from wordcloud import WordCloud
wordcloud = WordCloud(max_font_size=60).generate(normalized)
plt.figure(figsize=(16,12))

'''plot wordcloud in matplotlib'''

plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.show()
# -

# ### 2e. Analyze the frequency of words in the text. 

count = {}
for w in normalized.split():
    if w in count:
        count[w] += 1
    else:
        count[w] = 1
for word, times in count.items():
    if times > 3:
        print("%s was found %d times" % (word, times))

# ### 2f. Create a Dispersion plot
# The motivation behind using the Lexical Dispersion Plots was to give us an alternative means of visualising how prevalent  these words are in the text corpus, whether or not there was a clustering pattern that is whether or not a word featured heavily at certain point of the text corpus.

from nltk.book import text4 as content
plt.figure(figsize=(16,5))
topics = ['assimilation', 'Hispanics','ethnic', 'generation', 'economic', 'immigrant']
content.dispersion_plot(topics)

# ## 3. Topic Modelling

# ### 3a. Start the preprocessing for Topic Modelling
#
# Topic Modelling is an approach for finding topics in large amounts of text. Topic modeling is great for document clustering, information retrieval from unstructured text, and feature selection.
#  
# Topic Modeling with Latent Dirichlet Allocation technique.
#
# Why Latent Dirichlet Allocation? This technique can create model which can be generalized easily on any new text corpus and help us in identifying the important topics from the corpus. 
#
# Some of the advantages are :
#
# Training documents may come in sequentially, no random access required.
#
# Runs in constant memory w.r.t. the number of documents: size of the training corpus does not affect memory footprint, can process corpora larger than RAM.
#
# Is distributed & makes use of a cluster of machines, if available, to speed up model estimation.

# +
import gensim
from gensim import corpora

tokenized_sents = list(sent_to_words(filtered_sentence))

# Creating the term dictionary of our corpus, where every unique term is assigned an index. 
dictionary = corpora.Dictionary(tokenized_sents)

# Converting list of documents (corpus) into Document Term Matrix using dictionary prepared above.
doc_term_matrix = [dictionary.doc2bow(doc) for doc in tokenized_sents]
# -

# ### 3b. Creating the object for LDA model & train the model

# +

Lda = gensim.models.ldamodel.LdaModel

# Running and Training LDA model on the document term matrix by selecting minimum parameters required.
ldamodel = Lda(doc_term_matrix, num_topics=2, id2word = dictionary, passes=100)
# -

# ### 3c. Extract two topics with twenty words in each topic

print(ldamodel.print_topics(num_topics=6, num_words=300))

# ### 3d. Model perplexity and topic coherence provide a convenient measure to judge how good a given topic model is. 

# +
'''Compute Perplexity'''

# a measure of how good the model is. lower the better.

print('\nPerplexity: ', ldamodel.log_perplexity(doc_term_matrix))

'''Compute Coherence Score'''

coherence_model_lda = CoherenceModel(model=ldamodel, texts=tokenized_sents, dictionary=dictionary, coherence='c_v')
coherence_lda = coherence_model_lda.get_coherence()
print('\nCoherence Score: ', coherence_lda)
# -

# #### Coherence score is 'higher the better' metric and given the score of 0.86 we can be assured that we have selected the right number of topics for this corpus.

'''Visualize the topics'''
# pyLDAvis tool to visualize the fit of our LDA model across topics and their top words. 
pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(ldamodel, doc_term_matrix, dictionary)
vis

# #### We can observe that our LDA model has captured the prominent keywords under two topics in the text corpus which will give us a good understanding of what the text corpus is about. We can do further analysis by using this information to generate recommendations & classify the text for user profiling or push notifications. 

# ### We have seen how to summarize & visualize a document as well as a news article to get quick information about the data. This methodology can be applied to lot of usecases to extract insights from unstructured data.



StarcoderdataJupyterScriptsDedupFiltered